From eee9b7ea6d2e5e6a5bef706f9af070ca8611ff3b Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 11:15:43 -0700 Subject: [PATCH 001/108] build(svar2): dev-wire svar2-codec path-dep + genoray svar-2 wheel (M6b) Task 1 of the gvl two-source kernel plan. Dev-wiring only, release-gated: - svar2-codec: Cargo path-dep to the merged genoray svar-2 checkout. - genoray: pre-built cp310 wheel of genoray 2.15.0 (svar-2 with main merged), carrying both SparseVar2.overlap_batch and SparseVar2.decode. Wheel rather than editable to avoid pulling genoray's rust-htslib C toolchain into this env; Plan 3 does not modify genoray. - seqpro bumped ==0.20.0 -> ==0.21.1 to satisfy genoray 2.15.0's seqpro>=0.21.1,<0.22. Verified: `import genoray, genvarloader; from genoray import SparseVar2` OK (genoray 2.15.0); `cargo check` clean with svar2-codec linked from the path. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 5 + Cargo.toml | 1 + pixi.lock | 1569 +++++++++++++++++++++++++++++++++++----------------- pixi.toml | 9 +- 4 files changed, 1062 insertions(+), 522 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59c7c836..957d464f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,6 +449,7 @@ dependencies = [ "rayon", "rstest", "seqpro-core", + "svar2-codec", ] [[package]] @@ -1260,6 +1261,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "svar2-codec" +version = "0.1.0" + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 431165cd..d2c5196c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ ndarray = "0.17.2" numpy = "0.28.0" rayon = "1.12.0" seqpro-core = "0.1" +svar2-codec = { path = "/carter/users/dlaub/projects/genoray/svar2-codec" } [features] extension-module = ["pyo3/extension-module"] diff --git a/pixi.lock b/pixi.lock index 158e8a89..7d7bba80 100644 --- a/pixi.lock +++ b/pixi.lock @@ -164,48 +164,43 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - pypi: . + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -214,19 +209,23 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -338,50 +337,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -391,22 +385,26 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl dev: channels: @@ -548,33 +546,27 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - pypi: . + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -582,21 +574,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -605,12 +596,13 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl @@ -627,21 +619,26 @@ environments: - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl @@ -758,47 +755,42 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -807,22 +799,26 @@ environments: - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl docs: channels: @@ -1975,85 +1971,84 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.4.22-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-8_cp310.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f5/1a/60a505991247430fd9e4429526292e2b054381615801f3c7efdfa08dc9fb/hypothesis-6.156.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -2085,86 +2080,85 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a2/cf/e657b924ab6f24f29841ad2e461775a0f048c24013df5be5099be98303ca/hypothesis-6.156.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl notebook: channels: @@ -2433,47 +2427,42 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - pypi: . + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -2481,17 +2470,21 @@ environments: - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -2671,46 +2664,41 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -2718,19 +2706,23 @@ environments: - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py310: channels: @@ -2893,48 +2885,43 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - pypi: . + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -2943,19 +2930,23 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -3067,50 +3058,45 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -3120,22 +3106,26 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py311: channels: @@ -4599,55 +4589,48 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - pypi: . + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -4656,11 +4639,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl @@ -4676,18 +4660,23 @@ environments: - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl @@ -4872,46 +4861,41 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -4919,19 +4903,23 @@ environments: - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl packages: - conda: https://conda.anaconda.org/bioconda/linux-64/bcftools-1.23.1-hb2cee57_0.conda @@ -11525,6 +11513,39 @@ packages: - tbb ; extra == 'tbb' - pyomp ; extra == 'pyomp' requires_python: '>=3.10,<3.14' +- pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + name: genoray + version: 2.15.0 + requires_dist: + - seqpro>=0.21.1,<0.22 + - numpy>=1.26 + - pandas>=2.2.3 + - hirola>=0.3.0 + - pgenlib>=0.91.0 + - cyvcf2>=0.31.1 + - pysam>=0.22 + - polars>=1.37.1 + - polars-bio>=0.20.1,<0.21 + - pyranges>=0.1.3 + - typing-extensions>=4.14 + - pyarrow>=21 + - tqdm>=4.65 + - phantom-types>=3 + - more-itertools>=10 + - loguru>=0.7.0 + - attrs + - awkward + - numba + - cyclopts + - zstandard + - pydantic + - oxbow>=0.5.1,<0.6 + - joblib>=1.4.2,<2 + - joblib-progress>=1.0.6,<2 + - filelock>3,<4 + - scipy>=1.10 + - pooch>=1.7 + requires_python: '>=3.10,<3.14' - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.10.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl name: torch version: 2.10.0+cu126 @@ -11662,6 +11683,15 @@ packages: - ruff>=0.12.0 ; extra == 'dev' - cython-lint>=0.12.2 ; extra == 'dev' requires_python: '>=3.12' +- pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: yarl + version: 1.24.2 + sha256: f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90 + requires_dist: + - idna>=2.0 + - multidict>=4.0 + - propcache>=0.2.1 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl name: sphinx-book-theme version: 1.2.0 @@ -11815,30 +11845,6 @@ packages: - cudf-polars-cu12 ; extra == 'gpu' - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - name: zipp - version: 3.23.1 - sha256: 0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc - requires_dist: - - pytest>=6,!=8.1.* ; extra == 'test' - - jaraco-itertools ; extra == 'test' - - jaraco-functools ; extra == 'test' - - more-itertools ; extra == 'test' - - big-o ; extra == 'test' - - pytest-ignore-flaky ; extra == 'test' - - jaraco-test ; extra == 'test' - - sphinx>=3.5 ; extra == 'doc' - - jaraco-packaging>=9.3 ; extra == 'doc' - - rst-linker>=1.9 ; extra == 'doc' - - furo ; extra == 'doc' - - sphinx-lint ; extra == 'doc' - - jaraco-tidelift>=1.4 ; extra == 'doc' - - pytest-checkdocs>=2.4 ; extra == 'check' - - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' - - pytest-cov ; extra == 'cover' - - pytest-enabler>=2.2 ; extra == 'enabler' - - pytest-mypy ; extra == 'type' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: scipy version: 1.17.1 @@ -12346,11 +12352,6 @@ packages: version: '16.0' sha256: 9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - name: typing-extensions - version: 4.15.0 - sha256: f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/18/dc/1843828349729a86f8d9f79b19bd6e7eaa358a5682f13a0af667dae0c1d0/cyvcf2-0.32.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: cyvcf2 version: 0.32.1 @@ -12388,13 +12389,48 @@ packages: requires_dist: - numpy>=1.21.3 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl +- pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + name: seqpro + version: 0.21.1 + sha256: e40d7b20697685694a0e4ab7864f7cc93d784950d19b7160402c304e15ce725f + requires_dist: + - numba>=0.58.1 + - numpy>=1.26.0 + - polars>=1.21.0,<2 + - pyranges>=0.1.3,<0.2 + - pandera>=0.31.1 + - pandas + - pyarrow + - natsort + - narwhals>=2.20.0 + - setuptools>=70 + - awkward>=2.5.0 + - polars-config-meta[polars]>=0.3.2 + - attrs + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + name: idna + version: '3.18' + sha256: 7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-cufft-cu12 version: 11.3.3.83 sha256: 4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 requires_dist: - nvidia-nvjitlink-cu12 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: pgenlib + version: 0.94.1 + sha256: 1c267d592095c2363a3fb01e128740dea1b42e9a4a3a86bb0ab346ac7dda773b + requires_dist: + - numpy>=1.19.3 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl name: charset-normalizer version: 3.4.7 @@ -12453,6 +12489,13 @@ packages: - trio>=0.10.0 ; extra == 'trio' - pyyaml>=6.0.1 ; extra == 'yaml' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl + name: pgenlib + version: 0.94.1 + sha256: f6242c3afe0238d7180b6926e927c0626b3bded16c58af15732d829d6bd850c4 + requires_dist: + - numpy>=1.19.3 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl name: lightning-utilities version: 0.15.3 @@ -12483,13 +12526,6 @@ packages: - pytest ; extra == 'test' - defusedxml>=0.7.1 ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - name: arro3-core - version: 0.8.0 - sha256: 524e2ce13ea3d2739df4d52ea03977a53d103c1fd73f0fcc6a713903ea6ad4fa - requires_dist: - - typing-extensions ; python_full_version < '3.12' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl name: cslug version: 1.0.0 @@ -12528,6 +12564,25 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.11' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: aiohttp + version: 3.14.1 + sha256: 23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb + requires_dist: + - aiohappyeyeballs>=2.5.0 + - aiosignal>=1.4.0 + - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' + - attrs>=17.3.0 + - frozenlist>=1.1.1 + - multidict>=4.5,<7.0 + - propcache>=0.2.0 + - typing-extensions>=4.4 ; python_full_version < '3.13' + - yarl>=1.17.0,<2.0 + - aiodns>=3.3.0 ; sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotli>=1.2 ; platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' + - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl name: pandera version: 0.31.1 @@ -12647,24 +12702,6 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.12' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: aiohttp - version: 3.13.5 - sha256: bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95 - requires_dist: - - aiohappyeyeballs>=2.5.0 - - aiosignal>=1.4.0 - - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' - - attrs>=17.3.0 - - frozenlist>=1.1.1 - - multidict>=4.5,<7.0 - - propcache>=0.2.0 - - yarl>=1.17.0,<2.0 - - aiodns>=3.3.0 ; extra == 'speedups' - - brotli>=1.2 ; platform_python_implementation == 'CPython' and extra == 'speedups' - - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' - - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and extra == 'speedups' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/2f/97/9214bd9b860e680a281232e218d10b718a7280b593f4ab56240a558dc975/pgenlib-0.94.0-cp312-cp312-macosx_10_13_universal2.whl name: pgenlib version: 0.94.0 @@ -12811,6 +12848,13 @@ packages: version: 0.47.0 sha256: 74090f0dcfd6f24ebbef3f21f11e38111c4d7e6919b54c4416e1e357c3446b07 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: arro3-core + version: 0.8.1 + sha256: 31b966b0f9b2a6fb3f285339ed5ec4eac69591bbb92a9fdb2cf80ad995b759af + requires_dist: + - typing-extensions ; python_full_version < '3.12' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl name: sphinxcontrib-devhelp version: 2.0.0 @@ -12861,6 +12905,30 @@ packages: version: 0.5.2 sha256: b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl + name: zipp + version: 4.1.0 + sha256: 25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - jaraco-itertools ; extra == 'test' + - jaraco-functools ; extra == 'test' + - more-itertools ; extra == 'test' + - big-o ; extra == 'test' + - pytest-ignore-flaky ; extra == 'test' + - jaraco-test ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pytest-checkdocs>=2.14 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=3.4 ; extra == 'enabler' + - pytest-mypy>=1.0.1 ; platform_python_implementation != 'PyPy' and extra == 'type' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/3e/fe/1624eb5024e897bf4074bfc31f9e5e823160aed1ac14e7720e849a3d1109/selectolax-0.4.8-cp313-cp313-macosx_11_0_arm64.whl name: selectolax version: 0.4.8 @@ -13122,6 +13190,16 @@ packages: version: 12.6.80 sha256: 6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + name: typing-extensions + version: 4.16.0 + sha256: 481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl + name: filelock + version: 3.29.5 + sha256: 8af830889ba3a0ffcefbd6c7d2af8a54012058103771f2e10848222f476a1693 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl name: seqpro version: 0.20.0 @@ -13165,15 +13243,6 @@ packages: requires_dist: - selectolax>=0.3.29 requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: yarl - version: 1.23.0 - sha256: 31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 - requires_dist: - - idna>=2.0 - - multidict>=4.0 - - propcache>=0.2.1 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: fast-histogram version: '0.14' @@ -13183,6 +13252,13 @@ packages: - pytest ; extra == 'test' - hypothesis[numpy] ; extra == 'test' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: selectolax + version: 0.4.10 + sha256: cd436b0143edb5e1c4f126eeb4647344bedaab9e2a379eddb528ffac33c7adad + requires_dist: + - cython ; extra == 'cython' + requires_python: '>=3.9,<3.15' - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl name: sphinxcontrib-serializinghtml version: 2.0.0 @@ -13239,6 +13315,16 @@ packages: version: 0.24.0 sha256: 4a642f18649e59817de272173e9c27c031dceaca199809e4f8b338ebfc5d6698 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + name: vcfixture + version: 0.6.0 + sha256: 62e59f9ea7a1c8847057cff8f97c9e74588445afac15bd9dd43f34b7f8bb4ff9 + requires_dist: + - numpy>=1.24 + - pysam>=0.22 + - hypothesis>=6.100 + - typing-extensions>=4.4 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl name: ipywidgets version: 8.1.8 @@ -13298,13 +13384,6 @@ packages: - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and extra == 'speedups' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl - name: arro3-core - version: 0.8.0 - sha256: 29b3d1cbd2c4bac787f473d071e1eb02b71b2701a7118bb5d0a274ffbd26b16c - requires_dist: - - typing-extensions ; python_full_version < '3.12' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl name: tornado version: 6.5.5 @@ -13400,6 +13479,14 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl + name: typeguard + version: 4.5.2 + sha256: fcf9de18bd945cdb4c7b996e12b4c51ce83f92f191314a6d7cf1739586ec98cf + requires_dist: + - importlib-metadata>=3.6 ; python_full_version < '3.10' + - typing-extensions>=4.14.0 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl name: torch version: 2.10.0 @@ -13483,49 +13570,6 @@ packages: version: 1.8.0 sha256: f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl - name: hypothesis - version: 6.152.6 - sha256: b20ffc532e5f2901229348d10ed7cb37fd9723ebf4799df663d2dce1cdce4e32 - requires_dist: - - exceptiongroup>=1.0.0 ; python_full_version < '3.11' - - sortedcontainers>=2.1.0,<3.0.0 - - click>=7.0 ; extra == 'cli' - - black>=20.8b0 ; extra == 'cli' - - rich>=9.0.0 ; extra == 'cli' - - libcst>=0.3.16 ; extra == 'codemods' - - black>=20.8b0 ; extra == 'ghostwriter' - - pytz>=2014.1 ; extra == 'pytz' - - python-dateutil>=1.4 ; extra == 'dateutil' - - lark>=0.10.1 ; extra == 'lark' - - numpy>=1.21.6 ; extra == 'numpy' - - pandas>=1.1 ; extra == 'pandas' - - pytest>=4.6 ; extra == 'pytest' - - dpcontracts>=0.4 ; extra == 'dpcontracts' - - redis>=3.0.0 ; extra == 'redis' - - hypothesis-crosshair>=0.0.27 ; extra == 'crosshair' - - crosshair-tool>=0.0.102 ; extra == 'crosshair' - - tzdata>=2026.1 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') - - django>=4.2 ; extra == 'django' - - watchdog>=4.0.0 ; extra == 'watchdog' - - black>=20.8b0 ; extra == 'all' - - click>=7.0 ; extra == 'all' - - crosshair-tool>=0.0.102 ; extra == 'all' - - django>=4.2 ; extra == 'all' - - dpcontracts>=0.4 ; extra == 'all' - - hypothesis-crosshair>=0.0.27 ; extra == 'all' - - lark>=0.10.1 ; extra == 'all' - - libcst>=0.3.16 ; extra == 'all' - - numpy>=1.21.6 ; extra == 'all' - - pandas>=1.1 ; extra == 'all' - - pytest>=4.6 ; extra == 'all' - - python-dateutil>=1.4 ; extra == 'all' - - pytz>=2014.1 ; extra == 'all' - - redis>=3.0.0 ; extra == 'all' - - rich>=9.0.0 ; extra == 'all' - - tzdata>=2026.1 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') - - watchdog>=4.0.0 ; extra == 'all' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl name: imagesize version: 2.0.0 @@ -13561,13 +13605,6 @@ packages: requires_dist: - pytest ; extra == 'test' requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: awkward-cpp - version: '52' - sha256: cd7c09f29781805dc050329418168cc7568e0837d0adb2d523b7c3f45c187b90 - requires_dist: - - numpy>=1.21.3 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: zstandard version: 0.25.0 @@ -13866,6 +13903,11 @@ packages: - polars>=1.30 ; extra == 'polars' - pyarrow>=22.0.0 ; extra == 'pyarrow' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl + name: aiohappyeyeballs + version: 2.7.1 + sha256: 9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl name: beartype version: 0.22.9 @@ -14048,11 +14090,6 @@ packages: version: 12.6.77 sha256: 35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53 requires_python: '>=3' -- pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - name: platformdirs - version: 4.9.6 - sha256: e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl name: awkward version: 2.9.0 @@ -14147,13 +14184,6 @@ packages: version: 1.5.3 sha256: 5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - name: awkward-cpp - version: '52' - sha256: 610cec406c3f7c214e3e7673aaf55b5963e6c9695b97bd687b2f0f450ca66b64 - requires_dist: - - numpy>=1.21.3 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl name: polars-runtime-32 version: 1.37.1 @@ -14321,11 +14351,18 @@ packages: version: 0.1.4 sha256: 502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c requires_python: '>=3.6' -- pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl - name: filelock - version: 3.29.0 - sha256: 96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258 +- pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl + name: platformdirs + version: 4.10.0 + sha256: fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl + name: selectolax + version: 0.4.10 + sha256: 4d06ddf24ebcddead2c5a2ed96607f46651db899b8f331a16481d88a91d5c9af + requires_dist: + - cython ; extra == 'cython' + requires_python: '>=3.9,<3.15' - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl name: rich version: 15.0.0 @@ -14511,97 +14548,205 @@ packages: - typing-extensions>=4.6.0 ; python_full_version < '3.13' - pytest>=6 ; extra == 'test' requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - name: triton - version: 3.6.0 - sha256: a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea - requires_dist: - - importlib-metadata ; python_full_version < '3.10' - - cmake>=3.20,<4.0 ; extra == 'build' - - lit ; extra == 'build' - - autopep8 ; extra == 'tests' - - isort ; extra == 'tests' - - numpy ; extra == 'tests' - - pytest ; extra == 'tests' - - pytest-forked ; extra == 'tests' - - pytest-xdist ; extra == 'tests' - - scipy>=1.7.1 ; extra == 'tests' - - llnl-hatchet ; extra == 'tests' - - matplotlib ; extra == 'tutorials' - - pandas ; extra == 'tutorials' - - tabulate ; extra == 'tutorials' - requires_python: '>=3.10,<3.15' -- pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl - name: accessible-pygments - version: 0.0.5 - sha256: 88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7 - requires_dist: - - pygments>=1.5 - - pillow ; extra == 'dev' - - pkginfo>=1.10 ; extra == 'dev' - - playwright ; extra == 'dev' - - pre-commit ; extra == 'dev' - - setuptools ; extra == 'dev' - - twine>=5.0 ; extra == 'dev' - - hypothesis ; extra == 'tests' - - pytest ; extra == 'tests' - requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - name: pure-eval - version: 0.2.3 - sha256: 1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0 - requires_dist: - - pytest ; extra == 'tests' -- pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - name: scipy - version: 1.15.3 - sha256: 9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40 +- pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl + name: pytorch-lightning + version: 2.6.5 + sha256: 62d9c8549b2278fedc3364f0a5607a56c6063d18635008f8cf3fae8d802b0d76 requires_dist: - - numpy>=1.23.5,<2.5 - - pytest ; extra == 'test' - - pytest-cov ; extra == 'test' - - pytest-timeout ; extra == 'test' - - pytest-xdist ; extra == 'test' - - asv ; extra == 'test' - - mpmath ; extra == 'test' - - gmpy2 ; extra == 'test' - - threadpoolctl ; extra == 'test' - - scikit-umfpack ; extra == 'test' - - pooch ; extra == 'test' - - hypothesis>=6.30 ; extra == 'test' - - array-api-strict>=2.0,<2.1.1 ; extra == 'test' - - cython ; extra == 'test' - - meson ; extra == 'test' - - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx>=5.0.0,<8.0.0 ; extra == 'doc' - - intersphinx-registry ; extra == 'doc' - - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - - sphinx-copybutton ; extra == 'doc' - - sphinx-design>=0.4.0 ; extra == 'doc' - - matplotlib>=3.5 ; extra == 'doc' - - numpydoc ; extra == 'doc' - - jupytext ; extra == 'doc' - - myst-nb ; extra == 'doc' - - pooch ; extra == 'doc' - - jupyterlite-sphinx>=0.19.1 ; extra == 'doc' - - jupyterlite-pyodide-kernel ; extra == 'doc' - - mypy==1.10.0 ; extra == 'dev' - - typing-extensions ; extra == 'dev' - - types-psutil ; extra == 'dev' - - pycodestyle ; extra == 'dev' - - ruff>=0.0.292 ; extra == 'dev' - - cython-lint>=0.12.2 ; extra == 'dev' - - rich-click ; extra == 'dev' - - doit>=0.36.0 ; extra == 'dev' - - pydevtool ; extra == 'dev' + - torch>=2.1.0 + - tqdm>=4.57.0 + - pyyaml>5.4 + - fsspec[http]>=2022.5.0 + - torchmetrics>0.7.0 + - packaging>=23.0 + - typing-extensions>4.5.0 + - lightning-utilities>=0.10.0 + - coverage==7.13.4 ; python_full_version >= '3.10' and extra == 'test' + - coverage==7.10.7 ; python_full_version < '3.10' and extra == 'test' + - pytest==9.0.2 ; extra == 'test' + - pytest-cov==7.0.0 ; extra == 'test' + - pytest-timeout==2.4.0 ; extra == 'test' + - pytest-rerunfailures==16.0.1 ; python_full_version < '3.10' and extra == 'test' + - pytest-rerunfailures==16.1 ; python_full_version >= '3.10' and extra == 'test' + - pytest-random-order==1.2.0 ; extra == 'test' + - cloudpickle>=1.3 ; extra == 'test' + - scikit-learn>0.22.1 ; extra == 'test' + - numpy>1.21.0 ; python_full_version < '3.12' and extra == 'test' + - numpy>2.1.0 ; python_full_version >= '3.12' and extra == 'test' + - onnx>1.12.0 ; extra == 'test' + - onnxruntime>=1.12.0 ; extra == 'test' + - onnxscript>=0.1.0 ; extra == 'test' + - onnx-ir<0.1.16 ; extra == 'test' + - psutil<7.3.0 ; extra == 'test' + - pandas>2.0 ; extra == 'test' + - fastapi ; extra == 'test' + - uvicorn ; extra == 'test' + - tensorboard>=2.11 ; extra == 'test' + - huggingface-hub ; extra == 'test' + - requests<2.33.0 ; extra == 'examples' + - torchvision>=0.16.0 ; extra == 'examples' + - ipython[all]>=8.0.0 ; extra == 'examples' + - torchmetrics>=0.10.0 ; extra == 'examples' + - matplotlib>3.1 ; extra == 'extra' + - omegaconf>=2.2.3 ; extra == 'extra' + - hydra-core>=1.2.0 ; extra == 'extra' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'extra' + - rich>=12.3.0 ; extra == 'extra' + - tensorboardx>=2.2 ; extra == 'extra' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'extra' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'strategies' + - torch-tensorrt ; python_full_version >= '3.12' and sys_platform != 'darwin' and extra == 'test-gpu' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'deepspeed' + - matplotlib>3.1 ; extra == 'all' + - omegaconf>=2.2.3 ; extra == 'all' + - hydra-core>=1.2.0 ; extra == 'all' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'all' + - rich>=12.3.0 ; extra == 'all' + - tensorboardx>=2.2 ; extra == 'all' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'all' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'all' + - requests<2.33.0 ; extra == 'all' + - torchvision>=0.16.0 ; extra == 'all' + - ipython[all]>=8.0.0 ; extra == 'all' + - torchmetrics>=0.10.0 ; extra == 'all' + - matplotlib>3.1 ; extra == 'dev' + - omegaconf>=2.2.3 ; extra == 'dev' + - hydra-core>=1.2.0 ; extra == 'dev' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'dev' + - rich>=12.3.0 ; extra == 'dev' + - tensorboardx>=2.2 ; extra == 'dev' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'dev' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'dev' + - requests<2.33.0 ; extra == 'dev' + - torchvision>=0.16.0 ; extra == 'dev' + - ipython[all]>=8.0.0 ; extra == 'dev' + - torchmetrics>=0.10.0 ; extra == 'dev' + - coverage==7.13.4 ; python_full_version >= '3.10' and extra == 'dev' + - coverage==7.10.7 ; python_full_version < '3.10' and extra == 'dev' + - pytest==9.0.2 ; extra == 'dev' + - pytest-cov==7.0.0 ; extra == 'dev' + - pytest-timeout==2.4.0 ; extra == 'dev' + - pytest-rerunfailures==16.0.1 ; python_full_version < '3.10' and extra == 'dev' + - pytest-rerunfailures==16.1 ; python_full_version >= '3.10' and extra == 'dev' + - pytest-random-order==1.2.0 ; extra == 'dev' + - cloudpickle>=1.3 ; extra == 'dev' + - scikit-learn>0.22.1 ; extra == 'dev' + - numpy>1.21.0 ; python_full_version < '3.12' and extra == 'dev' + - numpy>2.1.0 ; python_full_version >= '3.12' and extra == 'dev' + - onnx>1.12.0 ; extra == 'dev' + - onnxruntime>=1.12.0 ; extra == 'dev' + - onnxscript>=0.1.0 ; extra == 'dev' + - onnx-ir<0.1.16 ; extra == 'dev' + - psutil<7.3.0 ; extra == 'dev' + - pandas>2.0 ; extra == 'dev' + - fastapi ; extra == 'dev' + - uvicorn ; extra == 'dev' + - tensorboard>=2.11 ; extra == 'dev' + - huggingface-hub ; extra == 'dev' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: selectolax - version: 0.4.8 - sha256: 239fb539f67a43cbd543d48a09314eb955b5f3fe1b9ede4e1a31feb7ccfdd6b0 +- pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: triton + version: 3.6.0 + sha256: a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' + - importlib-metadata ; python_full_version < '3.10' + - cmake>=3.20,<4.0 ; extra == 'build' + - lit ; extra == 'build' + - autopep8 ; extra == 'tests' + - isort ; extra == 'tests' + - numpy ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-forked ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - scipy>=1.7.1 ; extra == 'tests' + - llnl-hatchet ; extra == 'tests' + - matplotlib ; extra == 'tutorials' + - pandas ; extra == 'tutorials' + - tabulate ; extra == 'tutorials' + requires_python: '>=3.10,<3.15' +- pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl + name: accessible-pygments + version: 0.0.5 + sha256: 88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7 + requires_dist: + - pygments>=1.5 + - pillow ; extra == 'dev' + - pkginfo>=1.10 ; extra == 'dev' + - playwright ; extra == 'dev' + - pre-commit ; extra == 'dev' + - setuptools ; extra == 'dev' + - twine>=5.0 ; extra == 'dev' + - hypothesis ; extra == 'tests' + - pytest ; extra == 'tests' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: seqpro + version: 0.21.1 + sha256: 6b6720c75abff0f9e5826c4e6c09000d988b0b738a81172fdf226f6763b01170 + requires_dist: + - numba>=0.58.1 + - numpy>=1.26.0 + - polars>=1.21.0,<2 + - pyranges>=0.1.3,<0.2 + - pandera>=0.31.1 + - pandas + - pyarrow + - natsort + - narwhals>=2.20.0 + - setuptools>=70 + - awkward>=2.5.0 + - polars-config-meta[polars]>=0.3.2 + - attrs + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl + name: pure-eval + version: 0.2.3 + sha256: 1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0 + requires_dist: + - pytest ; extra == 'tests' +- pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: scipy + version: 1.15.3 + sha256: 9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40 + requires_dist: + - numpy>=1.23.5,<2.5 + - pytest ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-xdist ; extra == 'test' + - asv ; extra == 'test' + - mpmath ; extra == 'test' + - gmpy2 ; extra == 'test' + - threadpoolctl ; extra == 'test' + - scikit-umfpack ; extra == 'test' + - pooch ; extra == 'test' + - hypothesis>=6.30 ; extra == 'test' + - array-api-strict>=2.0,<2.1.1 ; extra == 'test' + - cython ; extra == 'test' + - meson ; extra == 'test' + - ninja ; sys_platform != 'emscripten' and extra == 'test' + - sphinx>=5.0.0,<8.0.0 ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' + - sphinx-copybutton ; extra == 'doc' + - sphinx-design>=0.4.0 ; extra == 'doc' + - matplotlib>=3.5 ; extra == 'doc' + - numpydoc ; extra == 'doc' + - jupytext ; extra == 'doc' + - myst-nb ; extra == 'doc' + - pooch ; extra == 'doc' + - jupyterlite-sphinx>=0.19.1 ; extra == 'doc' + - jupyterlite-pyodide-kernel ; extra == 'doc' + - mypy==1.10.0 ; extra == 'dev' + - typing-extensions ; extra == 'dev' + - types-psutil ; extra == 'dev' + - pycodestyle ; extra == 'dev' + - ruff>=0.0.292 ; extra == 'dev' + - cython-lint>=0.12.2 ; extra == 'dev' + - rich-click ; extra == 'dev' + - doit>=0.36.0 ; extra == 'dev' + - pydevtool ; extra == 'dev' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-cufft-cu12 version: 11.3.0.4 @@ -14920,6 +15065,13 @@ packages: requires_dist: - cython ; extra == 'cython' requires_python: '>=3.9,<3.15' +- pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: awkward-cpp + version: '54' + sha256: 0c9fae1b68b9821b375e6da9de00b90387509c1ec0be8c3b19e7d0d8f54a79b0 + requires_dist: + - numpy>=1.21.3 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl name: setuptools version: 82.0.1 @@ -15087,6 +15239,18 @@ packages: version: 1.6.0 sha256: 87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c requires_python: '>=3.5' +- pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + name: requests + version: 2.34.2 + sha256: 2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 + requires_dist: + - charset-normalizer>=2,<4 + - idna>=2.5,<4 + - urllib3>=1.26,<3 + - certifi>=2023.5.7 + - pysocks>=1.5.6,!=1.5.7 ; extra == 'socks' + - chardet>=3.0.2,<8 ; extra == 'use-chardet-on-py3' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl name: sympy version: 1.14.0 @@ -15096,6 +15260,56 @@ packages: - pytest>=7.1.0 ; extra == 'dev' - hypothesis>=6.70.0 ; extra == 'dev' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl + name: arro3-core + version: 0.8.1 + sha256: 0501662cb3c7c6a05dcd6b998db55f9a97743aa571da42203733366c42166a7a + requires_dist: + - typing-extensions ; python_full_version < '3.12' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a2/cf/e657b924ab6f24f29841ad2e461775a0f048c24013df5be5099be98303ca/hypothesis-6.156.1-cp310-cp310-macosx_11_0_arm64.whl + name: hypothesis + version: 6.156.1 + sha256: ad6e5960b36452f3b904a3849e85c62377cf4ecc904541706c934dae439ce933 + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-nvtx-cu12 version: 12.8.90 @@ -15254,13 +15468,6 @@ packages: - pytz ; extra == 'test' - pandas ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - name: click - version: 8.3.3 - sha256: a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613 - requires_dist: - - colorama ; sys_platform == 'win32' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl name: pydantic-core version: 2.46.4 @@ -15805,6 +16012,13 @@ packages: version: 2.4.4 sha256: 23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + name: awkward-cpp + version: '54' + sha256: 38c4d810a7ac7a8ed185a90a7e252e8bb240db20628284ef7a2b7bc8e046fda8 + requires_dist: + - numpy>=1.21.3 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/c6/3e/48599a38009ca60ff82a6f38c8a621ce3c0286aa7397c7d79e741bd9060e/polars_runtime_32-1.40.1-cp310-abi3-macosx_11_0_arm64.whl name: polars-runtime-32 version: 1.40.1 @@ -15844,15 +16058,76 @@ packages: version: 3.0.1 sha256: 6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064 requires_python: '!=3.0.*,!=3.1.*,!=3.2.*' -- pypi: https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl - name: pyarrow - version: 21.0.0 - sha256: 3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd +- pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + name: pandera + version: 0.32.1 + sha256: 1a17a3ffa906174d19207715f4f082ec3db3709647927ad8c095c147d74d8454 requires_dist: - - pytest ; extra == 'test' - - hypothesis ; extra == 'test' - - cffi ; extra == 'test' - - pytz ; extra == 'test' + - packaging>=20.0 + - pydantic + - typeguard + - typing-extensions + - typing-inspect>=0.6.0 + - numpy>=1.24.4 ; extra == 'pandas' + - pandas>=2.1.1 ; extra == 'pandas' + - hypothesis>=6.92.7 ; extra == 'strategies' + - scipy ; extra == 'hypotheses' + - pyyaml>=5.1 ; extra == 'io' + - frictionless<=4.40.8 ; extra == 'frictionless' + - pandas-stubs ; extra == 'mypy' + - scipy-stubs ; python_full_version >= '3.10' and extra == 'mypy' + - fastapi ; extra == 'fastapi' + - geopandas ; extra == 'geopandas' + - shapely ; extra == 'geopandas' + - pyspark[connect]>=3.2.0 ; extra == 'pyspark' + - modin ; extra == 'modin' + - ray ; extra == 'modin' + - dask[dataframe] ; extra == 'modin' + - distributed ; extra == 'modin' + - modin ; extra == 'modin-ray' + - ray ; extra == 'modin-ray' + - modin ; extra == 'modin-dask' + - dask[dataframe] ; extra == 'modin-dask' + - distributed ; extra == 'modin-dask' + - dask[dataframe] ; extra == 'dask' + - distributed ; extra == 'dask' + - ibis-framework>=9.0.0 ; extra == 'ibis' + - pyarrow-hotfix ; extra == 'ibis' + - polars>=0.20.0 ; extra == 'polars' + - xarray>=2024.10.0 ; extra == 'xarray' + - numpy>=1.24.4 ; extra == 'xarray' + - narwhals>=1.26.0 ; extra == 'narwhals' + - hypothesis>=6.92.7 ; extra == 'all' + - scipy ; extra == 'all' + - scipy-stubs ; python_full_version >= '3.10' and extra == 'all' + - pyyaml>=5.1 ; extra == 'all' + - black ; extra == 'all' + - frictionless<=4.40.8 ; extra == 'all' + - pyspark[connect]>=3.2.0 ; extra == 'all' + - modin ; extra == 'all' + - ray ; extra == 'all' + - dask[dataframe] ; extra == 'all' + - distributed ; extra == 'all' + - pandas-stubs ; extra == 'all' + - fastapi ; extra == 'all' + - geopandas ; extra == 'all' + - shapely ; extra == 'all' + - ibis-framework>=9.0.0 ; extra == 'all' + - pyarrow-hotfix ; extra == 'all' + - polars>=0.20.0 ; extra == 'all' + - xarray>=2024.10.0 ; extra == 'all' + - numpy>=1.24.4 ; extra == 'all' + - narwhals>=1.26.0 ; extra == 'all' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl + name: pyarrow + version: 21.0.0 + sha256: 3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd + requires_dist: + - pytest ; extra == 'test' + - hypothesis ; extra == 'test' + - cffi ; extra == 'test' + - pytz ; extra == 'test' - pandas ; extra == 'test' requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl @@ -16028,6 +16303,21 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl + name: rich-rst + version: 2.0.2 + sha256: 7bc2923929c9bbc61aadfef2069b742446823b306af84cf9e0f56b965a3d3035 + requires_dist: + - rich>=12.0.0 + - pygments>=2.0.0 + - docutils ; extra == 'docs' + - rich ; extra == 'docs' + - rich-rst ; extra == 'docs' + - sphinx ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' - pypi: https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: watchfiles version: 1.1.1 @@ -16045,6 +16335,11 @@ packages: version: 0.24.0 sha256: b9445a4c3be5ed4b60202690af3890a444452276372e3abb58564308cc6d5a45 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl + name: cuda-pathfinder + version: 1.5.6 + sha256: 7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/d2/89/b6d8c4ec1425b81ed588fa3ecc1b4ed5b9b8da21894eba5ac5d5288881b2/cyvcf2-0.32.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: cyvcf2 version: 0.32.1 @@ -16254,6 +16549,20 @@ packages: - httpx ; extra == 'test' - pytest>=6 ; extra == 'test' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl + name: tqdm + version: 4.68.3 + sha256: 39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03 + requires_dist: + - colorama ; sys_platform == 'win32' + - requests ; extra == 'discord' + - envwrap ; extra == 'discord' + - slack-sdk ; extra == 'slack' + - envwrap ; extra == 'slack' + - requests ; extra == 'telegram' + - envwrap ; extra == 'telegram' + - ipywidgets>=6 ; extra == 'notebook' + requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl name: pillow version: 12.2.0 @@ -16286,13 +16595,6 @@ packages: - trove-classifiers>=2024.10.12 ; extra == 'tests' - defusedxml ; extra == 'xmp' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: pgenlib - version: 0.94.0 - sha256: 5a9c26fc0de6282c82d95697b08b9f5fba2473b048952d810c9d01e094058861 - requires_dist: - - numpy>=1.19.3 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl name: oxbow version: 0.5.2 @@ -16409,6 +16711,45 @@ packages: - ruff>=0.12.0 ; extra == 'dev' - cython-lint>=0.12.2 ; extra == 'dev' requires_python: '>=3.12' +- pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + name: cyclopts + version: 4.20.0 + sha256: 0b4337e9c11303d86b33d3f37c629dc01638f84591681e0e5611286bdd507646 + requires_dist: + - attrs>=23.1.0 + - docstring-parser>=0.15,<4.0 + - rich-rst>=1.3.1,<3.0.0 + - rich>=13.6.0 + - tomli>=2.0.0 ; python_full_version < '3.11' + - typing-extensions>=4.8.0 ; python_full_version < '3.11' + - ipdb>=0.13.9 ; extra == 'debug' + - line-profiler>=3.5.1 ; extra == 'debug' + - coverage[toml]>=5.1 ; extra == 'dev' + - mkdocs>=1.4.0 ; extra == 'dev' + - pexpect>=4.9.0 ; sys_platform != 'win32' and extra == 'dev' + - pre-commit>=2.16.0 ; extra == 'dev' + - pydantic>=2.11.2,<3.0.0 ; extra == 'dev' + - pytest-cov>=3.0.0 ; extra == 'dev' + - pytest-mock>=3.7.0 ; extra == 'dev' + - pytest>=8.2.0 ; extra == 'dev' + - pyyaml>=6.0.1 ; extra == 'dev' + - syrupy>=4.0.0 ; extra == 'dev' + - toml>=0.10.2,<1.0.0 ; extra == 'dev' + - trio>=0.10.0 ; extra == 'dev' + - gitpython>=3.1.31 ; extra == 'docs' + - myst-parser[linkify]>=3.0.1,<5.0.0 ; extra == 'docs' + - sphinx-autodoc-typehints>=1.25.2,<4.0.0 ; extra == 'docs' + - sphinx-copybutton>=0.5,<1.0 ; extra == 'docs' + - sphinx-rtd-dark-mode>=1.3.0,<2.0.0 ; extra == 'docs' + - sphinx-rtd-theme>=3.0.0,<4.0.0 ; extra == 'docs' + - sphinx>=7.4.7,<8.2.0 ; extra == 'docs' + - markdown>=3.3 ; extra == 'mkdocs' + - mkdocs>=1.4.0 ; extra == 'mkdocs' + - pymdown-extensions>=10.0 ; extra == 'mkdocs' + - tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'toml' + - trio>=0.10.0 ; extra == 'trio' + - pyyaml>=6.0.1 ; extra == 'yaml' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl name: packaging version: '26.2' @@ -16476,13 +16817,114 @@ packages: - babel ; extra == 'i18n' - jinja2 ; extra == 'i18n' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl - name: selectolax - version: 0.4.8 - sha256: 4542a35f5ea993ae8bdb5850346f65848794afabc83976e576dcab07a6699020 +- pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + name: fsspec + version: 2026.6.0 + sha256: 02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1 requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' + - adlfs ; extra == 'abfs' + - adlfs ; extra == 'adl' + - pyarrow>=1 ; extra == 'arrow' + - dask ; extra == 'dask' + - distributed ; extra == 'dask' + - pre-commit ; extra == 'dev' + - ruff>=0.5 ; extra == 'dev' + - numpydoc ; extra == 'doc' + - sphinx ; extra == 'doc' + - sphinx-design ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - yarl ; extra == 'doc' + - dropbox ; extra == 'dropbox' + - dropboxdrivefs ; extra == 'dropbox' + - requests ; extra == 'dropbox' + - adlfs ; extra == 'full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'full' + - dask ; extra == 'full' + - distributed ; extra == 'full' + - dropbox ; extra == 'full' + - dropboxdrivefs ; extra == 'full' + - fusepy ; extra == 'full' + - gcsfs>2024.2.0 ; extra == 'full' + - libarchive-c ; extra == 'full' + - ocifs ; extra == 'full' + - panel ; extra == 'full' + - paramiko ; extra == 'full' + - pyarrow>=1 ; extra == 'full' + - pygit2 ; extra == 'full' + - requests ; extra == 'full' + - s3fs>2024.2.0 ; extra == 'full' + - smbprotocol ; extra == 'full' + - tqdm ; extra == 'full' + - fusepy ; extra == 'fuse' + - gcsfs>2024.2.0 ; extra == 'gcs' + - pygit2 ; extra == 'git' + - requests ; extra == 'github' + - gcsfs ; extra == 'gs' + - panel ; extra == 'gui' + - pyarrow>=1 ; extra == 'hdfs' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'http' + - libarchive-c ; extra == 'libarchive' + - ocifs ; extra == 'oci' + - s3fs>2024.2.0 ; extra == 's3' + - paramiko ; extra == 'sftp' + - smbprotocol ; extra == 'smb' + - paramiko ; extra == 'ssh' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test' + - numpy ; extra == 'test' + - pytest ; extra == 'test' + - pytest-asyncio!=0.22.0 ; extra == 'test' + - pytest-benchmark ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytest-recording ; extra == 'test' + - pytest-rerunfailures ; extra == 'test' + - requests ; extra == 'test' + - aiobotocore>=2.5.4,<3.0.0 ; extra == 'test-downstream' + - dask[dataframe,test] ; extra == 'test-downstream' + - moto[server]>4,<5 ; extra == 'test-downstream' + - pytest-timeout ; extra == 'test-downstream' + - xarray ; extra == 'test-downstream' + - adlfs ; extra == 'test-full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test-full' + - backports-zstd ; python_full_version < '3.14' and extra == 'test-full' + - cloudpickle ; extra == 'test-full' + - dask ; extra == 'test-full' + - distributed ; extra == 'test-full' + - dropbox ; extra == 'test-full' + - dropboxdrivefs ; extra == 'test-full' + - fastparquet ; extra == 'test-full' + - fusepy ; extra == 'test-full' + - gcsfs ; extra == 'test-full' + - jinja2 ; extra == 'test-full' + - kerchunk ; extra == 'test-full' + - libarchive-c ; extra == 'test-full' + - lz4 ; extra == 'test-full' + - notebook ; extra == 'test-full' + - numpy ; extra == 'test-full' + - ocifs ; extra == 'test-full' + - pandas<3.0.0 ; extra == 'test-full' + - panel ; extra == 'test-full' + - paramiko ; extra == 'test-full' + - pyarrow ; extra == 'test-full' + - pyarrow>=1 ; extra == 'test-full' + - pyftpdlib ; extra == 'test-full' + - pygit2 ; extra == 'test-full' + - pytest ; extra == 'test-full' + - pytest-asyncio!=0.22.0 ; extra == 'test-full' + - pytest-benchmark ; extra == 'test-full' + - pytest-cov ; extra == 'test-full' + - pytest-mock ; extra == 'test-full' + - pytest-recording ; extra == 'test-full' + - pytest-rerunfailures ; extra == 'test-full' + - python-snappy ; extra == 'test-full' + - requests ; extra == 'test-full' + - smbprotocol ; extra == 'test-full' + - tqdm ; extra == 'test-full' + - urllib3 ; extra == 'test-full' + - zarr<3.2.0 ; extra == 'test-full' + - zstandard ; python_full_version < '3.14' and extra == 'test-full' + - tqdm ; extra == 'tqdm' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl name: tinycss2 version: 1.4.0 @@ -16531,6 +16973,11 @@ packages: - pytest-timeout ; extra == 'test' - pytest<9 ; extra == 'test' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + name: more-itertools + version: 11.1.0 + sha256: 4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/e8/b3/7b0845952f65ecd3f02e8573ece4dd17c97ca27604cd28302c0467293e83/pgenlib-0.94.0-cp311-cp311-macosx_10_9_universal2.whl name: pgenlib version: 0.94.0 @@ -16656,6 +17103,23 @@ packages: name: pytz version: '2026.2' sha256: 04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 +- pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + name: awkward + version: 2.10.0 + sha256: 808c9536790865cbbcde07cb3678caa86c0369c443a3212a4552ec10dd187140 + requires_dist: + - awkward-cpp==54 + - fsspec>=2022.11.0 + - importlib-metadata>=4.13.0 ; python_full_version < '3.12' + - numpy>=1.21.3 + - packaging + - typing-extensions>=4.1.0 ; python_full_version < '3.11' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl + name: certifi + version: 2026.6.17 + sha256: 2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl name: natsort version: 8.4.0 @@ -16699,13 +17163,6 @@ packages: - nvidia-nvjitlink-cu12 - nvidia-cusparse-cu12 requires_python: '>=3' -- pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - name: pgenlib - version: 0.94.0 - sha256: ffaf1e9b8baa05da40213e12b950d4e612097d8fa81a921f2a2a797880b5f871 - requires_dist: - - numpy>=1.19.3 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl name: stack-data version: 0.6.3 @@ -16730,6 +17187,28 @@ packages: name: webencodings version: 0.5.1 sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 +- pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl + name: narwhals + version: 2.23.0 + sha256: 769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9 + requires_dist: + - cudf-cu12>=24.10.0 ; sys_platform == 'linux' and extra == 'cudf' + - dask[dataframe]>=2024.8 ; extra == 'dask' + - duckdb>=1.1 ; extra == 'duckdb' + - ibis-framework>=6.0.0 ; extra == 'ibis' + - rich>=12.4.4 ; extra == 'ibis' + - packaging>=21.3 ; extra == 'ibis' + - pyarrow-hotfix>=0.7 ; extra == 'ibis' + - modin>=0.22.0 ; extra == 'modin' + - pandas>=1.3.4 ; extra == 'pandas' + - polars>=0.20.4 ; extra == 'polars' + - pyarrow>=13.0.0 ; extra == 'pyarrow' + - pyspark>=3.5.0 ; extra == 'pyspark' + - pyspark[connect]>=3.5.0 ; extra == 'pyspark-connect' + - narwhals[duckdb] ; extra == 'sql' + - sqlparse>=0.5.5 ; extra == 'sql' + - sqlframe>=3.22.0,!=3.39.3 ; extra == 'sqlframe' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl name: pygments version: 2.20.0 @@ -16737,6 +17216,49 @@ packages: requires_dist: - colorama>=0.4.6 ; extra == 'windows-terminal' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/f5/1a/60a505991247430fd9e4429526292e2b054381615801f3c7efdfa08dc9fb/hypothesis-6.156.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: hypothesis + version: 6.156.1 + sha256: 63131c150146b44f49dd6be4d417e1ee62390951403569043e759e8513763426 + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl name: nvidia-nvjitlink-cu12 version: 12.8.93 @@ -16869,6 +17391,13 @@ packages: version: 10.3.9.90 sha256: b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + name: click + version: 8.4.2 + sha256: e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + requires_dist: + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/fb/ec/b0c23ec7fc9df5af527b2d63f15a92699f7fd0515986763ed8e50489a755/ncls-0.0.70-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: ncls version: 0.0.70 diff --git a/pixi.toml b/pixi.toml index e05e09ed..18df7eeb 100644 --- a/pixi.toml +++ b/pixi.toml @@ -93,8 +93,13 @@ numba = "==0.59.1" [feature.py310.pypi-dependencies] pyarrow = ">=21" hirola = "==0.3" -seqpro = "==0.20.0" -genoray = "==2.12.3" +seqpro = "==0.21.1" +# Dev-wiring (release-gated): genoray svar-2 (v2.15.0, main merged in) as a pre-built +# cp310 wheel. Editable path-install would require genoray's rust-htslib C toolchain +# (cxx-compiler/clangdev/zlib/LIBCLANG_PATH) inside this env; Plan 3 never edits genoray, +# so a frozen wheel is equivalent. Rebuild via `pixi run -e py310 maturin build --release` +# in the genoray checkout if genoray changes. Flip to the PyPI release at merge. +genoray = { path = "/carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl" } polars = "==1.37.1" loguru = "*" natsort = "*" From 2226d2e64f49cb74c3b28c675bd5cfad1a0bca16 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 11:34:07 -0700 Subject: [PATCH 002/108] =?UTF-8?q?feat(svar2):=20per-hap=20var=5Fkey?= =?UTF-8?q?=E2=8B=88dense=20merge=20+=20key=20decode=20source=20(gvl=20M6b?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 2: src/svar2/mod.rs with decode_alt (mirrors genoray decode_keyref: Inline→v_diff=len-1 owned; PureDel→raw negative ilen, empty; Lookup→LUT-borrowed, v_diff=len-1) and merge_hap (stable position sort, var_key before dense on ties). Adds pub mod svar2; to src/lib.rs. Unit tests cover PureDel/Lookup decode and the merge tie-break. cargo test --no-default-features svar2 → 2 passed. Co-Authored-By: Claude Opus 4.8 --- src/lib.rs | 1 + src/svar2/mod.rs | 108 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 src/svar2/mod.rs diff --git a/src/lib.rs b/src/lib.rs index 096545ef..19ac9f78 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod ragged; pub mod reconstruct; pub mod reference; pub mod reverse; +pub mod svar2; pub mod tables; pub mod tracks; pub mod variants; diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs new file mode 100644 index 00000000..5e545d9a --- /dev/null +++ b/src/svar2/mod.rs @@ -0,0 +1,108 @@ +//! SVAR2 two-source variant provider: merge a hap's var_key ⋈ dense channels and +//! decode keys via svar2-codec, feeding the reconstruction kernel with no +//! intermediate variant table. Additive to the SVAR 1.0 global-table path. + +use std::borrow::Cow; + +use svar2_codec::{decode_key, DecodedKey}; + +/// Decode one uniform key into `(v_diff, allele)`, resolving long-INS via the LUT +/// arrays. Mirrors genoray's `decode_keyref`. +pub fn decode_alt<'a>(key: u32, lut_bytes: &'a [u8], lut_off: &[i64]) -> (i64, Cow<'a, [u8]>) { + match decode_key(key) { + DecodedKey::Inline { alt } => (alt.len() as i64 - 1, Cow::Owned(alt)), + DecodedKey::PureDel { ilen } => (ilen as i64, Cow::Borrowed(&[][..])), + DecodedKey::Lookup { row } => { + let s = lut_off[row as usize] as usize; + let e = lut_off[row as usize + 1] as usize; + let alt = &lut_bytes[s..e]; + (alt.len() as i64 - 1, Cow::Borrowed(alt)) + } + } +} + +/// Merge one hap's `var_key` slice with its carried `dense` set-bits into a single +/// position-sorted `(pos, key)` list (stable: var_key before dense on ties, matching +/// genoray's merge). `dense` is region `r`'s `[ds, de)` window; `present` are this hap's +/// LSB-first presence bits over that window. +#[allow(clippy::too_many_arguments)] +pub fn merge_hap( + vk_pos: &[i32], + vk_key: &[i32], + vk_lo: usize, + vk_hi: usize, + dense_pos: &[i32], + dense_key: &[i32], + ds: usize, + de: usize, + present_bit: impl Fn(usize) -> bool, // present_bit(k) for k in 0..(de-ds) +) -> Vec<(u32, u32)> { + let mut a: Vec<(u32, u32)> = (vk_lo..vk_hi) + .map(|i| (vk_pos[i] as u32, vk_key[i] as u32)) + .collect(); + for (k, j) in (ds..de).enumerate() { + if present_bit(k) { + a.push((dense_pos[j] as u32, dense_key[j] as u32)); + } + } + a.sort_by_key(|&(p, _)| p); // stable; var_key pushed first keeps it ahead on ties + a +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_decode_inline_del_lookup() { + // Pure DEL of length 2 → v_diff -2, empty allele. + let del = svar2_codec::encode_pure_del(-2); + let (d, a) = decode_alt(del, &[], &[0]); + assert_eq!(d, -2); + assert!(a.is_empty()); + + // Lookup row 0 → "ACGT" from the LUT (v_diff = len-1 = 3). + let lut = b"ACGT".to_vec(); + let off = vec![0i64, 4]; + let lk = svar2_codec::encode_lookup(0); + let (d, a) = decode_alt(lk, &lut, &off); + assert_eq!(d, 3); + assert_eq!(a.as_ref(), b"ACGT"); + } + + #[test] + fn test_merge_hap_position_sorted_var_key_before_dense_on_tie() { + // var_key entries for this hap (positions 10 and 20; 20 ties with a dense entry). + let vk_pos = [10i32, 20]; + let vk_key = [100i32, 200]; + + // dense channel spans multiple haps/regions; this hap's window is [ds, de). + // dense positions: 15, 20 (ties with vk_pos[1]=20), 30. + let dense_pos = [15i32, 20, 30]; + let dense_key = [150i32, 250, 300]; + let ds = 0usize; + let de = 3usize; + + // Present bits (LSB-first) over the window: all three dense entries present. + let present = [true, true, true]; + let present_bit = |k: usize| present[k]; + + let merged = merge_hap( + &vk_pos, + &vk_key, + 0, + vk_pos.len(), + &dense_pos, + &dense_key, + ds, + de, + present_bit, + ); + + // Expect position-sorted (pos, key): 10, 15, 20 (var_key first on tie), 20 (dense), 30. + assert_eq!( + merged, + vec![(10, 100), (15, 150), (20, 200), (20, 250), (30, 300)] + ); + } +} From 9241ffe8c410b284708c8882def99c4dad2b8dd0 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 12:19:30 -0700 Subject: [PATCH 003/108] feat(svar2): two-source reconstruct driver via closure source (gvl M6b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 3. Refactor the reconstruction inner kernel to a variant-source closure and add the SVAR2 two-source batch driver, keeping SVAR 1.0 byte-identical. - reconstruct_haplotype_core<'a>(n_variants, provide: Fn(usize)->(v_pos, v_diff, Cow<'a,[u8]>, annot_id), ...) holds the (unchanged) reconstruction math; the per-variant reads and the annotation id now come from `provide`. - reconstruct_haplotype_from_sparse: thin wrapper building the global-table closure (Cow::Borrowed alt_flat, annot_id = global variant id). Behavior unchanged; 17 SVAR1 reconstruct tests + full lib (114) pass. - reconstruct_haplotypes_from_svar2: new driver mirroring the SVAR1 batch driver's parallel/serial disjoint-slice carving; per hap builds the var_key⋈dense merged list (svar2::merge_hap) and decodes via svar2::decode_alt; annot_id = per-hap seq. - Pure-DEL anchor fix: genoray decodes a pure DEL to an EMPTY ALT (anchor recovered from reference); the kernel needs the anchor byte, so the driver substitutes ref[pos] when the decoded allele is empty (⟺ PureDel). Without it the anchor is deleted too (off-by-one). Guarded by svar2_pure_del_keeps_anchor + the corrected svar2_reconstruct_snp_and_del. Reviewed (Opus): Spec ✅, Approved, no Critical/Important; 3 Minor (coverage of multi-hap/parallel + Lookup/unset-presence end-to-end — covered by Task 7; degenerate out-of-bounds PureDel pos). Co-Authored-By: Claude Opus 4.8 --- src/reconstruct/mod.rs | 731 +++++++++++++++++++++++++++++++++++------ 1 file changed, 624 insertions(+), 107 deletions(-) diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index 4b77ea77..d6f71de4 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -5,34 +5,41 @@ use ndarray::{s, ArrayView1, ArrayView2, ArrayViewMut1}; use rayon::prelude::*; -/// Reconstruct a single haplotype from reference sequence and variants. +/// Single-haplotype inner kernel, generic over the variant source. /// -/// Single-haplotype inner kernel. Mirror of numba -/// `reconstruct_haplotype_from_sparse` (`_genotypes.py:277-465`). +/// Mirror of numba `reconstruct_haplotype_from_sparse` (`_genotypes.py:277-465`), with +/// the per-variant reads factored out behind the `provide` closure so the same loop +/// body can be driven by either the SVAR 1.0 global variant table +/// (`reconstruct_haplotype_from_sparse`) or the SVAR2 two-channel (var_key ⋈ dense) +/// source (`reconstruct_haplotypes_from_svar2`). **Do not change the reconstruction +/// math here** — only the data source differs between call sites. /// /// # Parameters -/// - `v_idxs` – indices into the full variant table for this haplotype (i32) -/// - `v_starts` – genomic start position of each variant (i32, indexed by variant) -/// - `ilens` – insertion-length (ilen = alt_len − ref_len + 1) per variant (i32) +/// - `n_variants` – number of variants this haplotype sees, i.e. `0..n_variants` are +/// valid indices into `provide` +/// - `provide` – `Fn(v) -> (v_pos, v_diff, allele, annot_id)` for `v in 0..n_variants`: +/// `v_pos` genomic start, `v_diff` ilen (`alt_len - ref_len` for +/// atomized variants), `allele` the full ALT allele bytes (borrowed or +/// owned via `Cow`), `annot_id` the value written into `annot_v_idxs` +/// when the variant is applied. The named lifetime `'a` ties the +/// returned `Cow`'s borrow to the driver-call scope (`alt_flat` for +/// SVAR1; `lut_bytes` for SVAR2) — using `Cow<'_, ...>` here instead +/// would force a higher-ranked bound that fails to unify across the +/// two call sites. /// - `shift` – total amount to shift by (i64) -/// - `alt_alleles` – packed ALT allele bytes for all variants (u8) -/// - `alt_offsets` – byte offsets into `alt_alleles`; length = total_variants + 1 (i64) /// - `ref_` – reference contig bytes (u8) /// - `ref_start` – start position into the reference; may be negative (i64) /// - `out` – output buffer to fill (u8, length = desired haplotype length) /// - `pad_char` – byte used for padding where reference is unavailable /// - `keep` – optional per-haplotype-variant mask; `None` means use all -/// - `annot_v_idxs` – optional annotation: variant index per output position (i32; -1 = ref/pad) +/// - `annot_v_idxs` – optional annotation: `annot_id` per output position (i32; -1 = ref/pad) /// - `annot_ref_pos` – optional annotation: reference position per output position (i32; /// -1 = leading pad, i32::MAX = trailing pad) #[allow(clippy::too_many_arguments)] -pub fn reconstruct_haplotype_from_sparse( - v_idxs: ArrayView1, - v_starts: ArrayView1, - ilens: ArrayView1, +fn reconstruct_haplotype_core<'a>( + n_variants: usize, + provide: impl Fn(usize) -> (i64, i64, std::borrow::Cow<'a, [u8]>, i32), shift: i64, - alt_alleles: ArrayView1, - alt_offsets: ArrayView1, ref_: ArrayView1, ref_start: i64, mut out: ArrayViewMut1, @@ -42,13 +49,11 @@ pub fn reconstruct_haplotype_from_sparse( mut annot_ref_pos: Option>, ) { let length = out.len() as i64; - let n_variants = v_idxs.len(); // Hoist contiguous-slice pointers once so the hot loops use direct byte ops // (fill/copy_from_slice) instead of ndarray's stride/do_slice dispatch path. let out_flat: &mut [u8] = out.as_slice_mut().unwrap(); let ref_flat: &[u8] = ref_.as_slice().unwrap(); - let alt_flat: &[u8] = alt_alleles.as_slice().unwrap(); let mut av_flat: Option<&mut [i32]> = annot_v_idxs.as_mut().and_then(|a| a.as_slice_mut()); let mut ap_flat: Option<&mut [i32]> = annot_ref_pos.as_mut().and_then(|a| a.as_slice_mut()); @@ -84,13 +89,8 @@ pub fn reconstruct_haplotype_from_sparse( } } - let variant = v_idxs[v] as usize; - let v_pos = v_starts[variant] as i64; - let v_diff = ilens[variant] as i64; - let ao_s = alt_offsets[variant] as usize; - let ao_e = alt_offsets[variant + 1] as usize; - // full allele slice; may be sub-sliced below for shift consumption - let allele_full = &alt_flat[ao_s..ao_e]; + let (v_pos, v_diff, allele_cow, annot_id) = provide(v); + let allele_full: &[u8] = allele_cow.as_ref(); let v_len_full = allele_full.len() as i64; // +1 assumes atomized variants, exactly 1 nt shared between REF and ALT let v_ref_end: i64 = v_pos - 0i64.min(v_diff) + 1; @@ -181,7 +181,7 @@ pub fn reconstruct_haplotype_from_sparse( let oe = (out_idx + writable_length) as usize; out_flat[os..oe].copy_from_slice(&allele[..writable_length as usize]); if let Some(av) = av_flat.as_deref_mut() { - av[os..oe].fill(variant as i32); + av[os..oe].fill(annot_id); } if let Some(ap) = ap_flat.as_deref_mut() { ap[os..oe].fill(v_pos as i32); @@ -255,6 +255,69 @@ pub fn reconstruct_haplotype_from_sparse( } } +/// Reconstruct a single haplotype from reference sequence and variants (SVAR 1.0 +/// global variant table). Thin wrapper over [`reconstruct_haplotype_core`]: builds a +/// closure that indexes the global table and delegates. **Behavior is unchanged** from +/// before the closure-source refactor — this only changes where the per-variant data +/// comes from. +/// +/// # Parameters +/// - `v_idxs` – indices into the full variant table for this haplotype (i32) +/// - `v_starts` – genomic start position of each variant (i32, indexed by variant) +/// - `ilens` – insertion-length (ilen = alt_len − ref_len + 1) per variant (i32) +/// - `shift` – total amount to shift by (i64) +/// - `alt_alleles` – packed ALT allele bytes for all variants (u8) +/// - `alt_offsets` – byte offsets into `alt_alleles`; length = total_variants + 1 (i64) +/// - `ref_` – reference contig bytes (u8) +/// - `ref_start` – start position into the reference; may be negative (i64) +/// - `out` – output buffer to fill (u8, length = desired haplotype length) +/// - `pad_char` – byte used for padding where reference is unavailable +/// - `keep` – optional per-haplotype-variant mask; `None` means use all +/// - `annot_v_idxs` – optional annotation: variant index per output position (i32; -1 = ref/pad) +/// - `annot_ref_pos` – optional annotation: reference position per output position (i32; +/// -1 = leading pad, i32::MAX = trailing pad) +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotype_from_sparse( + v_idxs: ArrayView1, + v_starts: ArrayView1, + ilens: ArrayView1, + shift: i64, + alt_alleles: ArrayView1, + alt_offsets: ArrayView1, + ref_: ArrayView1, + ref_start: i64, + out: ArrayViewMut1, + pad_char: u8, + keep: Option>, + annot_v_idxs: Option>, + annot_ref_pos: Option>, +) { + let alt_flat: &[u8] = alt_alleles.as_slice().unwrap(); + let provide = |v: usize| { + let variant = v_idxs[v] as usize; + let ao_s = alt_offsets[variant] as usize; + let ao_e = alt_offsets[variant + 1] as usize; + ( + v_starts[variant] as i64, + ilens[variant] as i64, + std::borrow::Cow::Borrowed(&alt_flat[ao_s..ao_e]), + variant as i32, + ) + }; + reconstruct_haplotype_core( + v_idxs.len(), + provide, + shift, + ref_, + ref_start, + out, + pad_char, + keep, + annot_v_idxs, + annot_ref_pos, + ); +} + /// Batch driver: reconstruct haplotypes for all (query, hap) pairs. /// /// Mirrors `reconstruct_haplotypes_from_sparse` (plural) in @@ -498,9 +561,307 @@ pub fn reconstruct_haplotypes_from_sparse( // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent // aliasing. let av_view: Option> = av_raw.map(|p| { - let chunk = unsafe { - std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) - }; + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; + ArrayViewMut1::from(chunk) + }); + + // SAFETY: same invariant as out_chunk — `out_offsets` non-decreasing guarantees + // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent + // aliasing. + let ap_view: Option> = ap_raw.map(|p| { + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; + ArrayViewMut1::from(chunk) + }); + + do_work(k, out_view, av_view, ap_view); + } + } +} + +/// Batch driver: reconstruct haplotypes for all (query, hap) pairs from the SVAR2 +/// two-channel (var_key ⋈ dense) source. Mirrors +/// [`reconstruct_haplotypes_from_sparse`]'s parallel/serial disjoint-slice carving +/// exactly — only the per-work-item variant source and decode differ. Additive: SVAR +/// 1.0's `reconstruct_haplotype[s]_from_sparse` are untouched by this function. +/// +/// # Parameters +/// - `out` – flat output buffer, length = out_offsets[-1] (u8); written in place +/// - `out_offsets` – shape (batch*ploidy + 1,) offsets into `out` +/// - `regions` – shape (batch, 3) as (contig_idx, start, end) i32 +/// - `shifts` – shape (batch, ploidy) i32 +/// - `vk_pos` / `vk_key` – this hap's `var_key` channel: position + uniform key (i32) +/// - `vk_off` – shape (n_work + 1) CSR offsets into `vk_pos`/`vk_key`, indexed by flat +/// work index `k = query * ploidy + hap` +/// - `dense_pos` / `dense_key` – the shared `dense` channel: position + uniform key (i32) +/// - `dense_range` – shape (batch, 2) as `[ds, de)`, the window into `dense_pos`/`dense_key` +/// for each query row +/// - `dense_present` – packed presence bits over the dense window, LSB-first per byte (u8) +/// - `dense_present_off` – shape (n_work + 1) BIT offsets into `dense_present`, indexed by +/// flat work index `k` +/// - `lut_bytes` / `lut_off` – long-allele-bank bytes + CSR offsets, for `Lookup` keys +/// - `ref_` – packed reference bytes u8 +/// - `ref_offsets` – per-contig offsets into ref_ i64 +/// - `pad_char` – padding byte u8 +/// - `annot_v_idxs` – optional annotation output i32 (same layout as out); the value +/// written per applied variant is its sequential index within the merged per-hap list +/// (`0..merged.len()`), not a global variant id (SVAR2 has no global variant table) +/// - `annot_ref_pos` – optional annotation output i32 (same layout as out) +/// - `parallel` – if true, use rayon to process work items concurrently +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2( + mut out: ArrayViewMut1, + out_offsets: ArrayView1, + regions: ArrayView2, + shifts: ArrayView2, + vk_pos: ArrayView1, + vk_key: ArrayView1, + vk_off: ArrayView1, + dense_pos: ArrayView1, + dense_key: ArrayView1, + dense_range: ArrayView2, + dense_present: ArrayView1, + dense_present_off: ArrayView1, + lut_bytes: ArrayView1, + lut_off: ArrayView1, + ref_: ArrayView1, + ref_offsets: ArrayView1, + pad_char: u8, + mut annot_v_idxs: Option>, + mut annot_ref_pos: Option>, + parallel: bool, +) { + let batch_size = regions.nrows(); + let ploidy = shifts.ncols(); + let n_work = batch_size * ploidy; + + // Hoist contiguous-slice pointers once, exactly as SVAR1's do_work captures + // read-only views — these are Send+Sync so the rayon parallel path is unchanged. + let vk_pos_s: &[i32] = vk_pos.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off.as_slice().unwrap(); + + // Per-k inner work: merge this hap's var_key ⋈ dense entries, then reconstruct via + // the shared core with a decode closure. All read-only ArrayViews/slices are + // Send+Sync so the closure can borrow them freely. + let do_work = |k: usize, + out_view: ArrayViewMut1, + av_view: Option>, + ap_view: Option>| { + let query = k / ploidy; + let hap = k % ploidy; + + // region/ref + let c_idx = regions[[query, 0]] as usize; + let c_s = ref_offsets[c_idx] as usize; + let c_e = ref_offsets[c_idx + 1] as usize; + let contig_ref = ref_.slice(s![c_s..c_e]); + let ref_start = regions[[query, 1]] as i64; + let shift = shifts[[query, hap]] as i64; + + // var_key window for this hap + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + + // dense window for this query + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + + // presence bits for this hap start at bit `dense_present_off[k]` + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| -> bool { + let bit = base_bit + j; + (dense_present_s[bit / 8] >> (bit % 8)) & 1 == 1 // LSB-first within each byte + }; + + let merged = crate::svar2::merge_hap( + vk_pos_s, + vk_key_s, + vk_lo, + vk_hi, + dense_pos_s, + dense_key_s, + ds, + de, + present_bit, + ); + + let contig_ref_s: &[u8] = contig_ref.as_slice().unwrap(); + let provide = |v: usize| { + let (pos, key) = merged[v]; + let (v_diff, allele) = crate::svar2::decode_alt(key, lut_bytes_s, lut_off_s); + // A pure DEL decodes to an empty ALT: genoray stores no anchor base in the key + // (recovered from the reference downstream). But the reconstruction kernel writes + // `allele_full` as the replacement and advances ref by |v_diff|+1, so it needs the + // anchor base = ref[pos] present; an empty allele would delete the anchor too + // (off-by-one: -(|v_diff|+1) instead of -|v_diff|). SNP/INS/Lookup alleles are + // already non-empty and pass through unchanged. + let allele = if allele.is_empty() { + std::borrow::Cow::Borrowed(&contig_ref_s[pos as usize..pos as usize + 1]) + } else { + allele + }; + (pos as i64, v_diff, allele, v as i32) + }; + + reconstruct_haplotype_core( + merged.len(), + provide, + shift, + contig_ref, + ref_start, + out_view, + pad_char, + None, // keep: SVAR2 has no per-haplotype keep mask + av_view, + ap_view, + ); + }; + + if parallel { + // Build disjoint per-k mutable slices for all active buffers using the + // proven split_at_mut chain idiom (mirrors get_reference in reference/mod.rs). + // &mut [_] slices are Send, unlike raw *mut pointers — safe for rayon closures. + let bounds: Vec<(usize, usize)> = (0..n_work) + .map(|k| (out_offsets[k] as usize, out_offsets[k + 1] as usize)) + .collect(); + + let out_slice = out.as_slice_mut().unwrap(); + let mut out_chunks: Vec<&mut [u8]> = Vec::with_capacity(n_work); + { + let mut rest = &mut out_slice[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + // Contract: `out_offsets` is monotonically non-decreasing, so each + // work item's range starts at or after the previous one's end. This + // guarantees `s - cursor` does not underflow and the carved slices + // are disjoint. The same `bounds` drives the annotation carves below. + debug_assert!( + s >= cursor && e >= s, + "out_offsets must be monotonically non-decreasing (got s={s}, e={e}, cursor={cursor})" + ); + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + out_chunks.push(mid); + rest = tail2; + cursor = e; + } + } + + // Carve annotation buffers only when they are Some. + let av_chunks: Option> = annot_v_idxs.as_mut().map(|av| { + let av_slice = av.as_slice_mut().unwrap(); + let mut chunks: Vec<&mut [i32]> = Vec::with_capacity(n_work); + let mut rest = &mut av_slice[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + chunks.push(mid); + rest = tail2; + cursor = e; + } + chunks + }); + + let ap_chunks: Option> = annot_ref_pos.as_mut().map(|ap| { + let ap_slice = ap.as_slice_mut().unwrap(); + let mut chunks: Vec<&mut [i32]> = Vec::with_capacity(n_work); + let mut rest = &mut ap_slice[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + chunks.push(mid); + rest = tail2; + cursor = e; + } + chunks + }); + + // Zip all chunk vecs and dispatch in parallel. + // Handle the four combinations of av/ap presence. + match (av_chunks, ap_chunks) { + (Some(avc), Some(apc)) => { + out_chunks + .into_par_iter() + .zip(avc.into_par_iter()) + .zip(apc.into_par_iter()) + .enumerate() + .for_each(|(k, ((out_chunk, av_chunk), ap_chunk))| { + do_work( + k, + ArrayViewMut1::from(out_chunk), + Some(ArrayViewMut1::from(av_chunk)), + Some(ArrayViewMut1::from(ap_chunk)), + ); + }); + } + (Some(avc), None) => { + out_chunks + .into_par_iter() + .zip(avc.into_par_iter()) + .enumerate() + .for_each(|(k, (out_chunk, av_chunk))| { + do_work( + k, + ArrayViewMut1::from(out_chunk), + Some(ArrayViewMut1::from(av_chunk)), + None, + ); + }); + } + (None, Some(apc)) => { + out_chunks + .into_par_iter() + .zip(apc.into_par_iter()) + .enumerate() + .for_each(|(k, (out_chunk, ap_chunk))| { + do_work( + k, + ArrayViewMut1::from(out_chunk), + None, + Some(ArrayViewMut1::from(ap_chunk)), + ); + }); + } + (None, None) => { + out_chunks + .into_par_iter() + .enumerate() + .for_each(|(k, out_chunk)| { + do_work(k, ArrayViewMut1::from(out_chunk), None, None); + }); + } + } + } else { + // Serial path: use raw pointers for disjoint sub-range access, exactly as before. + // The serial loop prevents concurrent aliasing. + let out_raw: *mut u8 = out.as_mut_ptr(); + let av_raw: Option<*mut i32> = annot_v_idxs.as_mut().map(|a| a.as_mut_ptr()); + let ap_raw: Option<*mut i32> = annot_ref_pos.as_mut().map(|a| a.as_mut_ptr()); + + for k in 0..n_work { + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; + + // SAFETY: `out_offsets` is required by the calling contract to be monotonically + // non-decreasing, so consecutive (out_s, out_e) pairs are strictly non-overlapping + // address ranges within the same allocation. Because the loop is serial there are + // no concurrent borrows, so constructing a `&mut [u8]` from each disjoint sub-range + // is free of aliasing UB. + let out_chunk = + unsafe { std::slice::from_raw_parts_mut(out_raw.add(out_s), out_e - out_s) }; + let out_view = ArrayViewMut1::from(out_chunk); + + // SAFETY: same invariant as out_chunk — `out_offsets` non-decreasing guarantees + // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent + // aliasing. + let av_view: Option> = av_raw.map(|p| { + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; ArrayViewMut1::from(chunk) }); @@ -508,9 +869,7 @@ pub fn reconstruct_haplotypes_from_sparse( // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent // aliasing. let ap_view: Option> = ap_raw.map(|p| { - let chunk = unsafe { - std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) - }; + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; ArrayViewMut1::from(chunk) }); @@ -595,9 +954,9 @@ mod tests { &[], // alt_alleles &[0i64], // alt_offsets (1 sentinel for 0 variants) &[10, 20, 30, 40, 50], - 1, // ref_start - 3, // out_len - 0, // pad_char + 1, // ref_start + 3, // out_len + 0, // pad_char None, false, ); @@ -627,7 +986,11 @@ mod tests { ); assert_eq!(out, vec![9, 9, 1, 2, 3]); assert_eq!(&av[..2], &[-1i32, -1]); - assert_eq!(&ap[..2], &[-1i32, -1], "leading pad annot_ref_pos must be -1"); + assert_eq!( + &ap[..2], + &[-1i32, -1], + "leading pad annot_ref_pos must be -1" + ); assert_eq!(&ap[2..], &[0i32, 1, 2]); } @@ -644,14 +1007,14 @@ mod tests { // variant at pos=2 (G→T), ilen=0 → v_ref_end = 2 - 0 + 1 = 3 // out: A C [T] T A let (out, av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[2], // v_starts: variant 0 is at pos 2 - &[0], // ilens: SNP, no length change - 0, // shift - &[84u8], // alt_alleles: T - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[2], // v_starts: variant 0 is at pos 2 + &[0], // ilens: SNP, no length change + 0, // shift + &[84u8], // alt_alleles: T + &[0i64, 1], // alt_offsets &[65, 67, 71, 84, 65], // A C G T A - 0, // ref_start + 0, // ref_start 5, 0, None, @@ -676,8 +1039,8 @@ mod tests { fn two_bp_insertion() { let (out, _av, _ap) = run( &[0], - &[2], // variant 0 at pos 2 - &[2], // ilen=+2 + &[2], // variant 0 at pos 2 + &[2], // ilen=+2 0, &[10u8, 11, 12], &[0i64, 3], @@ -705,10 +1068,10 @@ mod tests { fn deletion() { let (out, _av, _ap) = run( &[0], - &[2], // variant 0 at pos 2 - &[-2], // ilen=-2 + &[2], // variant 0 at pos 2 + &[-2], // ilen=-2 0, - &[30u8], // anchor allele byte + &[30u8], // anchor allele byte &[0i64, 1], &[1, 2, 3, 4, 5, 6, 7], 0, @@ -735,13 +1098,13 @@ mod tests { fn del_spanning_ref_start() { let (out, _av, ap) = run( &[0], - &[1], // v_pos=1 - &[-3], // ilen=-3 + &[1], // v_pos=1 + &[-3], // ilen=-3 0, &[99u8], &[0i64, 1], &[1, 2, 3, 4, 5, 6, 7], - 3, // ref_start=3 + 3, // ref_start=3 5, 0, None, @@ -767,10 +1130,10 @@ mod tests { fn overshoot_ref_past_contig() { let (out, _av, _ap) = run( &[0], - &[2], // v_pos=2 - &[-5], // ilen=-5 (deletion past contig end) - 0, // shift - &[50u8], // anchor allele + &[2], // v_pos=2 + &[-5], // ilen=-5 (deletion past contig end) + 0, // shift + &[50u8], // anchor allele &[0i64, 1], &[1, 2, 3, 4], // ref, len 4 0, // ref_start @@ -793,9 +1156,9 @@ mod tests { #[test] fn overlapping_alts_first_applied() { let (out, _av, _ap) = run( - &[0, 1], // v_idxs: variants 0 then 1 - &[2, 2], // both at pos=2 - &[0, 0], // both SNPs + &[0, 1], // v_idxs: variants 0 then 1 + &[2, 2], // both at pos=2 + &[0, 0], // both SNPs 0, &[20u8, 30], // alleles: 20 and 30 &[0i64, 1, 2], @@ -844,9 +1207,9 @@ mod tests { // out_len=5: [3, 99, 88, 5, 6] let (out, _av, _ap) = run( &[0], - &[3], // v_pos=3 - &[1], // ilen=+1 - 2, // shift=2 + &[3], // v_pos=3 + &[1], // ilen=+1 + 2, // shift=2 &[99u8, 88], &[0i64, 2], &[1, 2, 3, 4, 5, 6], @@ -878,8 +1241,8 @@ mod tests { let (out, _av, _ap) = run( &[0], &[3], - &[1], // ilen=+1, allele 2 bytes - 4, // shift=4 + &[1], // ilen=+1, allele 2 bytes + 4, // shift=4 &[99u8, 88], &[0i64, 2], &[1, 2, 3, 4, 5, 6, 7, 8], @@ -955,16 +1318,16 @@ mod tests { #[test] fn allele_start_idx_eq_v_len_continue() { let (out, _av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[3], // v_starts: variant 0 at pos 3 - &[0], // ilens: SNP, ilen=0 - 4, // shift=4 - &[88u8], // alt_allele - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[3], // v_starts: variant 0 at pos 3 + &[0], // ilens: SNP, ilen=0 + 4, // shift=4 + &[88u8], // alt_allele + &[0i64, 1], // alt_offsets &[1, 2, 3, 4, 5, 6, 7, 8], - 0, // ref_start - 4, // out_len - 0, // pad_char + 0, // ref_start + 4, // out_len + 0, // pad_char None, false, ); @@ -996,16 +1359,16 @@ mod tests { fn skip_variant_not_enough_distance() { let ref_: Vec = (1u8..=15).collect(); let (out, _av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[3], // v_starts: variant 0 at pos 3 - &[0], // ilens: SNP, ilen=0 - 10, // shift=10 - &[77u8], // alt_allele (never used) - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[3], // v_starts: variant 0 at pos 3 + &[0], // ilens: SNP, ilen=0 + 10, // shift=10 + &[77u8], // alt_allele (never used) + &[0i64, 1], // alt_offsets &ref_, - 0, // ref_start - 3, // out_len - 0, // pad_char + 0, // ref_start + 3, // out_len + 0, // pad_char None, false, ); @@ -1037,18 +1400,18 @@ mod tests { #[test] fn keep_mask_excludes_variant() { let (out, av, _ap) = run( - &[0, 1], // v_idxs: variants 0 and 1 - &[1, 3], // v_starts: variant 0 at pos 1, variant 1 at pos 3 - &[0, 0], // ilens: both SNPs - 0, // shift=0 - &[55u8, 99], // alleles: 55 for v0, 99 for v1 - &[0i64, 1, 2], // alt_offsets + &[0, 1], // v_idxs: variants 0 and 1 + &[1, 3], // v_starts: variant 0 at pos 1, variant 1 at pos 3 + &[0, 0], // ilens: both SNPs + 0, // shift=0 + &[55u8, 99], // alleles: 55 for v0, 99 for v1 + &[0i64, 1, 2], // alt_offsets &[1, 2, 3, 4, 5], - 0, // ref_start - 5, // out_len - 0, // pad_char + 0, // ref_start + 5, // out_len + 0, // pad_char Some(&[false, true]), // keep mask: skip v0, apply v1 - true, // annotate + true, // annotate ); // variant 0 (pos=1, allele=55) excluded by keep mask: ref[1] NOT replaced // variant 1 (pos=3, allele=99) applied: ref[3] replaced by 99 @@ -1064,28 +1427,29 @@ mod tests { #[test] fn annotated_vs_non_annotated_identical_out() { let params = ( - &[0i32][..], // v_idxs - &[2i32][..], // v_starts - &[0i32][..], // ilens - 0i64, // shift - &[77u8][..], // alt_alleles - &[0i64, 1][..],// alt_offsets - &[1u8,2,3,4,5][..], // ref_ - 0i64, // ref_start - 5usize, // out_len - 0u8, // pad_char + &[0i32][..], // v_idxs + &[2i32][..], // v_starts + &[0i32][..], // ilens + 0i64, // shift + &[77u8][..], // alt_alleles + &[0i64, 1][..], // alt_offsets + &[1u8, 2, 3, 4, 5][..], // ref_ + 0i64, // ref_start + 5usize, // out_len + 0u8, // pad_char ); let (out_annot, _, _) = run( - params.0, params.1, params.2, params.3, - params.4, params.5, params.6, params.7, + params.0, params.1, params.2, params.3, params.4, params.5, params.6, params.7, params.8, params.9, None, true, ); let (out_plain, _, _) = run( - params.0, params.1, params.2, params.3, - params.4, params.5, params.6, params.7, + params.0, params.1, params.2, params.3, params.4, params.5, params.6, params.7, params.8, params.9, None, false, ); - assert_eq!(out_annot, out_plain, "annotated and non-annotated must produce identical out bytes"); + assert_eq!( + out_annot, out_plain, + "annotated and non-annotated must produce identical out bytes" + ); } #[test] @@ -1202,7 +1566,160 @@ mod tests { false, ); - assert_eq!(&out.as_slice().unwrap()[0..4], b"ATGT", "region 0 with SNP applied"); - assert_eq!(&out.as_slice().unwrap()[4..8], b"ACGT", "region 1 reference-only"); + assert_eq!( + &out.as_slice().unwrap()[0..4], + b"ATGT", + "region 0 with SNP applied" + ); + assert_eq!( + &out.as_slice().unwrap()[4..8], + b"ACGT", + "region 1 reference-only" + ); + } + + // ------------------------------------------------------------------------- + // SVAR2 driver: one region (R=1), one sample-slot (S=1), one ploid (P=1) → + // n_work = 1, k = query = hap = 0. Hand-built two-channel input: a SNP in the + // var_key channel and a pure DEL in the dense channel (with its presence bit set), + // exercising both the merge and the decode paths in a single call. + // + // ref = "ACGTACGT" (8 bp, positions 0..8: A C G T A C G T) + // var_key: SNP at pos=1 (ref 'C' -> 'T'), Inline key, decode_alt -> (v_diff=0, "T") + // dense: pure DEL at pos=4, ilen=-1, decode_alt -> (v_diff=-1, "" empty). The driver's + // `provide` closure substitutes the anchor base ref[pos]='A' for the empty ALT + // (genoray stores no anchor in the key; the kernel needs it, else the anchor is + // deleted too). So the DEL contributes allele="A" and deletes the following 'C'. + // merge_hap orders by position: v=0 is the SNP (pos=1), v=1 is the DEL (pos=4). + // + // Hand derivation of reconstruct_haplotype_core(n_variants=2, ..., shift=0): + // start: ref_idx=0, out_idx=0 (ref_start=0, not negative: no leading pad) + // v=0 (SNP, pos=1, v_diff=0, allele="T", v_len_full=1): + // v_ref_end = 1 - min(0,0) + 1 = 2 + // not a DEL-spanning-start case; pos(1) >= ref_idx(0); shift=0 so no shift branch + // ref_len = 1-0 = 1 -> out[0..1] = ref[0..1] = "A"; out_idx=1 + // writable_length = min(1, 8-1)=1 -> out[1..2] = "T"; out_idx=2 + // ref_idx = v_ref_end = 2 + // v=1 (DEL, pos=4, v_diff=-1, allele="A" (anchor = ref[4])): + // v_ref_end = 4 - min(0,-1) + 1 = 4+1+1 = 6 + // pos(4) >= ref_idx(2); shift=0 so no shift branch + // ref_len = 4-2 = 2 -> out[2..4] = ref[2..4] = "GT"; out_idx=4 + // writable_length = min(1, 8-4) = 1 -> out[4..5] = "A"; out_idx=5 + // ref_idx = v_ref_end = 6 (ref 'C' at pos 5 is skipped = deleted) + // tail: unfilled_length = 8-5 = 3; writable_ref = min(3, 8-6) = 2 + // -> out[5..7] = ref[6..8] = "GT"; out_end_idx=7 + // right-pad out[7..8] with pad_char 'N' + // out = "A" + "T" + "GT" + "A" + "GT" + "N" = "ATGTAGTN" + // (haplotype = ref with SNP C->T at pos1 and the 'C' at pos5 deleted) + #[test] + fn svar2_reconstruct_snp_and_del() { + let reference = b"ACGTACGT"; + let ref_ = arr1(reference.as_ref()); + let ref_offsets = arr1(&[0i64, 8]); + + // One region on contig 0, one sample-slot, one ploid -> n_work = 1. + let regions = ndarray::arr2(&[[0i32, 0, 8]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // var_key channel: one SNP at pos=1 (Inline key, 1-base ALT "T"). + let snp_key = svar2_codec::encode_alt_inline(b"T", 0) as i32; + let vk_pos = arr1(&[1i32]); + let vk_key = arr1(&[snp_key]); + let vk_off = arr1(&[0i64, 1]); + + // dense channel: one pure DEL (1bp) at pos=4, present for this hap. + let del_key = svar2_codec::encode_pure_del(-1) as i32; + let dense_pos = arr1(&[4i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + // Presence bits, LSB-first: bit 0 (this hap's only dense entry) is set. + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + + // No long-allele-bank lookups exercised in this test. + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + + let out_offsets = arr1(&[0i64, 8]); + let pad_char = b'N'; + let mut out = Array1::::from_elem(8, pad_char); + + super::reconstruct_haplotypes_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + ref_.view(), + ref_offsets.view(), + pad_char, + None, + None, + false, // serial + ); + + assert_eq!(out.as_slice().unwrap(), b"ATGTAGTN"); + } + + // Regression guard for the pure-DEL anchor fix: a lone 1bp DEL must delete exactly + // one base (the one AFTER the anchor), not two. ref="ACGT", DEL at pos=1 (ilen=-1): + // keep anchor 'C' at pos1, delete 'G' at pos2 -> "ACT" + right-pad 'N' = "ACTN". + #[test] + fn svar2_pure_del_keeps_anchor() { + let reference = b"ACGT"; + let ref_ = arr1(reference.as_ref()); + let ref_offsets = arr1(&[0i64, 4]); + let regions = ndarray::arr2(&[[0i32, 0, 4]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // No var_key entries; a single dense pure DEL at pos=1. + let vk_pos = arr1::(&[]); + let vk_key = arr1::(&[]); + let vk_off = arr1(&[0i64, 0]); + let del_key = svar2_codec::encode_pure_del(-1) as i32; + let dense_pos = arr1(&[1i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + let out_offsets = arr1(&[0i64, 4]); + let pad_char = b'N'; + let mut out = Array1::::from_elem(4, pad_char); + + super::reconstruct_haplotypes_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + ref_.view(), + ref_offsets.view(), + pad_char, + None, + None, + false, + ); + + assert_eq!(out.as_slice().unwrap(), b"ACTN"); } } From 26c3faef8d32734459fe5e8699d1e363dd286c19 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 12:43:54 -0700 Subject: [PATCH 004/108] feat(svar2): two-source track realign driver (gvl M6b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 4. Track-realign analogue of Task 3's reconstruct refactor. - shift_and_realign_track_core(n_variants, provide: Fn(usize)->(v_start, v_diff), ...) holds the (unchanged) realignment math; shift_and_realign_track_sparse is now a thin wrapper building the global-table closure. SVAR 1.0 behavior byte-identical. - shift_and_realign_tracks_from_svar2: new driver mirroring the SVAR1 batch driver's parallel/serial carving; per hap builds the var_key⋈dense merged list (svar2::merge_hap) and takes ilen from svar2::decode_alt (allele discarded — tracks need only v_diff; a DEL writes track[v_rel_pos] implicitly, so no anchor fix). No per-hap allele table. - Test svar2_track_realign_del: DEL(ilen=-2) keeps anchor, deletes 2 -> [10,20,50,0] (controller hand-verified). 29 tracks tests pass (SVAR1 regression + new). Also includes cosmetic rustfmt reflow of pre-existing track test code (fmt-clean now; no logic change). Reviewed (Sonnet): Spec ✅, Approved, no defects, 2 Low/cosmetic notes. Co-Authored-By: Claude Opus 4.8 --- src/tracks/mod.rs | 575 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 484 insertions(+), 91 deletions(-) diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index a0bfcb0c..b7dbb5d6 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -189,10 +189,17 @@ pub fn apply_insertion_fill( } } -/// Shift and realign a single track to correspond to one haplotype. +/// Shift and realign a single track to correspond to one haplotype, generic over the +/// variant source. /// -/// Mirrors numba `shift_and_realign_track_sparse` (lines 230-401 of `_tracks.py`) -/// statement-by-statement. +/// Mirrors numba `shift_and_realign_track_sparse` (lines 230-401 of `_tracks.py`), with +/// the per-variant reads factored out behind the `provide` closure so the same loop body +/// can be driven by either the SVAR 1.0 global variant table +/// (`shift_and_realign_track_sparse`) or the SVAR2 two-channel (var_key ⋈ dense) source +/// (`shift_and_realign_tracks_from_svar2`). **Do not change the realignment math +/// here** — only the data source differs between call sites. Unlike the reconstruction +/// core, there is no allele/annotation plumbing and no anchor fix: a DEL implicitly +/// writes `track[v_rel_pos]` via the `else` branch below. /// /// Three key differences from the haplotype reconstruction kernel: /// 1. SNPs (`v_diff == 0`) are SKIPPED — tracks match reference at SNP positions. @@ -201,11 +208,10 @@ pub fn apply_insertion_fill( /// 3. Trailing fill pads with `0.0` (NOT a pad_char byte). /// /// # Parameters -/// - `offset_idx`: index into geno_o_starts/geno_o_stops for this (query, hap) pair -/// - `geno_v_idxs`: flat variant index array -/// - `geno_o_starts`, `geno_o_stops`: normalized (2, n) offsets split into two rows -/// - `v_starts`: variant start positions (absolute genomic coordinates) -/// - `ilens`: variant insertion-length differences (signed) +/// - `n_variants`: number of variants this haplotype sees, i.e. `0..n_variants` are +/// valid indices into `provide` +/// - `provide`: `Fn(v) -> (v_start_abs, v_diff)` for `v in 0..n_variants`: `v_start_abs` +/// genomic start, `v_diff` ilen (signed) /// - `shift`: total shift for this haplotype /// - `track`: reference track values for this query (f32 slice) /// - `query_start`: the genomic start of this query region @@ -215,13 +221,9 @@ pub fn apply_insertion_fill( /// - `strategy_id`: insertion-fill strategy /// - `base_seed`, `query`, `hap`: seed components for FlankSample strategy #[allow(clippy::too_many_arguments)] -pub fn shift_and_realign_track_sparse( - offset_idx: usize, - geno_v_idxs: ndarray::ArrayView1, - geno_o_starts: ndarray::ArrayView1, - geno_o_stops: ndarray::ArrayView1, - v_starts: ndarray::ArrayView1, - ilens: ndarray::ArrayView1, +fn shift_and_realign_track_core( + n_variants: usize, + provide: impl Fn(usize) -> (i64, i64), // (v_start_abs, v_diff) shift: i64, track: ndarray::ArrayView1, query_start: i64, @@ -233,14 +235,7 @@ pub fn shift_and_realign_track_sparse( query: u64, hap: u64, ) { - // Numba: o_s, o_e = geno_offsets[offset_idx], geno_offsets[offset_idx + 1] (1-D branch) - // or geno_offsets[:, offset_idx] (2-D branch — normalized form) - // We receive the pre-split (2, n) rows directly. - let o_s = geno_o_starts[offset_idx] as usize; - let o_e = geno_o_stops[offset_idx] as usize; - let variant_idxs = &geno_v_idxs.as_slice().unwrap()[o_s..o_e]; let length = out.len(); - let n_variants = variant_idxs.len(); if n_variants == 0 { // Numba: out[:] = track[:length] @@ -263,12 +258,11 @@ pub fn shift_and_realign_track_sparse( } } - let variant = variant_idxs[v] as usize; + let (v_start, v_diff) = provide(v); // Numba: v_rel_pos = v_starts[variant] - query_start - let v_rel_pos = v_starts[variant] as i64 - query_start; - // Numba: v_diff = ilens[variant] - let v_diff = ilens[variant] as i64; + let v_rel_pos = v_start - query_start; + // v_diff already i64 // Numba: v_rel_end = v_rel_pos - min(0, v_diff) + 1 let v_rel_end = v_rel_pos - v_diff.min(0) + 1; @@ -411,6 +405,71 @@ pub fn shift_and_realign_track_sparse( } } +/// Shift and realign a single track to correspond to one haplotype (SVAR 1.0 global +/// variant table). Thin wrapper over [`shift_and_realign_track_core`]: builds a closure +/// that indexes the global table and delegates. **Behavior is unchanged** from before +/// the closure-source refactor — this only changes where the per-variant data comes +/// from. +/// +/// # Parameters +/// - `offset_idx`: index into geno_o_starts/geno_o_stops for this (query, hap) pair +/// - `geno_v_idxs`: flat variant index array +/// - `geno_o_starts`, `geno_o_stops`: normalized (2, n) offsets split into two rows +/// - `v_starts`: variant start positions (absolute genomic coordinates) +/// - `ilens`: variant insertion-length differences (signed) +/// - `shift`: total shift for this haplotype +/// - `track`: reference track values for this query (f32 slice) +/// - `query_start`: the genomic start of this query region +/// - `out`: output slice to fill (length = haplotype output length) +/// - `params`: per-strategy parameter (f64) +/// - `keep`: optional boolean mask over the variant group for this (query, hap) +/// - `strategy_id`: insertion-fill strategy +/// - `base_seed`, `query`, `hap`: seed components for FlankSample strategy +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_track_sparse( + offset_idx: usize, + geno_v_idxs: ndarray::ArrayView1, + geno_o_starts: ndarray::ArrayView1, + geno_o_stops: ndarray::ArrayView1, + v_starts: ndarray::ArrayView1, + ilens: ndarray::ArrayView1, + shift: i64, + track: ndarray::ArrayView1, + query_start: i64, + out: &mut ndarray::ArrayViewMut1, + params: ndarray::ArrayView1, + keep: Option>, + strategy_id: i64, + base_seed: u64, + query: u64, + hap: u64, +) { + // Numba: o_s, o_e = geno_offsets[offset_idx], geno_offsets[offset_idx + 1] (1-D branch) + // or geno_offsets[:, offset_idx] (2-D branch — normalized form) + // We receive the pre-split (2, n) rows directly. + let o_s = geno_o_starts[offset_idx] as usize; + let o_e = geno_o_stops[offset_idx] as usize; + let variant_idxs = &geno_v_idxs.as_slice().unwrap()[o_s..o_e]; + let provide = |v: usize| { + let variant = variant_idxs[v] as usize; + (v_starts[variant] as i64, ilens[variant] as i64) + }; + shift_and_realign_track_core( + variant_idxs.len(), + provide, + shift, + track, + query_start, + out, + params, + keep, + strategy_id, + base_seed, + query, + hap, + ); +} + /// Shift and realign tracks for a batch of (query, hap) pairs in place (writes `out`). /// /// Mirrors numba `shift_and_realign_tracks_sparse` (lines 141-228 of `_tracks.py`) @@ -463,11 +522,16 @@ pub fn shift_and_realign_tracks_sparse( // applied the same pattern: out.as_slice_mut().unwrap() once, then index [a..b] // directly. Here we do the same for out, tracks, and keep. // geno_v_idxs already uses .as_slice().unwrap() (inner fn line 240) — same contract. - let out_flat = out.as_slice_mut().expect("out must be contiguous (C-order)"); - let tracks_flat = tracks.as_slice().expect("tracks must be contiguous (C-order)"); + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + let tracks_flat = tracks + .as_slice() + .expect("tracks must be contiguous (C-order)"); // Hoist keep flat option once (avoids repeated .as_slice() per hap). - let keep_flat: Option<&[bool]> = - keep.as_ref().map(|k| k.as_slice().expect("keep must be contiguous (C-order)")); + let keep_flat: Option<&[bool]> = keep + .as_ref() + .map(|k| k.as_slice().expect("keep must be contiguous (C-order)")); if parallel { // Build disjoint per-k mutable output slices using the split_at_mut cursor @@ -507,15 +571,14 @@ pub fn shift_and_realign_tracks_sparse( let o_idx = geno_offset_idx[[query, hap]] as usize; let qh_shift = shifts[[query, hap]] as i64; - let qh_keep: Option> = - match (&keep_flat, &keep_offsets) { - (Some(k_flat), Some(ko)) => { - let ks = ko[k] as usize; - let ke = ko[k + 1] as usize; - Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) - } - _ => None, - }; + let qh_keep: Option> = match (&keep_flat, &keep_offsets) { + (Some(k_flat), Some(ko)) => { + let ks = ko[k] as usize; + let ke = ko[k + 1] as usize; + Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) + } + _ => None, + }; let mut qh_out = ndarray::ArrayViewMut1::from(out_chunk); shift_and_realign_track_sparse( @@ -562,15 +625,14 @@ pub fn shift_and_realign_tracks_sparse( // qh_keep = keep[keep_offsets[k_idx]:keep_offsets[k_idx+1]] // ArrayView1::from(&slice[..]) avoids the do_slice call that // k.slice(s![ks..ke]) would generate. - let qh_keep: Option> = - match (&keep_flat, &keep_offsets) { - (Some(k_flat), Some(ko)) => { - let ks = ko[k_idx] as usize; - let ke = ko[k_idx + 1] as usize; - Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) - } - _ => None, - }; + let qh_keep: Option> = match (&keep_flat, &keep_offsets) { + (Some(k_flat), Some(ko)) => { + let ks = ko[k_idx] as usize; + let ke = ko[k_idx + 1] as usize; + Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) + } + _ => None, + }; // Numba: out_s, out_e = out_offsets[k_idx], out_offsets[k_idx + 1] let out_s = out_offsets[k_idx] as usize; @@ -604,6 +666,182 @@ pub fn shift_and_realign_tracks_sparse( } } +/// Shift and realign tracks for a batch of (query, hap) pairs from the SVAR2 +/// two-channel (var_key ⋈ dense) source. Mirrors +/// [`shift_and_realign_tracks_sparse`]'s parallel/serial disjoint-slice carving +/// exactly — only the per-work-item variant source and decode differ. Additive: SVAR +/// 1.0's `shift_and_realign_track[s]_sparse` are untouched by this function. Unlike +/// [`crate::reconstruct::reconstruct_haplotypes_from_svar2`], there are no annotation +/// buffers here (tracks never had them), so the carving is out-chunks only. +/// +/// # Parameters +/// - `out`: flat output buffer (f32), written in place +/// - `out_offsets`: ragged offsets into out, shape (n_q * ploidy + 1,) +/// - `regions`: (n_q, 3) array of (contig_idx, start, end) per query +/// - `shifts`: (n_q, ploidy) shift per (query, hap) +/// - `vk_pos` / `vk_key`: this hap's `var_key` channel: position + uniform key (i32) +/// - `vk_off`: shape (n_work + 1) CSR offsets into `vk_pos`/`vk_key`, indexed by flat +/// work index `k = query * ploidy + hap` +/// - `dense_pos` / `dense_key`: the shared `dense` channel: position + uniform key (i32) +/// - `dense_range`: shape (n_q, 2) as `[ds, de)`, the window into `dense_pos`/`dense_key` +/// for each query row +/// - `dense_present`: packed presence bits over the dense window, LSB-first per byte (u8) +/// - `dense_present_off`: shape (n_work + 1) BIT offsets into `dense_present`, indexed by +/// flat work index `k` +/// - `lut_bytes` / `lut_off`: long-allele-bank bytes + CSR offsets, for `Lookup` keys +/// - `tracks`: flat reference track buffer (f32), ragged by track_offsets +/// - `track_offsets`: (n_q + 1,) offsets into tracks (one track per query) +/// - `params`: per-strategy parameter (f64), shape (1,) +/// - `strategy_id`, `base_seed`: insertion-fill strategy parameters +/// - `parallel`: if true, use rayon to process work items concurrently +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2( + mut out: ndarray::ArrayViewMut1, + out_offsets: ndarray::ArrayView1, + regions: ndarray::ArrayView2, + shifts: ndarray::ArrayView2, + vk_pos: ndarray::ArrayView1, + vk_key: ndarray::ArrayView1, + vk_off: ndarray::ArrayView1, + dense_pos: ndarray::ArrayView1, + dense_key: ndarray::ArrayView1, + dense_range: ndarray::ArrayView2, + dense_present: ndarray::ArrayView1, + dense_present_off: ndarray::ArrayView1, + lut_bytes: ndarray::ArrayView1, + lut_off: ndarray::ArrayView1, + tracks: ndarray::ArrayView1, + track_offsets: ndarray::ArrayView1, + params: ndarray::ArrayView1, + strategy_id: i64, + base_seed: u64, + parallel: bool, +) { + let ploidy = shifts.ncols(); + let n_work = regions.nrows() * ploidy; + + // Hoist contiguous-slice pointers once, exactly as the SVAR1 driver hoists + // `tracks_flat` — these are Send+Sync so the rayon parallel path is unchanged. + let vk_pos_s: &[i32] = vk_pos.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off.as_slice().unwrap(); + let tracks_flat: &[f32] = tracks + .as_slice() + .expect("tracks must be contiguous (C-order)"); + + // Per-k inner work: merge this hap's var_key ⋈ dense entries, then realign via the + // shared core with a decode closure. All read-only ArrayViews/slices are Send+Sync + // so the closure can borrow them freely. + let do_work = |k: usize, out_chunk: &mut [f32]| { + let query = k / ploidy; + let hap = k % ploidy; + + let t_s = track_offsets[query] as usize; + let t_e = track_offsets[query + 1] as usize; + let q_track = ndarray::ArrayView1::from(&tracks_flat[t_s..t_e]); + let q_start = regions[[query, 1]] as i64; + let qh_shift = shifts[[query, hap]] as i64; + + // var_key window for this hap + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + + // dense window for this query + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + + // presence bits for this hap start at bit `dense_present_off[k]` + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| -> bool { + let bit = base_bit + j; + (dense_present_s[bit / 8] >> (bit % 8)) & 1 == 1 // LSB-first within each byte + }; + + let merged = crate::svar2::merge_hap( + vk_pos_s, + vk_key_s, + vk_lo, + vk_hi, + dense_pos_s, + dense_key_s, + ds, + de, + present_bit, + ); + + let provide = |v: usize| { + let (pos, key) = merged[v]; + let (v_diff, _allele) = crate::svar2::decode_alt(key, lut_bytes_s, lut_off_s); + (pos as i64, v_diff) + }; + + let mut qh_out = ndarray::ArrayViewMut1::from(out_chunk); + shift_and_realign_track_core( + merged.len(), + provide, + qh_shift, + q_track, + q_start, + &mut qh_out, + params, + None, // keep: SVAR2 has no per-haplotype keep mask + strategy_id, + base_seed, + query as u64, + hap as u64, + ); + }; + + if parallel { + // Build disjoint per-k mutable output slices using the split_at_mut cursor + // idiom (mirrors shift_and_realign_tracks_sparse's parallel path). + let bounds: Vec<(usize, usize)> = (0..n_work) + .map(|k| (out_offsets[k] as usize, out_offsets[k + 1] as usize)) + .collect(); + + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + let mut out_chunks: Vec<&mut [f32]> = Vec::with_capacity(n_work); + { + let mut rest = &mut out_flat[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + debug_assert!( + s >= cursor && e >= s, + "out_offsets must be monotonically non-decreasing (got s={s}, e={e}, cursor={cursor})" + ); + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + out_chunks.push(mid); + rest = tail2; + cursor = e; + } + } + + out_chunks + .into_par_iter() + .enumerate() + .for_each(|(k, out_chunk)| { + do_work(k, out_chunk); + }); + } else { + // Serial path. + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + for k in 0..n_work { + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; + do_work(k, &mut out_flat[out_s..out_e]); + } + } +} + /// RLE-encode a ragged f32 track buffer into (starts, ends, values, offsets) intervals. /// /// Mirrors numba `tracks_to_intervals` + `_scanned_mask` + `_compact_mask` in @@ -738,7 +976,12 @@ pub fn tracks_to_intervals( // Build disjoint per-query mutable slices from all_starts/ends/values using // interval_offsets (which have already been computed sequentially above). let itv_bounds: Vec<(usize, usize)> = (0..n_queries) - .map(|q| (interval_offsets[q] as usize, interval_offsets[q + 1] as usize)) + .map(|q| { + ( + interval_offsets[q] as usize, + interval_offsets[q + 1] as usize, + ) + }) .collect(); let mut starts_chunks: Vec<&mut [i32]> = Vec::with_capacity(n_queries); @@ -861,7 +1104,7 @@ pub fn tracks_to_intervals( #[cfg(test)] mod tests { use super::*; - use ndarray::Array1; + use ndarray::{arr1, Array1}; /// Expected values hand-derived from the numba algorithm (verified by running /// the Python reference implementation with np.uint64 arithmetic). @@ -972,7 +1215,10 @@ mod tests { ); assert_eq!(result.len(), writable_length); for &v in &result { - assert_eq!(v, expected_val, "REPEAT_5P_NORM: expected {expected_val}, got {v}"); + assert_eq!( + v, expected_val, + "REPEAT_5P_NORM: expected {expected_val}, got {v}" + ); } // Sum preservation check let sum: f32 = result.iter().sum(); @@ -1090,7 +1336,10 @@ mod tests { hap, ); - assert_eq!(result, expected, "FLANK_SAMPLE: result did not match expected"); + assert_eq!( + result, expected, + "FLANK_SAMPLE: result did not match expected" + ); // Spot-check the first index by computing hash4 explicitly: // hash4(42, 7, 1, 0): @@ -1192,7 +1441,9 @@ mod tests { for a in 0..2usize { let mut term = ys[a]; for b in 0..2usize { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1217,9 +1468,15 @@ mod tests { assert_eq!(result.len(), writable_length); // Endpoint check: at i=0, result should equal ys[0]=track[v_rel_pos]=4.0 - assert_eq!(result[0], 4.0f32, "order=1 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 4.0f32, + "order=1 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=1 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=1 at i={i}: got {got}, expected {exp}" + ); } } @@ -1256,7 +1513,9 @@ mod tests { for a in 0..n { let mut term = ys[a]; for b in 0..n { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1280,9 +1539,15 @@ mod tests { ); // At x=0, result should equal ys[0] = track[v_rel_pos] = 4.0 - assert_eq!(result[0], 4.0f32, "order=2 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 4.0f32, + "order=2 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=2 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=2 at i={i}: got {got}, expected {exp}" + ); } } @@ -1318,7 +1583,9 @@ mod tests { for a in 0..n { let mut term = ys[a]; for b in 0..n { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1342,9 +1609,15 @@ mod tests { ); // At x=0, result should equal track[v_rel_pos]=5.0 - assert_eq!(result[0], 5.0f32, "order=3 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 5.0f32, + "order=3 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=3 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=3 at i={i}: got {got}, expected {exp}" + ); } } @@ -1390,7 +1663,9 @@ mod tests { for a in 0..2 { let mut term = ys[a]; for b in 0..2 { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1498,7 +1773,11 @@ mod tests { 0, 0, ); - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "no variants: copy track[:length]"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "no variants: copy track[:length]" + ); } /// Deletion: track[v_rel_pos] repeated for writable_length; track advances by @@ -1519,13 +1798,13 @@ mod tests { let track = [10.0f32, 20.0, 30.0, 40.0, 50.0]; let v_starts = [1i32]; // v_start = 1 let ilens = [-2i32]; // deletion of 2 → v_rel_end = 1 - (-2) + 1 = 4... wait - // v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 - // Actually: v_rel_end = 1 - min(0, -2) + 1 = 1 - (-2) + 1 = 4 - // v_len = max(0, -2) + 1 = 0 + 1 = 1 - // track up to v_rel_pos=1: track[0..1] = [10.0], out[0] = 10.0 - // v_len=1 repeated: out[1] = track[1] = 20.0 - // track_idx = 4; remaining: track[4..5] = [50.0] → out[2] = 50.0 - // out[3] = 0.0 (trailing pad) + // v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 + // Actually: v_rel_end = 1 - min(0, -2) + 1 = 1 - (-2) + 1 = 4 + // v_len = max(0, -2) + 1 = 0 + 1 = 1 + // track up to v_rel_pos=1: track[0..1] = [10.0], out[0] = 10.0 + // v_len=1 repeated: out[1] = track[1] = 20.0 + // track_idx = 4; remaining: track[4..5] = [50.0] → out[2] = 50.0 + // out[3] = 0.0 (trailing pad) let geno_v_idxs = [0i32]; let geno_offsets = [0i64, 1]; @@ -1605,8 +1884,14 @@ mod tests { // out[0..4] from main loop; zero-pad covers out[4..8] from out_idx (not index 2). assert_eq!(result[0], 1.0f32, "ref[0]"); assert_eq!(result[1], 2.0f32, "ref[1]"); - assert_eq!(result[2], 3.0f32, "ref[2] — must NOT be overwritten by zero-pad"); - assert_eq!(result[3], 4.0f32, "deletion REPEAT_5P value — must NOT be overwritten"); + assert_eq!( + result[2], 3.0f32, + "ref[2] — must NOT be overwritten by zero-pad" + ); + assert_eq!( + result[3], 4.0f32, + "deletion REPEAT_5P value — must NOT be overwritten" + ); assert_eq!(result[4], 0.0f32, "zero-pad[4]"); assert_eq!(result[5], 0.0f32, "zero-pad[5]"); assert_eq!(result[6], 0.0f32, "zero-pad[6]"); @@ -1696,7 +1981,11 @@ mod tests { 0, ); // SNP is skipped — output equals track[:length] - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "SNP must be skipped for tracks"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "SNP must be skipped for tracks" + ); } /// Insertion with REPEAT_5P strategy: repeated track[v_rel_pos]. @@ -1813,7 +2102,11 @@ mod tests { 0, 0, ); - assert_eq!(result, [0.0f32, 1.0, 2.0, 3.0], "no variants + shift=0: copy track[:4]"); + assert_eq!( + result, + [0.0f32, 1.0, 2.0, 3.0], + "no variants + shift=0: copy track[:4]" + ); } /// Shift=2 with one insertion variant: verify shift-through-variant logic. @@ -1873,7 +2166,7 @@ mod tests { out_len: usize, out_offsets: &[i64], regions: &[[i32; 3]], - shifts: &[i32], // flat, will be reshaped (n_q, ploidy) + shifts: &[i32], // flat, will be reshaped (n_q, ploidy) geno_offset_idx: &[i64], // flat (n_q * ploidy) geno_v_idxs: &[i32], geno_offsets_1d: &[i64], @@ -1900,16 +2193,8 @@ mod tests { regions.iter().flat_map(|r| r.iter().cloned()).collect(), ) .unwrap(); - let shifts_arr = Array2::from_shape_vec( - (n_q, ploidy), - shifts.to_vec(), - ) - .unwrap(); - let goi_arr = Array2::from_shape_vec( - (n_q, ploidy), - geno_offset_idx.to_vec(), - ) - .unwrap(); + let shifts_arr = Array2::from_shape_vec((n_q, ploidy), shifts.to_vec()).unwrap(); + let goi_arr = Array2::from_shape_vec((n_q, ploidy), geno_offset_idx.to_vec()).unwrap(); let out_offsets_arr = Array1::from_vec(out_offsets.to_vec()); let gvi_arr = Array1::from_vec(geno_v_idxs.to_vec()); @@ -1991,7 +2276,11 @@ mod tests { 1, // ploidy false, ); - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "batch single: copy track[:4]"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "batch single: copy track[:4]" + ); } /// Batch with 2 queries, 1 hap each, SNPs — must pass through unchanged. @@ -2031,9 +2320,17 @@ mod tests { false, ); // SNP skipped → query 0 output = track[0..3] - assert_eq!(result[..3], [1.0f32, 2.0, 3.0], "q0: SNP skipped, track copied"); + assert_eq!( + result[..3], + [1.0f32, 2.0, 3.0], + "q0: SNP skipped, track copied" + ); // No variants in q1 → track[3..6] - assert_eq!(result[3..], [4.0f32, 5.0, 6.0], "q1: no variants, track copied"); + assert_eq!( + result[3..], + [4.0f32, 5.0, 6.0], + "q1: no variants, track copied" + ); } // ================================================================== // @@ -2054,7 +2351,7 @@ mod tests { // regions: (n_queries, 3) — (contig_idx, start, end) let regions_data = vec![ - 0i32, 0, 0, // q0: empty length + 0i32, 0, 0, // q0: empty length 0i32, 10, 13, // q1: [10, 13), length 3 0i32, 20, 25, // q2: [20, 25), length 5 ]; @@ -2071,7 +2368,11 @@ mod tests { tracks_to_intervals(regions.view(), tracks.view(), track_offsets.view(), false); // offsets: [0, 0, 1, 3] - assert_eq!(offsets.as_slice().unwrap(), &[0i64, 0, 1, 3], "offsets mismatch"); + assert_eq!( + offsets.as_slice().unwrap(), + &[0i64, 0, 1, 3], + "offsets mismatch" + ); // Total intervals = 3 assert_eq!(starts.len(), 3); @@ -2148,7 +2449,11 @@ mod tests { tracks_to_intervals(regions.view(), tracks.view(), track_offsets.view(), false); assert_eq!(offsets.as_slice().unwrap(), &[0i64, 3]); - assert_eq!(starts.len(), 3, "must have 3 intervals including zero-value ones"); + assert_eq!( + starts.len(), + 3, + "must have 3 intervals including zero-value ones" + ); assert_eq!(values[0], 0.0f32, "first interval is zero-value"); assert_eq!(starts[0], 0i32); assert_eq!(ends[0], 2i32); @@ -2157,4 +2462,92 @@ mod tests { assert_eq!(starts[2], 3i32); assert_eq!(ends[2], 4i32); } + + // ------------------------------------------------------------------ // + // shift_and_realign_tracks_from_svar2 (Task 4 Part C) // + // ------------------------------------------------------------------ // + + /// SVAR2 two-source track driver: a single pure DEL, carried entirely in the + /// `dense` channel (no `var_key` entries), realigns a track identically to the + /// SVAR1 kernel's `test_singular_deletion` case above (same track, v_start, ilen, + /// strategy_id=REPEAT_5P — any strategy_id works for a DEL, since v_diff<0 always + /// takes the `else` branch and repeats `track[v_rel_pos]`, bypassing + /// `apply_insertion_fill` entirely; REPEAT_5P is reused here only to match that + /// existing test's derivation for an easy cross-check). + /// + /// Setup: R=1, S=1 (ploidy=1) -> n_work=1. + /// track = [10.0, 20.0, 30.0, 40.0, 50.0], query_start = 0, out_len = 4 + /// dense pure DEL at pos=1, ilen=-2 — `svar2_codec::encode_pure_del(x)` round-trips + /// through `decode_alt` to `v_diff = x` (see `DecodedKey::PureDel { ilen }`), so + /// `encode_pure_del(-2)` below yields `v_diff = -2` to match the derivation. + /// + /// Hand-derivation (identical math to `test_singular_deletion`): + /// v_rel_pos = 1 - 0 = 1; v_diff = -2 + /// v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 + /// v_len = max(0, -2) + 1 = 1 + /// track_len = v_rel_pos - track_idx = 1 - 0 = 1 -> out[0] = track[0] = 10.0 + /// writable_length = min(1, 4-1) = 1; v_diff<0 -> else branch: + /// out[1] = track[v_rel_pos=1] = 20.0 + /// track_idx = v_rel_end = 4; out_idx = 2 + /// tail: unfilled_length = 4-2 = 2; writable_ref = min(2, track.len()(5)-4) = 1 + /// -> out[2] = track[4] = 50.0; out_end_idx = 3 + /// out[3] = 0.0 (trailing pad, out_end_idx(3) < length(4)) + /// expected out = [10.0, 20.0, 50.0, 0.0] + #[test] + fn svar2_track_realign_del() { + let track = arr1(&[10.0f32, 20.0, 30.0, 40.0, 50.0]); + let track_offsets = arr1(&[0i64, 5]); + + // One region on contig 0 (unused by tracks beyond regions[[q,1]]=query_start), + // one sample-slot, one ploid -> n_work = 1. + let regions = ndarray::arr2(&[[0i32, 0, 4]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // No var_key entries for this hap. + let vk_pos = arr1::(&[]); + let vk_key = arr1::(&[]); + let vk_off = arr1(&[0i64, 0]); + + // dense channel: one pure DEL (ilen=-2) at pos=1, present for this hap. + let del_key = svar2_codec::encode_pure_del(-2) as i32; + let dense_pos = arr1(&[1i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + // Presence bits, LSB-first: bit 0 (this hap's only dense entry) is set. + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + + // No long-allele-bank lookups exercised in this test. + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + + let out_offsets = arr1(&[0i64, 4]); + let params = arr1(&[0.0f64]); + let mut out = Array1::::zeros(4); + + shift_and_realign_tracks_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + track.view(), + track_offsets.view(), + params.view(), + REPEAT_5P, + 0, + false, // serial + ); + + assert_eq!(out.as_slice().unwrap(), &[10.0f32, 20.0, 50.0, 0.0]); + } } From 4af5ac0b960c41328fbf7f3308766fab83d52a11 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 13:16:58 -0700 Subject: [PATCH 005/108] feat(svar2): PyO3 fused wrappers for two-source reconstruct/realign (gvl M6b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 5. Expose the SVAR2 drivers to Python, sizing + allocating output Rust-side. - svar2::hap_diffs_svar2: per-hap applied-ilen diff, an exact port of genotypes::get_diffs_sparse's q_start/q_end-clipped branch, sourcing (v_start, v_ilen) from merge_hap + decode_alt instead of the global table. Sizes the fused outputs. - ffi::reconstruct_haplotypes_from_svar2 (#[pyfunction]): mirrors reconstruct_haplotypes_fused — hap_diffs -> out_offsets (ragged or fixed output_length) -> uninit_output -> reconstruct_haplotypes_from_svar2 driver -> (out_data u8, out_offsets) into_pyarray. - ffi::shift_and_realign_tracks_from_svar2 (#[pyfunction]): same sizing (realigned track has haplotype length), zeros f32 buffer, calls the track driver, returns (out f32, offsets). - Registered both in the #[pymodule]. Verified: hap_diffs unit test passes; full lib 116 pass; maturin develop builds (abi3); both pyfunctions register; a no-variant FFI call round-trips (ref passthrough, offsets [0,8]). Reviewed (Sonnet): Spec ✅, Approved, no correctness issues, 2 Informational (rustfmt reflow of pre-existing code — whitespace-only; stale report text). FFI numeric correctness is validated end-to-end in Task 7. Also folds in accumulated rustfmt of pre-existing ffi/lib code. Co-Authored-By: Claude Opus 4.8 --- src/ffi/mod.rs | 442 ++++++++++++++++++++++++++++++++++++++++------- src/lib.rs | 22 ++- src/svar2/mod.rs | 109 ++++++++++++ 3 files changed, 502 insertions(+), 71 deletions(-) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 913af48b..47f03a8e 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1,6 +1,8 @@ //! PyO3 boundary for migrated core kernels. The ONLY place new kernels touch Python. use ndarray::Array1; -use numpy::{IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2, PyReadwriteArray1}; +use numpy::{ + IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2, PyReadwriteArray1, +}; use pyo3::prelude::*; use pyo3::types::PyDict; @@ -204,11 +206,8 @@ pub fn compact_keep_i32<'py>( row_offsets: PyReadonlyArray1, keep: PyReadonlyArray1, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::compact_keep_i32( - values.as_array(), - row_offsets.as_array(), - keep.as_array(), - ); + let (v, off) = + variants::compact_keep_i32(values.as_array(), row_offsets.as_array(), keep.as_array()); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -221,11 +220,8 @@ pub fn compact_keep_f32<'py>( row_offsets: PyReadonlyArray1, keep: PyReadonlyArray1, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::compact_keep_f32( - values.as_array(), - row_offsets.as_array(), - keep.as_array(), - ); + let (v, off) = + variants::compact_keep_f32(values.as_array(), row_offsets.as_array(), keep.as_array()); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -238,11 +234,7 @@ pub fn fill_empty_scalar_i32<'py>( offsets: PyReadonlyArray1, fill: i32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_scalar_i32( - data.as_array(), - offsets.as_array(), - fill, - ); + let (v, off) = variants::fill_empty_scalar_i32(data.as_array(), offsets.as_array(), fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -255,11 +247,7 @@ pub fn fill_empty_scalar_f32<'py>( offsets: PyReadonlyArray1, fill: f32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_scalar_f32( - data.as_array(), - offsets.as_array(), - fill, - ); + let (v, off) = variants::fill_empty_scalar_f32(data.as_array(), offsets.as_array(), fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -273,12 +261,7 @@ pub fn fill_empty_fixed_i32<'py>( inner: i64, fill: i32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_fixed_i32( - data.as_array(), - offsets.as_array(), - inner, - fill, - ); + let (v, off) = variants::fill_empty_fixed_i32(data.as_array(), offsets.as_array(), inner, fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -292,12 +275,7 @@ pub fn fill_empty_fixed_f32<'py>( inner: i64, fill: f32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_fixed_f32( - data.as_array(), - offsets.as_array(), - inner, - fill, - ); + let (v, off) = variants::fill_empty_fixed_f32(data.as_array(), offsets.as_array(), inner, fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -322,7 +300,11 @@ pub fn fill_empty_seq_u8<'py>( seq_offsets.as_array(), dummy.as_array(), ); - (nd.into_pyarray(py), nvar.into_pyarray(py), nseq.into_pyarray(py)) + ( + nd.into_pyarray(py), + nvar.into_pyarray(py), + nseq.into_pyarray(py), + ) } /// Two-level dummy-fill for token windows (int32). @@ -346,7 +328,11 @@ pub fn fill_empty_seq_i32<'py>( seq_offsets.as_array(), dummy.as_array(), ); - (nd.into_pyarray(py), nvar.into_pyarray(py), nseq.into_pyarray(py)) + ( + nd.into_pyarray(py), + nvar.into_pyarray(py), + nseq.into_pyarray(py), + ) } /// Build the `{name: (data, seq_offsets)}` dict from assembled buffers. @@ -461,9 +447,26 @@ pub fn assemble_variant_buffers_u8<'py>( pad_char: u8, ) -> Bound<'py, PyDict> { assemble_variant_buffers_impl::( - py, mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, - ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, - lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char, + py, + mode, + v_idxs, + row_offsets, + alt_global, + alt_off_global, + ref_global, + ref_off_global, + want_ref_bytes, + want_flank, + ref_mode, + alt_mode, + flank_len, + lut, + v_contigs, + v_starts, + ilens, + reference, + ref_offsets, + pad_char, ) } @@ -493,9 +496,26 @@ pub fn assemble_variant_buffers_i32<'py>( pad_char: u8, ) -> Bound<'py, PyDict> { assemble_variant_buffers_impl::( - py, mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, - ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, - lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char, + py, + mode, + v_idxs, + row_offsets, + alt_global, + alt_off_global, + ref_global, + ref_off_global, + want_ref_bytes, + want_flank, + ref_mode, + alt_mode, + flank_len, + lut, + v_contigs, + v_starts, + ilens, + reference, + ref_offsets, + pad_char, ) } @@ -733,6 +753,270 @@ pub fn reconstruct_haplotypes_fused<'py>( (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) } +/// Fused SVAR2 two-source haplotype reconstruction: merge each hap's `var_key` ⋈ +/// `dense` channels and decode via `svar2-codec` inline (no materialized global +/// variant table), sizing and allocating the output buffer in Rust — one FFI +/// crossing, mirrors `reconstruct_haplotypes_fused` above. +/// +/// `output_length`: +/// - ``-1`` → ragged mode (each haplotype gets its natural length = ref_len + diff). +/// - ``>= 0`` → fixed-length mode (every haplotype is padded/truncated to this length). +/// +/// No annotation, no to_rc — first cut minimal, mirrors the plain fused path. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2<'py>( + py: Python<'py>, + regions: PyReadonlyArray2, + shifts: PyReadonlyArray2, + vk_pos: PyReadonlyArray1, + vk_key: PyReadonlyArray1, + vk_off: PyReadonlyArray1, + dense_pos: PyReadonlyArray1, + dense_key: PyReadonlyArray1, + dense_range: PyReadonlyArray2, + dense_present: PyReadonlyArray1, + dense_present_off: PyReadonlyArray1, + lut_bytes: PyReadonlyArray1, + lut_off: PyReadonlyArray1, + ref_: PyReadonlyArray1, + ref_offsets: PyReadonlyArray1, + pad_char: u8, + output_length: i64, + parallel: bool, +) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { + use crate::reconstruct; + use crate::svar2; + + let regions_a = regions.as_array(); + let shifts_a = shifts.as_array(); + let vk_pos_a = vk_pos.as_array(); + let vk_key_a = vk_key.as_array(); + let vk_off_a = vk_off.as_array(); + let dense_pos_a = dense_pos.as_array(); + let dense_key_a = dense_key.as_array(); + let dense_range_a = dense_range.as_array(); + let dense_present_a = dense_present.as_array(); + let dense_present_off_a = dense_present_off.as_array(); + let lut_bytes_a = lut_bytes.as_array(); + let lut_off_a = lut_off.as_array(); + let ref_a = ref_.as_array(); + let ref_offsets_a = ref_offsets.as_array(); + + let ploidy = shifts_a.ncols(); + let n_q = regions_a.nrows(); + let n_work = n_q * ploidy; + + let (out_data, out_offsets_vec) = py.detach(move || { + // Step 1: compute per-haplotype length diffs via the two-source diff core. + let vk_pos_s: &[i32] = vk_pos_a.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key_a.as_slice().unwrap(); + let vk_off_s: &[i64] = vk_off_a.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos_a.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key_a.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present_a.as_slice().unwrap(); + let dense_present_off_s: &[i64] = dense_present_off_a.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes_a.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off_a.as_slice().unwrap(); + + let diffs = svar2::hap_diffs_svar2( + regions_a, + ploidy, + vk_pos_s, + vk_key_s, + vk_off_s, + dense_pos_s, + dense_key_s, + dense_range_a, + dense_present_s, + dense_present_off_s, + lut_bytes_s, + lut_off_s, + ); + + // Step 2: compute per-haplotype output lengths and prefix-sum offsets. + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let len: i64 = if output_length >= 0 { + output_length + } else { + let ref_len = (regions_a[[query, 2]] - regions_a[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + (ref_len + diff).max(0) + }; + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust — Python never calls np.empty. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = uninit_output(total); + + // Step 4: reconstruct all haplotypes into the owned buffer. + reconstruct::reconstruct_haplotypes_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions_a, + shifts_a, + vk_pos_a, + vk_key_a, + vk_off_a, + dense_pos_a, + dense_key_a, + dense_range_a, + dense_present_a, + dense_present_off_a, + lut_bytes_a, + lut_off_a, + ref_a, + ref_offsets_a, + pad_char, + None, // annot_v_idxs — not supported in fused plain path + None, // annot_ref_pos — not supported in fused plain path + parallel, + ); + + (out_data, out_offsets_vec) + }); + + (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) +} + +/// Fused SVAR2 two-source track shift+realign: merge each hap's `var_key` ⋈ `dense` +/// channels and decode via `svar2-codec` inline, sizing and allocating the output +/// buffer in Rust — one FFI crossing, mirrors `reconstruct_haplotypes_from_svar2` +/// above but for f32 tracks (see `tracks::shift_and_realign_tracks_from_svar2`). +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2<'py>( + py: Python<'py>, + regions: PyReadonlyArray2, + shifts: PyReadonlyArray2, + vk_pos: PyReadonlyArray1, + vk_key: PyReadonlyArray1, + vk_off: PyReadonlyArray1, + dense_pos: PyReadonlyArray1, + dense_key: PyReadonlyArray1, + dense_range: PyReadonlyArray2, + dense_present: PyReadonlyArray1, + dense_present_off: PyReadonlyArray1, + lut_bytes: PyReadonlyArray1, + lut_off: PyReadonlyArray1, + tracks: PyReadonlyArray1, + track_offsets: PyReadonlyArray1, + params: PyReadonlyArray1, + strategy_id: i64, + base_seed: u64, + parallel: bool, +) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { + use crate::svar2; + use crate::tracks; + + let regions_a = regions.as_array(); + let shifts_a = shifts.as_array(); + let vk_pos_a = vk_pos.as_array(); + let vk_key_a = vk_key.as_array(); + let vk_off_a = vk_off.as_array(); + let dense_pos_a = dense_pos.as_array(); + let dense_key_a = dense_key.as_array(); + let dense_range_a = dense_range.as_array(); + let dense_present_a = dense_present.as_array(); + let dense_present_off_a = dense_present_off.as_array(); + let lut_bytes_a = lut_bytes.as_array(); + let lut_off_a = lut_off.as_array(); + let tracks_a = tracks.as_array(); + let track_offsets_a = track_offsets.as_array(); + let params_a = params.as_array(); + + let ploidy = shifts_a.ncols(); + let n_q = regions_a.nrows(); + let n_work = n_q * ploidy; + + let (out_data, out_offsets_vec) = py.detach(move || { + // Step 1: compute per-haplotype length diffs via the two-source diff core + // (a realigned track has haplotype length = ref_len + diff, same as reconstruct). + let vk_pos_s: &[i32] = vk_pos_a.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key_a.as_slice().unwrap(); + let vk_off_s: &[i64] = vk_off_a.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos_a.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key_a.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present_a.as_slice().unwrap(); + let dense_present_off_s: &[i64] = dense_present_off_a.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes_a.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off_a.as_slice().unwrap(); + + let diffs = svar2::hap_diffs_svar2( + regions_a, + ploidy, + vk_pos_s, + vk_key_s, + vk_off_s, + dense_pos_s, + dense_key_s, + dense_range_a, + dense_present_s, + dense_present_off_s, + lut_bytes_s, + lut_off_s, + ); + + // Step 2: compute per-haplotype output lengths and prefix-sum offsets. + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let ref_len = (regions_a[[query, 2]] - regions_a[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + let len: i64 = (ref_len + diff).max(0); + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust — Python never calls np.empty. + // f32 track fill writes every position it needs; zeros is a safe default. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = Array1::::zeros(total); + + // Step 4: realign all tracks into the owned buffer. + tracks::shift_and_realign_tracks_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions_a, + shifts_a, + vk_pos_a, + vk_key_a, + vk_off_a, + dense_pos_a, + dense_key_a, + dense_range_a, + dense_present_a, + dense_present_off_a, + lut_bytes_a, + lut_off_a, + tracks_a, + track_offsets_a, + params_a, + strategy_id, + base_seed, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) +} + /// Fused spliced-haplotype reconstruction: reconstruct in one FFI crossing using /// precomputed output offsets. /// @@ -940,7 +1224,7 @@ pub fn reconstruct_annotated_haplotypes_spliced_fused<'py>( pad_char, keep_a, keep_offsets_a, - Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide + Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide Some(annot_pos.view_mut()), // annot_ref_pos — reference coordinate per nucleotide parallel, ); @@ -956,9 +1240,21 @@ pub fn reconstruct_annotated_haplotypes_spliced_fused<'py>( out_offsets_a.len() - 1, "to_rc mask length must equal number of output rows (offsets.len() - 1)" ); - crate::reverse::rc_flat_rows_inplace(out_data.as_slice_mut().unwrap(), out_offsets_a, m); - crate::reverse::reverse_flat_rows_inplace(annot_v.as_slice_mut().unwrap(), out_offsets_a, m); - crate::reverse::reverse_flat_rows_inplace(annot_pos.as_slice_mut().unwrap(), out_offsets_a, m); + crate::reverse::rc_flat_rows_inplace( + out_data.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_v.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_pos.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); } (out_data, annot_v, annot_pos) @@ -1116,7 +1412,7 @@ pub fn reconstruct_annotated_haplotypes_fused<'py>( pad_char, keep_a, keep_offsets_a, - Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide + Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide Some(annot_pos.view_mut()), // annot_ref_pos — reference coordinate per nucleotide parallel, ); @@ -1128,9 +1424,21 @@ pub fn reconstruct_annotated_haplotypes_fused<'py>( out_offsets_vec.len() - 1, "to_rc mask length must equal number of output rows (offsets.len() - 1)" ); - crate::reverse::rc_flat_rows_inplace(out_data.as_slice_mut().unwrap(), out_offsets_vec.view(), m); - crate::reverse::reverse_flat_rows_inplace(annot_v.as_slice_mut().unwrap(), out_offsets_vec.view(), m); - crate::reverse::reverse_flat_rows_inplace(annot_pos.as_slice_mut().unwrap(), out_offsets_vec.view(), m); + crate::reverse::rc_flat_rows_inplace( + out_data.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_v.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_pos.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); } (out_data, annot_v, annot_pos, out_offsets_vec) @@ -1301,22 +1609,22 @@ pub fn tracks_to_intervals<'py>( #[allow(clippy::too_many_arguments)] pub fn intervals_and_realign_track_fused( py: Python<'_>, - mut out: PyReadwriteArray1, // (b*p*l) — caller's per-track slice - out_offsets: PyReadonlyArray1, // (b*p + 1) - regions: PyReadonlyArray2, // (b, 3) - shifts: PyReadonlyArray2, // (b, p) - geno_offset_idx: PyReadonlyArray2, // (b, p) - geno_v_idxs: PyReadonlyArray1, // (r*s*p*v) - geno_offsets: PyReadonlyArray2, // (2, r*s*p) - v_starts: PyReadonlyArray1, // (tot_v) - ilens: PyReadonlyArray1, // (tot_v) + mut out: PyReadwriteArray1, // (b*p*l) — caller's per-track slice + out_offsets: PyReadonlyArray1, // (b*p + 1) + regions: PyReadonlyArray2, // (b, 3) + shifts: PyReadonlyArray2, // (b, p) + geno_offset_idx: PyReadonlyArray2, // (b, p) + geno_v_idxs: PyReadonlyArray1, // (r*s*p*v) + geno_offsets: PyReadonlyArray2, // (2, r*s*p) + v_starts: PyReadonlyArray1, // (tot_v) + ilens: PyReadonlyArray1, // (tot_v) // intervals (reference-coordinate, for this track) - offset_idxs: PyReadonlyArray1, // (b) — per-query index into itv_offsets - itv_starts: PyReadonlyArray1, // (n_intervals) - itv_ends: PyReadonlyArray1, // (n_intervals) - itv_values: PyReadonlyArray1, // (n_intervals) - itv_offsets: PyReadonlyArray1, // (n_samples*n_regions + 1) - track_offsets: PyReadonlyArray1, // (b+1) — out_offsets for scratch buffer + offset_idxs: PyReadonlyArray1, // (b) — per-query index into itv_offsets + itv_starts: PyReadonlyArray1, // (n_intervals) + itv_ends: PyReadonlyArray1, // (n_intervals) + itv_values: PyReadonlyArray1, // (n_intervals) + itv_offsets: PyReadonlyArray1, // (n_samples*n_regions + 1) + track_offsets: PyReadonlyArray1, // (b+1) — out_offsets for scratch buffer // insertion-fill strategy params: PyReadonlyArray1, strategy_id: i64, @@ -1456,9 +1764,9 @@ mod tests { #[test] fn annotated_rc_complements_bytes_reverses_indices() { - let mut bytes = b"ACG".to_vec(); // revcomp -> "CGT" - let mut vidx = vec![5i32, 6, 7]; // reverse -> [7,6,5] - let mut rpos = vec![100i32, 101, 102]; // reverse -> [102,101,100] + let mut bytes = b"ACG".to_vec(); // revcomp -> "CGT" + let mut vidx = vec![5i32, 6, 7]; // reverse -> [7,6,5] + let mut rpos = vec![100i32, 101, 102]; // reverse -> [102,101,100] let offsets = ndarray::array![0i64, 3]; let m = ndarray::array![true]; crate::reverse::rc_flat_rows_inplace(&mut bytes, offsets.view(), m.view()); diff --git a/src/lib.rs b/src/lib.rs index 19ac9f78..b8a9db15 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,12 +39,26 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(ffi::assemble_variant_buffers_i32, m)?)?; m.add_function(wrap_pyfunction!(ffi::rc_alleles, m)?)?; m.add_function(wrap_pyfunction!(ffi::get_reference, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_from_sparse, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_from_sparse, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_annotated_haplotypes_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_spliced_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_annotated_haplotypes_spliced_fused, m)?)?; + m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_from_svar2, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_annotated_haplotypes_fused, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_spliced_fused, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_annotated_haplotypes_spliced_fused, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::shift_and_realign_tracks_sparse, m)?)?; + m.add_function(wrap_pyfunction!(ffi::shift_and_realign_tracks_from_svar2, m)?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 5e545d9a..cf79705f 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; +use ndarray::{Array2, ArrayView2}; use svar2_codec::{decode_key, DecodedKey}; /// Decode one uniform key into `(v_diff, allele)`, resolving long-INS via the LUT @@ -49,6 +50,72 @@ pub fn merge_hap( a } +/// Per-hap applied-ilen diff for the two-source path, mirroring +/// `genotypes::get_diffs_sparse`'s q_start/q_end-clipped branch. Used to size the fused +/// SVAR2 reconstruct/track outputs. Serial (n is tiny; the fused callers already parallelize +/// the heavy reconstruct pass). +#[allow(clippy::too_many_arguments)] +pub fn hap_diffs_svar2( + regions: ArrayView2, // (n_q, 3) + ploidy: usize, + vk_pos: &[i32], + vk_key: &[i32], + vk_off: &[i64], // (n_work+1) + dense_pos: &[i32], + dense_key: &[i32], + dense_range: ArrayView2, // (n_q, 2) + dense_present: &[u8], + dense_present_off: &[i64], // (n_work+1) BIT offsets + lut_bytes: &[u8], + lut_off: &[i64], +) -> Array2 { + let n_q = regions.nrows(); + let mut diffs = Array2::::zeros((n_q, ploidy)); + for k in 0..(n_q * ploidy) { + let query = k / ploidy; + let hap = k % ploidy; + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| -> bool { + let bit = base_bit + j; + (dense_present[bit / 8] >> (bit % 8)) & 1 == 1 + }; + let merged = merge_hap(vk_pos, vk_key, vk_lo, vk_hi, dense_pos, dense_key, ds, de, present_bit); + if merged.is_empty() { + continue; + } + let q_start = regions[[query, 1]] as i64; + let q_end = regions[[query, 2]] as i64; + let mut ref_idx = q_start; + let mut acc: i64 = 0; + for &(pos, key) in &merged { + let v_start = pos as i64; + let (mut v_ilen, _allele) = decode_alt(key, lut_bytes, lut_off); + let v_end = v_start - v_ilen.min(0) + 1; + if v_end <= q_start { + continue; + } + if v_start >= q_end { + break; + } + if v_start >= q_start && v_start < ref_idx { + continue; + } + ref_idx = ref_idx.max(v_end); + if v_ilen < 0 { + v_ilen += (q_start - v_start - 1).max(0); + } + v_ilen += (v_end - q_end).max(0); + acc += v_ilen; + } + diffs[[query, hap]] = acc as i32; + } + diffs +} + #[cfg(test)] mod tests { use super::*; @@ -105,4 +172,46 @@ mod tests { vec![(10, 100), (15, 150), (20, 200), (20, 250), (30, 300)] ); } + + #[test] + fn test_hap_diffs_svar2_snp_and_del() { + // 1 query, 1 hap, region [0, 100). Two var_key entries, no dense entries: + // a SNP at pos 10 (ilen 0) and a single-base DEL at pos 20 (ilen -1), both + // fully inside the region. Expected diff = 0 + (-1) = -1. + let regions = ndarray::array![[0i32, 0, 100]]; + let ploidy = 1usize; + + let vk_pos = [10i32, 20]; + let vk_key = [ + svar2_codec::encode_alt_inline(b"A", 0) as i32, + svar2_codec::encode_pure_del(-1) as i32, + ]; + let vk_off: [i64; 2] = [0, 2]; + + let dense_pos: [i32; 0] = []; + let dense_key: [i32; 0] = []; + let dense_range = ndarray::array![[0i32, 0]]; + let dense_present: [u8; 0] = []; + let dense_present_off: [i64; 2] = [0, 0]; + + let lut_bytes: [u8; 0] = []; + let lut_off: [i64; 0] = []; + + let diffs = hap_diffs_svar2( + regions.view(), + ploidy, + &vk_pos, + &vk_key, + &vk_off, + &dense_pos, + &dense_key, + dense_range.view(), + &dense_present, + &dense_present_off, + &lut_bytes, + &lut_off, + ); + + assert_eq!(diffs[[0, 0]], -1); + } } From ed00d484680c7707add0ae60291fae7704945304 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 13:31:13 -0700 Subject: [PATCH 006/108] feat(svar2): additive Python SVAR2 reconstruction adapter (gvl M6b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 6 (Step 1). SparseVar2Source: bridges genoray SparseVar2.overlap_batch's raw two-channel dict to the SVAR2 kernels (reconstruct_haplotypes_from_svar2 / shift_and_realign_tracks_from_svar2), returning the same Ragged/_Flat types as SVAR 1.0. Layout mapping (crux): genoray lays out H=R*S*P haps region-major h=(r*S+s)*P+p; the gvl driver indexes k=query*P+hap and reads dense_range/regions by query. So n_q=R*S and dense_range/regions are expanded (np.repeat S times per region) to align dense_range_gvl[r*S+s] == genoray dense_range[r]. vk_off/dense_present_off are already H-indexed (=k), passed through. Scope: Step 1 (self-contained adapter) only. The plan's Step 2 (wiring into Haps/Dataset dispatch) is DEFERRED — it touches the central SVAR 1.0 reconstructor and is gated by no test in this plan; Task 7 validates this adapter directly. Noted as TODO(svar2-dataset-dispatch) in the module docstring. SVAR 1.0 path untouched. Verified: module imports; exposes reconstruct + realign_tracks. End-to-end correctness is validated in Task 7 vs genoray's SparseVar2.decode oracle. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_source.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 python/genvarloader/_dataset/_svar2_source.py diff --git a/python/genvarloader/_dataset/_svar2_source.py b/python/genvarloader/_dataset/_svar2_source.py new file mode 100644 index 00000000..38b49a15 --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_source.py @@ -0,0 +1,136 @@ +"""SVAR2 two-source reconstruction adapter. + +Bridges genoray ``SparseVar2.overlap_batch``'s raw two-channel dict to gvl's SVAR2 kernels +(``reconstruct_haplotypes_from_svar2`` / ``shift_and_realign_tracks_from_svar2``), decoding +``var_key ⋈ dense`` inline with no intermediate variant table. Additive to the SVAR 1.0 path. + +TODO(svar2-dataset-dispatch): wiring this into ``Haps``/``Dataset`` (an svar2-source flag on the +dataset that routes reconstruction here) is deferred — it touches the central SVAR 1.0 reconstructor. +This adapter is self-contained and is validated end-to-end in tests against genoray's decode oracle. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import numpy as np +from seqpro.rag import Ragged + +from .._flat import _Flat +from ..genvarloader import ( + reconstruct_haplotypes_from_svar2, + shift_and_realign_tracks_from_svar2, +) + +if TYPE_CHECKING: + from genoray import SparseVar2 + from numpy.typing import NDArray + + +class SparseVar2Source: + """Reconstruct haplotypes / realign tracks from a genoray ``SparseVar2`` via the two-source path.""" + + def __init__(self, svar2: "SparseVar2") -> None: + self.svar2 = svar2 + + def _query(self, contig, regions): + d = self.svar2.overlap_batch(contig, [(int(s), int(e)) for s, e in regions]) + R = int(d["n_regions"]) + S = int(d["n_samples"]) + P = int(d["ploidy"]) + reg = np.asarray(regions, dtype=np.int32).reshape(R, 2) + # (R*S, 3): contig_idx=0, start, end — repeat each query region S times. + reg_rs = np.repeat(reg, S, axis=0) # (R*S, 2) + regions_gvl = np.zeros((R * S, 3), dtype=np.int32) + regions_gvl[:, 1:] = reg_rs + dense_range_gvl = np.ascontiguousarray( + np.repeat(np.asarray(d["dense_range"], np.int32), S, axis=0), np.int32 + ) # (R*S, 2) + return d, R, S, P, regions_gvl, dense_range_gvl + + def reconstruct( + self, + contig: str, + regions, # iterable of (start, end), length R + ref_: "NDArray[np.uint8]", # the contig reference bytes + ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) + pad_char: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + output_length: int = -1, + parallel: bool = False, + ) -> "Ragged[np.bytes_]": + d, R, S, P, regions_gvl, dense_range_gvl = self._query(contig, regions) + n_q = R * S + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + out_data, out_offsets = reconstruct_haplotypes_from_svar2( + np.ascontiguousarray(regions_gvl, np.int32), + shifts_a, + np.ascontiguousarray(d["vk_pos"], np.int32), + np.ascontiguousarray(d["vk_key"], np.int32), + np.ascontiguousarray(d["vk_off"], np.int64), + np.ascontiguousarray(d["dense_pos"], np.int32), + np.ascontiguousarray(d["dense_key"], np.int32), + dense_range_gvl, + np.ascontiguousarray(d["dense_present"], np.uint8), + np.ascontiguousarray(d["dense_present_off"], np.int64), + np.ascontiguousarray(d["lut_bytes"], np.uint8), + np.ascontiguousarray(d["lut_off"], np.int64), + np.ascontiguousarray(ref_, np.uint8), + np.ascontiguousarray(ref_offsets, np.int64), + np.uint8(pad_char), + np.int64(output_length), + parallel, + ) + shape = (R, S, P, None) + return cast("Ragged[np.bytes_]", _Flat.from_offsets(out_data, shape, out_offsets).view("S1")) + + def realign_tracks( + self, + contig: str, + regions, + tracks: "NDArray[np.float32]", # flat per-query track buffer + track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks + params: "NDArray[np.float64]", + strategy_id: int, + base_seed: int, + shifts: "NDArray[np.int32] | None" = None, + parallel: bool = False, + ) -> "Ragged[np.float32]": + d, R, S, P, regions_gvl, dense_range_gvl = self._query(contig, regions) + n_q = R * S + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + # tracks are per query REGION (R of them); the driver reads track_offsets by `query` + # (= r*S+s), so expand the R track windows to R*S by repeating each S times. + t = np.asarray(tracks, np.float32) + toff = np.asarray(track_offsets, np.int64) + tracks_rs = np.concatenate([t[toff[r]:toff[r + 1]] for r in range(R) for _ in range(S)]) if R else t + lengths = np.repeat(np.diff(toff), S) + track_offsets_rs = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + out_data, out_offsets = shift_and_realign_tracks_from_svar2( + np.ascontiguousarray(regions_gvl, np.int32), + shifts_a, + np.ascontiguousarray(d["vk_pos"], np.int32), + np.ascontiguousarray(d["vk_key"], np.int32), + np.ascontiguousarray(d["vk_off"], np.int64), + np.ascontiguousarray(d["dense_pos"], np.int32), + np.ascontiguousarray(d["dense_key"], np.int32), + dense_range_gvl, + np.ascontiguousarray(d["dense_present"], np.uint8), + np.ascontiguousarray(d["dense_present_off"], np.int64), + np.ascontiguousarray(d["lut_bytes"], np.uint8), + np.ascontiguousarray(d["lut_off"], np.int64), + np.ascontiguousarray(tracks_rs, np.float32), + track_offsets_rs, + np.ascontiguousarray(params, np.float64), + np.int64(strategy_id), + np.uint64(base_seed), + parallel, + ) + shape = (R, S, P, None) + return cast("Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets)) From 7fe7a1abd7a6f8478fe5340a272c046ddf221c1b Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 14:00:51 -0700 Subject: [PATCH 007/108] test(svar2): e2e two-source reconstruction vs genoray decode oracle (gvl M6b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 7. Builds a known SVAR2 store (genoray run_conversion_pipeline on a 40bp chr1 fixture: SNP@2, INS@6 C>CAT, DEL@11 GTA>G across 2 samples × 2 ploids), reconstructs haplotypes through gvl's two-source path (SparseVar2Source.reconstruct → overlap_batch → reconstruct_haplotypes_from_svar2), and compares byte-for-byte against an INDEPENDENT pure-Python consensus applied to genoray's materialized SparseVar2 decode records (M6c oracle). Agreement proves gvl's var_key⋈dense merge+decode == genoray decode AND the reconstruction loop == an independent reference, across all 4 haps (counts [2,2,1,2]). Asserts non-triviality + indel length sensitivity (min<40 --- tests/test_svar2_reconstruct.py | 147 ++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/test_svar2_reconstruct.py diff --git a/tests/test_svar2_reconstruct.py b/tests/test_svar2_reconstruct.py new file mode 100644 index 00000000..83c9d662 --- /dev/null +++ b/tests/test_svar2_reconstruct.py @@ -0,0 +1,147 @@ +"""Cross-repo end-to-end validation of the SVAR2 two-source reconstruction kernel. + +Builds a known SVAR2 store from a VCF+FASTA fixture (genoray's conversion pipeline), +reconstructs haplotypes through gvl's two-source path (SparseVar2Source, which decodes +genoray's raw two-channel overlap_batch inline), and compares byte-for-byte against an +INDEPENDENT pure-Python consensus applied to genoray's materialized decode records +(the M6c oracle). Agreement proves: (a) gvl's var_key⋈dense merge+decode matches +genoray's decode, and (b) gvl's reconstruction loop matches an independent reference. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], + 25_000, 2, 1, 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _consensus(ref: bytes, pos, ilen, alleles, q_start: int, q_end: int) -> bytes: + """Independent reference reconstruction: apply position-sorted (pos, ilen, allele) + records to `ref[q_start:q_end]`. A pure DEL has an empty allele — the anchor base + ref[pos] is retained and the following |ilen| bases are dropped (genoray's convention). + """ + order = np.argsort(pos, kind="stable") + out = bytearray() + ref_idx = q_start + for i in order: + p = int(pos[i]) + il = int(ilen[i]) + al = bytes(alleles[i]) + v_end = p - min(0, il) + 1 + # DEL spanning the region start: advance ref past it, emit nothing. + if il < 0 and p < q_start and v_end >= q_start: + ref_idx = v_end + continue + if p < ref_idx: # overlapping variant already consumed — first-one-wins + continue + if p >= q_end: + break + out += ref[ref_idx:p] + seq = al if len(al) > 0 else ref[p : p + 1] + out += seq + ref_idx = v_end + out += ref[ref_idx:q_end] + return bytes(out) + + +def test_svar2_two_source_matches_decode_oracle(svar2_store): + import genoray + from genvarloader._dataset._svar2_source import SparseVar2Source + + contig = "chr1" + q_start, q_end = 0, 40 + regions = [(q_start, q_end)] + ref_bytes = _REF.encode() + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # --- two-source reconstruction (the path under test) --- + src = SparseVar2Source(sv) + hap_rag = src.reconstruct( + contig, + regions, + np.frombuffer(ref_bytes, np.uint8), + np.array([0, len(ref_bytes)], np.int64), + pad_char=ord("N"), + shifts=None, # no jitter + output_length=-1, # ragged + parallel=False, + ) + ts_data = np.asarray(hap_rag.data).view("S1").tobytes() + ts_off = np.asarray(hap_rag.offsets) + + # --- oracle: genoray's materialized decode records (raw flat dict) --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + H = R * So * Po + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + str_off = np.asarray(raw["str_off"]) # per-variant allele-byte offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + d_allele = np.asarray(raw["allele"]).tobytes() + + # Non-triviality: the fixture yields per-hap variant counts [2, 2, 1, 2] + # (S0h0, S0h1, S1h0, S1h1) — SNP/INS/DEL spread across samples and ploids. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [2, 2, 1, 2], per_hap_counts + + for h in range(H): + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = d_pos[gi0:gi1] + ilen_h = d_ilen[gi0:gi1] + alleles_h = [ + d_allele[int(str_off[gi]) : int(str_off[gi + 1])] for gi in range(gi0, gi1) + ] + expected = _consensus(ref_bytes, pos_h, ilen_h, alleles_h, q_start, q_end) + got = ts_data[int(ts_off[h]) : int(ts_off[h + 1])] + assert got == expected, ( + f"hap {h}: two-source {got!r} != oracle {expected!r} " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})" + ) + + # Sensitivity anchor: a DEL-carrying hap must be shorter than the reference, + # and an INS-only hap longer — proving indels actually change output length. + hap_lens = (ts_off[1:] - ts_off[:-1]).tolist() + assert min(hap_lens) < len(ref_bytes) < max(hap_lens), hap_lens From bead2cd22e2b80d5d695dffd5eafc48aa7e5d0c3 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 14:52:55 -0700 Subject: [PATCH 008/108] docs: spec for SVAR2 gvl MVP validate-and-benchmark session Design for A (worktree cleanup), C (realign_tracks e2e test), D (real-data chr21 validation), E (SVAR1-vs-SVAR2 benchmark), and a B (Dataset dispatch) design sketch. Sequencing: prove value on the SparseVar2Source adapter first, then wire B informed by the benchmark. Co-Authored-By: Claude Opus 4.8 --- ...svar2-gvl-mvp-validate-benchmark-design.md | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md diff --git a/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md new file mode 100644 index 00000000..c6f64710 --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md @@ -0,0 +1,193 @@ +# SVAR2 gvl MVP — validate & benchmark first, wire second + +**Date:** 2026-07-03 · **Epic:** SVAR 2.0 · **Repo:** GenVarLoader +**Worktree:** `.claude/worktrees/svar2-m6b-kernel` (branch `svar2-m6b-kernel`) +**Follows:** `tmp/handoffs/2026-07-03-svar2-m6-plan3-mvp-benchmark-handoff.md` + +## Summary + +Plan 3 delivered the SVAR2 two-source reconstruction kernels and a validated +`SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`). This +work finishes the MVP by **proving the SVAR2 value proposition on real chr21 data +before committing to invasive Dataset integration**. The session runs: + +- **A** — clean up merged genoray worktrees (mechanical) +- **C** — end-to-end test for the `realign_tracks` adapter path +- **D** — validate both backends on real chr21 germline + somatic data +- **E** — benchmark SVAR1-backed vs SVAR2-backed gvl (latency + store size) +- **B** — Dataset dispatch wiring — **design sketch only**, finalized in a later + brainstorm once E's numbers are in + +### Guiding decision + +Task D (validation) and Task E (benchmark) **do not depend on Task B** (full Dataset +dispatch): both run against `SparseVar2Source` directly. Task B is invasive to the +central reconstructor and its cost/shape depend on facts E will surface (notably the +all-samples-per-batch decode cost). So we **prove value first (A→C→D→E), then design +and wire B** with the benchmark in hand. This is measure-first and YAGNI: we do not +build invasive core integration before the payoff is confirmed. + +## Background: why SVAR2 doesn't fit the SVAR1 reconstructor + +The SVAR1 path (`Haps`, `_haps.py`) materializes per-region genotype offsets plus a +variant table on disk and memmaps them; `Haps` holds `genotypes` (a Ragged of +`v_idxs`), a `_Variants` table, and cached `ffi_static` arrays. + +The SVAR2 path is a **live-query** model: `SparseVar2Source` calls genoray +`SparseVar2.overlap_batch(contig, regions)` at request time and decodes +`var_key ⋈ dense` inline through the Plan-3 kernels — there is no materialized +genotype table. The two models share none of that state, which is why B is a new +reconstructor rather than a flag on `Haps` (see Task B). + +Three frictions make B genuinely invasive (deferred, but recorded here so the +validation/benchmark methodology accounts for them): + +1. **Query-granularity mismatch.** `overlap_batch` is *per-contig, all-samples, + all-ploidy* — it has no sample-subset argument (confirmed in genoray + `python/genoray/_svar2_batch.py`). gvl's `Reconstructor.__call__` receives an + arbitrary `idx` of `(region, sample)` pairs, possibly across multiple contigs. +2. **No write/detection path.** `_write.py` only handles `SparseVar`/VCF/PGEN; nothing + writes an SVAR2-backed gvl dataset, and `_build_seqs` (`_open.py`) has no SVAR2 + branch. +3. **Reconstructor shape.** A branch inside `Haps` would leave `genotypes`/`_Variants`/ + `ffi_static` dead for SVAR2. + +## Environment (from the handoff — do not re-derive) + +- gvl worktree build: `pixi run -e default maturin develop` after Rust edits; the + compiled module is `genvarloader.genvarloader` (abi3). +- Rust tests: `pixi run -e default cargo test --no-default-features [FILTER]`. +- Commits: prek hooks intentionally not installed here — use `git commit --no-verify`. +- genoray is the pre-built **2.15.0 wheel** in gvl's `default` env (not editable). + Rebuild via `cd /carter/users/dlaub/projects/genoray && pixi run -e py310 maturin + build --release` then re-point `pixi.toml` + `pixi install -e default`. + +## Task A — Worktree cleanup (mechanical) + +Confirm `git -C status` is clean, then remove the three merged/stale genoray +worktrees: + +``` +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6b +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6c +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6-core +``` + +No design surface. Abort a removal if that worktree reports uncommitted work. + +## Task C — `realign_tracks` e2e test + +`SparseVar2Source.realign_tracks` (the region→R·S track expansion) has no end-to-end +test; only the Rust driver unit test `svar2_track_realign_del` exists. + +**Oracle: option (a)** — gvl's own SVAR1 `shift_and_realign_tracks_sparse`, fed the +variants materialized from genoray `SparseVar2.decode`. It is a fully independent, +already-trusted code path (9 passing tests) and mirrors how +`tests/test_svar2_reconstruct.py` validates against a separate oracle. We reject the +pure-Python-consensus option (b): a second implementation to get right, and weaker than +reusing the trusted kernel. + +**Construction:** +- Reuse the store builder from `tests/test_svar2_reconstruct.py`. +- Use **DEL-only variants** so insertion-fill is bypassed and the strategy choice is + irrelevant (any valid `strategy_id`/`params` works — borrow from an existing gvl + track test). +- Define a per-region reference track (f32); realign through the SVAR2 adapter path and + through the SVAR1 oracle fed the decoded variants. +- **Assert per-`(r,s,p)`** equality between the two. + +Place alongside `tests/test_svar2_reconstruct.py`. + +## Task D — Real-data validation (exploratory, not pytest) + +Prove the whole thing on real chr21 in `/carter/users/dlaub/repos/for_loukik/`. + +**Inputs** +- Germline (1000G): `chr21.bcf` (177 MB, 3202 samples) — **no `.csi`**, must index. +- Somatic (GDC): `gdc.chr21.bcf` (1.1 GB, 16007 samples), `.csi` present. +- **Reference:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` (GDC GRCh38; + `.fai` and a gvl-preprocessed `.gvlfa` already present). + +**Steps** +1. Verify contig naming (`chr21` vs `21`) between the FASTA and each BCF; if the + germline BCF uses `21`, `bcftools annotate --rename-chrs` (or query with the BCF's + own naming). +2. `bcftools index --csi chr21.bcf` (germline). +3. Normalize+atomize both BCFs to biallelic: `bcftools norm -m -any --atomize -Ob`, + reusing the `genoray_pipeline.py` / `run_conversion_pipeline` recipe (samples via + `bcftools query -l`, chroms via the canonical `##contig` regex, + `run_conversion_pipeline(..., chunk_size=25_000, ploidy=2, ...)`). +4. Build **both** stores per source: `.svar` (SVAR1, `SparseVar.from_vcf`) and `.svar2` + (SVAR2, `run_conversion_pipeline`) — genoray 2.15.0 wheel. +5. **Validate** gvl returns haplotypes + variants through both backends: + - SVAR2 haplotypes: `SparseVar2Source(sv2).reconstruct("chr21", regions, + ref_bytes, ref_offsets, pad_char)`. + - SVAR2 variants: genoray `SparseVar2.decode`. + - SVAR1: a gvl `Dataset` over the `.svar` store (haplotype + variants modes). + - Spot-check a few regions agree in spirit (both from the same source variants). + +**Scope:** small — a handful of regions × a few samples for the correctness spot-check. +Correctness is already proven by the Rust + adapter tests; D proves the **real-data +plumbing**: that genoray converts real multi-thousand-sample BCFs and that the adapter +works on a non-synthetic store. + +**Compute:** germline may run interactively; the somatic conversion (16k samples, +1.1 GB) is heavy — run via `sbatch -p carter-compute`. + +**Success:** both stores build from the real BCFs, and gvl returns non-empty, sane +haplotypes + variants through both backends, with spot-checks agreeing. + +## Task E — Benchmark SVAR1-backed vs SVAR2-backed gvl + +Once D produces both stores, measure and tabulate. + +**Fairness rule (the crux):** query the **same workload** on both backends — **all +samples for a fixed region set**. This matches genoray's per-contig/all-samples query +granularity and is a realistic population-scale workload; a `(region, sample)`-subset +workload would unfairly tax the SVAR2 adapter, which always decodes all samples. + +**Method:** warm caches, N repeats, report **median**. Same regions/sample set for both +backends. + +**Measurements** (germline **and** somatic): +- **Hap latency** — SVAR1 gvl `Dataset` vs SVAR2 `SparseVar2Source`. +- **Variant latency** — SVAR1 gvl variants mode vs genoray `SparseVar2.decode`. +- **Store size** — `du` of the `.svar` (SVAR1) vs `.svar2` (SVAR2) directory. + +**Output:** a 2×3 table — germline/somatic × {hap latency, variant latency, store +size}. + +**Recorded caveat:** this compares adapter-vs-Dataset, not Dataset-vs-Dataset (B is not +wired). SVAR2 latency therefore excludes gvl's batching/collation overhead. Somatic +(dense somatic mutations) is where SVAR2's two-channel `var_key ⋈ dense` layout should +win on store size. + +## Task B — Dataset dispatch (design sketch; finalize after E) + +Deferred by the session's sequencing decision. Direction for the later brainstorm: + +- A **new `SparseVar2Reconstructor`** implementing the `Reconstructor` protocol + (`_protocol.py`) — *not* a branch inside `Haps` (SVAR2 shares none of `Haps`'s + `genotypes`/`_Variants`/`ffi_static` state). +- `__call__` adapts gvl's batch model to genoray's: group `idx`→`(r,s)` **by contig**, + decode all-samples-per-region via `SparseVar2Source`, **select** the requested + `(r,s,p)` rows, thread jitter `shifts` and contig `ref` bytes from `Reference` + (`_reference.py`), and return the identical `_Flat`/`Ragged` output contract. +- A new **write branch** (record an svar2 link + a `backend:"svar2"` flag in + `metadata.json`, no genotype materialization) plus **open-path** detection in + `_build_seqs` (`_open.py`). +- **Open question E will inform:** the all-samples-per-batch decode cost — whether to + push sample-subsetting down into genoray (`overlap_batch`) or accept + decode-all-then-select for the MVP. +- **Additive guarantee:** the SVAR 1.0 path must stay byte-identical; existing SVAR1 + haplotype/track tests must remain green. + +## Carried-over gotchas (from the handoff) + +- Subagents default to the **main repo, not the worktree** — every dispatch must `cd` + the gvl worktree and guard with `git rev-parse --show-toplevel`. The gvl worktree is + a different repo from genoray. +- `cargo test` needs `--no-default-features`; `maturin develop` after Rust edits. +- genoray `SparseVar2.decode`/`overlap_batch` return **empty ALT for pure DELs** — + reconstruction injects the anchor `ref[pos]`. Any SVAR2 consumer must honor this. +- genoray positions are **0-based**; hap order is region-major `h=(r·S+s)·P+p`. From cc085c3ad49bb8f36d757ccaa9e2de827db4d7e0 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 15:15:59 -0700 Subject: [PATCH 009/108] docs: implementation plan for SVAR2 gvl MVP validate+benchmark Adds the bite-sized task plan (A cleanup, C realign_tracks e2e test, D real-data chr21 validation, E SVAR1-vs-SVAR2 benchmark) and refines the spec's Task C oracle to the pure-Python SVAR1 shift_and_realign_track_sparse fed genoray decode records. Co-Authored-By: Claude Opus 4.8 --- ...-07-03-svar2-gvl-mvp-validate-benchmark.md | 564 ++++++++++++++++++ ...svar2-gvl-mvp-validate-benchmark-design.md | 34 +- 2 files changed, 586 insertions(+), 12 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md diff --git a/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md b/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md new file mode 100644 index 00000000..5193f6b4 --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md @@ -0,0 +1,564 @@ +# SVAR2 gvl MVP — Validate & Benchmark Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prove the SVAR2 gvl value proposition on real chr21 data — an e2e track test, real-data validation, and a SVAR1-vs-SVAR2 benchmark — before wiring SVAR2 into the Dataset (Task B, deferred). + +**Architecture:** SVAR2 reconstruction runs through the already-validated `SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`), which queries genoray `SparseVar2.overlap_batch` live. Task C adds the missing track-path test; Tasks D/E exercise the adapter on real germline/somatic chr21 stores. No changes to the SVAR 1.0 path. + +**Tech Stack:** Rust (PyO3/maturin) kernels, Python (numpy, polars, seqpro), genoray 2.15.0 wheel, bcftools/samtools, SLURM (`sbatch -p carter-compute`), pixi (`-e default`). + +**Spec:** `docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md` + +## Global Constraints + +- **Worktree guard:** all work happens in `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`. Every shell/subagent step must first `cd` there and verify `git rev-parse --show-toplevel` ends in `svar2-m6b-kernel`. The gvl worktree is a **different repo** from genoray. +- **Env:** run Python/pytest via `pixi run -e default ...`. Only the `default` env is installed. +- **Rust rebuild:** after any Rust edit, `pixi run -e default maturin develop` (editable install does not auto-rebuild). Not needed for this plan — no Rust edits are planned. +- **Rust tests** (if ever needed): `pixi run -e default cargo test --no-default-features [FILTER]`. +- **Commits:** prek hooks are intentionally not installed here — use `git commit --no-verify`. +- **Commit trailer:** end every commit message with `Co-Authored-By: Claude Opus 4.8 `. +- **genoray** is the pre-built **2.15.0 wheel** in gvl's `default` env (not editable). +- **Reference FASTA:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` (GDC GRCh38; `.fai` present). +- **Real data:** `/carter/users/dlaub/repos/for_loukik/chr21.bcf` (germline, 3202 samples, no `.csi`), `/carter/users/dlaub/repos/for_loukik/gdc.chr21.bcf` (somatic, 16007 samples, `.csi` present). +- **DEL-anchor convention:** genoray `decode`/`overlap_batch` return empty ALT for pure DELs; the anchor `ref[pos]` is injected downstream. Honor this in any oracle. +- **Out of scope:** Task B (Dataset dispatch) — deferred to its own brainstorm after Task E. + +--- + +### Task 1: Clean up merged genoray worktrees (Task A) + +**Files:** none in this repo. Operates on the **genoray** repo at `/carter/users/dlaub/projects/genoray`. + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing consumed by later tasks (independent housekeeping). + +- [ ] **Step 1: Confirm each target worktree has no uncommitted work** + +```bash +for wt in svar-2-m6b svar-2-m6c svar-2-m6-core; do + echo "=== $wt ===" + git -C /carter/users/dlaub/projects/genoray/.claude/worktrees/$wt status --porcelain 2>&1 || echo "(missing)" +done +``` +Expected: empty output under each header (clean). If any prints file paths, **STOP** and report — do not remove a worktree with uncommitted work. + +- [ ] **Step 2: Remove the three worktrees** + +```bash +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6b +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6c +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6-core +``` +Expected: no output (success). If a worktree is already gone, `git worktree remove` errors with "is not a working tree" — that's fine, continue. + +- [ ] **Step 3: Verify removal** + +```bash +git -C /carter/users/dlaub/projects/genoray worktree list +``` +Expected: none of `svar-2-m6b`, `svar-2-m6c`, `svar-2-m6-core` appear. + +No commit (worktree removal is not tracked in this repo). + +--- + +### Task 2: End-to-end test for `realign_tracks` (Task C) + +**Files:** +- Create: `tests/test_svar2_realign_tracks.py` + +**Interfaces:** +- Consumes: `genvarloader._dataset._svar2_source.SparseVar2Source.realign_tracks(contig, regions, tracks, track_offsets, params, strategy_id, base_seed, shifts=None, parallel=False) -> Ragged[np.float32]` (shape `(R, S, P, None)`); `genvarloader._dataset._tracks.shift_and_realign_track_sparse(offset_idx, geno_v_idxs, geno_offsets, v_starts, ilens, shift, track, query_start, out, params, keep=None, strategy_id=0, base_seed=0, query=0, hap=0)` (pure-Python oracle, fills `out` in place); genoray `SparseVar2` + `sv._readers[contig].decode_batch([(start, end)])`. +- Produces: nothing consumed by later tasks. + +**Oracle rationale (from spec):** the pure-Python SVAR1 `shift_and_realign_track_sparse` is a distinct implementation from the SVAR2 **Rust** kernel under test and already carries the DEL-anchor branch (`_tracks.py:755`). Feed it genoray's decoded `(pos, ilen)` per hap via a trivial synthetic layout. DEL-only variants bypass insertion-fill, so the strategy is irrelevant. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_svar2_realign_tracks.py`: + +```python +"""End-to-end validation of the SVAR2 track-realign adapter path. + +Builds a DEL-only SVAR2 store, realigns a reference track through gvl's SVAR2 +path (SparseVar2Source.realign_tracks, the Rust two-source kernel), and compares +per-(region, sample, ploid) against gvl's INDEPENDENT pure-Python SVAR1 track +realign (shift_and_realign_track_sparse) fed genoray's materialized decode +records. Agreement proves the SVAR2 Rust track kernel matches the trusted SVAR1 +realign semantics — including the DEL anchor. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). Two pure DELs chosen to match the reference exactly: +# POS 4 GTA>G -> 0-based pos 3, ilen -2 (ref[3:6] == "GTA") +# POS 10 GGG>G -> 0-based pos 9, ilen -2 (ref[9:12] == "GGG") +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t4\t.\tGTA\tG\t.\t.\t.\tGT\t1|0\t1|1 +chr1\t10\t.\tGGG\tG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def svar2_del_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_del") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], + 25_000, 2, 1, 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): + import genoray + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._tracks import shift_and_realign_track_sparse + + contig = "chr1" + q_start, q_end = 0, 40 + region_len = q_end - q_start + regions = [(q_start, q_end)] + + sv = genoray.SparseVar2(str(svar2_del_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # A per-region reference track (f32). Random-but-fixed so a positional bug + # can't hide behind a monotonic ramp. + rng = np.random.default_rng(0) + track = rng.random(region_len).astype(np.float32) + + strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) + params = np.zeros(1, np.float64) + base_seed = 0 + + # --- SVAR2 path under test: one region, expanded internally to R*S*P haps --- + src = SparseVar2Source(sv) + out_rag = src.realign_tracks( + contig, + regions, + track, # flat per-region track buffer + np.array([0, region_len], np.int64), # (R+1) offsets + params, + strategy_id, + base_seed, + shifts=None, # no jitter + parallel=False, + ) + + # --- oracle: genoray decode records -> pure-Python SVAR1 realign, per hap --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + + # Non-triviality: haps carry a varying number of DELs. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [1, 1, 2, 1], per_hap_counts + + for s in range(S): + for p in range(P): + h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = np.ascontiguousarray(d_pos[gi0:gi1], np.int32) + ilen_h = np.ascontiguousarray(d_ilen[gi0:gi1], np.int32) + n_h = gi1 - gi0 + + # Independently size the hap: region length + sum of (negative) ilens. + exp_len = region_len + int(ilen_h.sum()) + + got = np.asarray(out_rag[0, s, p]) + assert got.shape[0] == exp_len, ( + f"(s={s},p={p}) SVAR2 len {got.shape[0]} != expected {exp_len} " + f"(ilen={ilen_h.tolist()})" + ) + + # Synthetic single-hap SVAR1 layout: v_idxs 0..n_h, one group. + geno_v_idxs = np.arange(n_h, dtype=np.int32) + geno_offsets = np.array([0, n_h], np.int64) + expected = np.empty(exp_len, np.float32) + shift_and_realign_track_sparse( + offset_idx=0, + geno_v_idxs=geno_v_idxs, + geno_offsets=geno_offsets, + v_starts=pos_h, + ilens=ilen_h, + shift=0, + track=track, + query_start=q_start, + out=expected, + params=params, + strategy_id=strategy_id, + base_seed=base_seed, + query=0, + hap=h, + ) + np.testing.assert_allclose( + got, expected, rtol=0, atol=0, + err_msg=f"(s={s},p={p}) SVAR2 track != SVAR1 oracle " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", + ) +``` + +- [ ] **Step 2: Run the test** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +pixi run -e default pytest tests/test_svar2_realign_tracks.py -v +``` +Expected: **PASS**. This test validates already-working code (the SVAR2 track kernel), so it should pass on first run. If it FAILS: +- On the `per_hap_counts` assert → the decode layout differs from expectation; print `per_hap_counts` and adjust the expected list to the observed counts (the store is deterministic). +- On the length or `assert_allclose` → a **real** discrepancy between the SVAR2 Rust track kernel and the SVAR1 oracle. Do **not** loosen the tolerance. Use superpowers:systematic-debugging: inspect the failing `(s,p)`'s `got` vs `expected` arrays and the DEL anchor handling (`_tracks.py:755` vs the Rust `shift_and_realign_tracks_from_svar2`). + +- [ ] **Step 3: Commit** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tests/test_svar2_realign_tracks.py +git commit --no-verify -m "test: e2e SVAR2 realign_tracks vs SVAR1 pure-Python oracle + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +### Task 3: Real-data validation on chr21 (Task D) + +**Nature:** exploratory validation, **not** pytest. Deliverable is a committed notes file recording what was built and observed. Involves real BCFs and SLURM — has human-checkpointable steps. + +**Files:** +- Create: `tmp/svar2_mvp/build_stores.py` (store-builder script) +- Create: `tmp/svar2_mvp/validate.py` (backend-comparison script) +- Create: `docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md` (results/notes) + +**Interfaces:** +- Consumes: genoray `SparseVar.from_vcf(out, VCF(bcf), max_mem, samples=..., overwrite=True)`, `genoray._core.run_conversion_pipeline(bcf, ref, chroms, out, samples, chunk_size, ploidy, threads, long_allele_cap)`, `SparseVar2(store)`, `SparseVar2.decode(contig, regions)`, `SparseVar2Source.reconstruct(contig, regions, ref_, ref_offsets, pad_char, shifts=None, output_length=-1)`, `gvl.write(path, bed, variants=..., samples=..., overwrite=True)`, `gvl.Dataset.open(path, reference=...)`. +- Produces: two store pairs on disk (`/germline.svar`, `/germline.svar2`, `/somatic.svar`, `/somatic.svar2`) consumed by Task 4 (benchmark). + +Let `WORK=/carter/users/dlaub/repos/for_loukik/svar2_mvp` (create it; big outputs live outside the repo). Let `REF=/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. + +- [ ] **Step 1: Resolve contig naming and index the germline BCF** + +```bash +mkdir -p /carter/users/dlaub/repos/for_loukik/svar2_mvp +REF=/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa +echo "=== FASTA contigs (chr21 region) ==="; grep -E "^>.*(21|chr21)\b" "$REF.fai" 2>/dev/null || cut -f1 "$REF.fai" | grep -E "21$|chr21" +for b in /carter/users/dlaub/repos/for_loukik/chr21.bcf /carter/users/dlaub/repos/for_loukik/gdc.chr21.bcf; do + echo "=== $b contigs ==="; bcftools view -h "$b" | grep -E "^##contig" | grep -E "ID=(chr)?21," | head +done +# germline lacks a .csi: +[ -f /carter/users/dlaub/repos/for_loukik/chr21.bcf.csi ] || bcftools index --csi /carter/users/dlaub/repos/for_loukik/chr21.bcf +``` +Expected: identify the contig name each file uses. **Checkpoint:** if the FASTA uses `chr21` but a BCF uses `21` (common for 1000G germline), record the mismatch — later steps must query each store with **its own** contig name, and `run_conversion_pipeline`/`from_vcf` must be given the BCF's chrom name (`21`), while `reconstruct`/`decode` use that same store's naming. Note the resolved names in the notes file (Step 6). + +- [ ] **Step 2: Normalize+atomize both BCFs to biallelic** + +```bash +cd /carter/users/dlaub/repos/for_loukik +for src in chr21 gdc.chr21; do + bcftools norm -m -any --atomize -Ob -o svar2_mvp/${src}.norm.bcf ${src}.bcf + bcftools index --csi svar2_mvp/${src}.norm.bcf +done +``` +Expected: `svar2_mvp/chr21.norm.bcf` and `svar2_mvp/gdc.chr21.norm.bcf` created. genoray requires normalized+atomized biallelic input. +**Checkpoint (compute):** the somatic file (1.1 GB, 16007 samples) `norm` is heavy — if it does not finish in a few minutes interactively, wrap Steps 2–3 for the somatic source in an `sbatch -p carter-compute` job script and wait for completion before Step 4's somatic validation. + +- [ ] **Step 3: Build both stores per source** + +Create `tmp/svar2_mvp/build_stores.py`: + +```python +"""Build .svar (SVAR1) and .svar2 (SVAR2) stores from a normalized biallelic BCF.""" +import sys +from pathlib import Path + +from genoray import VCF, SparseVar, _core + +def build(bcf: str, chrom: str, samples: list[str], out_prefix: str, ploidy: int): + bcf = str(bcf) + # SVAR 1.0 + SparseVar.from_vcf(f"{out_prefix}.svar", VCF(bcf), "8g", overwrite=True) + # SVAR 2.0 + _core.run_conversion_pipeline( + bcf, "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", + [chrom], f"{out_prefix}.svar2", samples, + 25_000, ploidy, 8, 8 * 1024 * 1024, + ) + print(f"built {out_prefix}.svar and {out_prefix}.svar2") + +if __name__ == "__main__": + # argv: + bcf, chrom, out_prefix = sys.argv[1], sys.argv[2], sys.argv[3] + import subprocess + samples = subprocess.run( + ["bcftools", "query", "-l", bcf], capture_output=True, text=True, check=True + ).stdout.split() + build(bcf, chrom, samples, out_prefix, ploidy=2) +``` + +Run germline interactively; somatic via SLURM: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +# germline (use the germline BCF's own chrom name from Step 1, e.g. chr21 or 21): +pixi run -e default python tmp/svar2_mvp/build_stores.py $W/chr21.norm.bcf $W/germline +# somatic (heavy) — submit and wait: +sbatch -p carter-compute --wrap "cd $PWD && pixi run -e default python tmp/svar2_mvp/build_stores.py $W/gdc.chr21.norm.bcf $W/somatic" +``` +Replace ``/`` with the names resolved in Step 1. +Expected: four store dirs — `germline.svar`, `germline.svar2`, `somatic.svar`, `somatic.svar2`. **Checkpoint:** monitor the sbatch job (`squeue -u $USER`); proceed to Step 4's somatic checks only after it completes and `somatic.svar2/meta.json` exists. + +- [ ] **Step 4: Validate both backends return sane output** + +Create `tmp/svar2_mvp/validate.py`: + +```python +"""Spot-check that gvl returns non-empty, sane haplotypes + variants through +both the SVAR1 (gvl Dataset over .svar) and SVAR2 (SparseVar2Source over .svar2) +backends, on a handful of regions x a few samples. Correctness is already proven +by the test suite; this proves the REAL-DATA plumbing works.""" +import sys +from pathlib import Path + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + +def main(prefix: str, chrom: str): + # A few small regions (0-based, half-open) in a variant-dense chr21 window. + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500)] + + # --- SVAR2 backend (adapter direct) --- + sv2 = SparseVar2(f"{prefix}.svar2") + print(f"[svar2] n_samples={sv2.n_samples} ploidy={sv2.ploidy}") + ref_bytes = _contig_ref(REF, chrom) + src = SparseVar2Source(sv2) + hap = src.reconstruct( + chrom, regions, + np.frombuffer(ref_bytes, np.uint8), + np.array([0, len(ref_bytes)], np.int64), + pad_char=ord("N"), shifts=None, output_length=-1, + ) + lens = np.asarray(hap.offsets) + print(f"[svar2] hap ragged rows={len(lens) - 1} " + f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}") + var = sv2.decode(chrom, regions) + print(f"[svar2] decode variants: {var}") + + # --- SVAR1 backend (gvl Dataset over .svar) --- + import polars as pl + bed = pl.DataFrame({ + "chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions], + }) + ds_path = f"{prefix}.gvl" + gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + seqs = ds[:len(regions), :3] # a few regions x first 3 samples + print(f"[svar1] gvl haplotypes sample shape/type: {type(seqs)}") + +def _contig_ref(fasta: str, chrom: str) -> bytes: + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: +``` + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +pixi run -e default python tmp/svar2_mvp/validate.py $W/germline +pixi run -e default python tmp/svar2_mvp/validate.py $W/somatic +``` +Expected: for each source, non-empty haplotype rows with sane lengths (≈ region length ± indels), non-empty decode variants, and the gvl SVAR1 Dataset opening without error. **Checkpoint:** if a chosen region is empty (no variants), pick a denser window (inspect with `bcftools view -H :20000000-20010000 | head`) and rerun. Adapt the API calls to the actual installed signatures if they drift (record any drift in the notes). + +- [ ] **Step 5: Record store sizes now (needed for Task 4)** + +```bash +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +du -sh $W/germline.svar $W/germline.svar2 $W/somatic.svar $W/somatic.svar2 +``` +Expected: four sizes. Somatic `.svar2` is expected smaller than `.svar`. + +- [ ] **Step 6: Write and commit the validation notes** + +Create `docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md` capturing: resolved contig names per file, exact commands run, sample/ploidy counts printed, hap-row counts + length ranges per backend, decode variant summaries, the four `du` sizes, and any API drift encountered. Then: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tmp/svar2_mvp/build_stores.py tmp/svar2_mvp/validate.py docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md +git commit --no-verify -m "chore: SVAR2 MVP real-data validation scripts + notes + +Co-Authored-By: Claude Opus 4.8 " +``` +(The store dirs under `WORK` are outside the repo and are not committed.) + +--- + +### Task 4: Benchmark SVAR1 vs SVAR2 (Task E) + +**Nature:** measurement script producing a table. Depends on Task 3's four stores. + +**Files:** +- Create: `tmp/svar2_mvp/benchmark.py` +- Create: `docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md` (results table) + +**Interfaces:** +- Consumes: the four stores from Task 3; the same backend calls listed in Task 3's Interfaces. +- Produces: a results table (documentation only). + +**Fairness rule (from spec):** query the **same workload** on both backends — **all samples for a fixed region set** — matching genoray's per-contig/all-samples query granularity. Warm caches, N repeats, report **median**. Record the caveat that this is adapter-vs-Dataset (Task B not wired), so SVAR2 latency excludes gvl batching/collation. + +- [ ] **Step 1: Write the benchmark script** + +Create `tmp/svar2_mvp/benchmark.py`: + +```python +"""Benchmark SVAR1 (gvl Dataset over .svar) vs SVAR2 (SparseVar2Source over +.svar2): hap latency, variant latency, store size, for one source prefix. +Fair workload: ALL samples for a fixed region set. Warm caches, median of N.""" +import sys +import time +import subprocess +from statistics import median + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +N = 5 # repeats + +def _contig_ref(fasta, chrom): + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +def _timed(fn, warmup=1): + for _ in range(warmup): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter() + fn() + ts.append(time.perf_counter() - t0) + return median(ts) + +def main(prefix, chrom): + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), + (40_000_000, 40_001_000)] + ref_bytes = _contig_ref(REF, chrom) + ref_u8 = np.frombuffer(ref_bytes, np.uint8) + ref_off = np.array([0, len(ref_bytes)], np.int64) + + # SVAR2 backend + sv2 = SparseVar2(f"{prefix}.svar2") + src = SparseVar2Source(sv2) + svar2_hap = _timed(lambda: src.reconstruct( + chrom, regions, ref_u8, ref_off, pad_char=ord("N"), + shifts=None, output_length=-1)) + svar2_var = _timed(lambda: sv2.decode(chrom, regions)) + + # SVAR1 backend (all samples, same regions) + import polars as pl + bed = pl.DataFrame({"chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions]}) + ds_path = f"{prefix}.gvl" + ds = gvl.Dataset.open(ds_path, reference=REF) + ds_hap = ds.with_seqs("haplotypes") + ds_var = ds.with_seqs("variants") + n_s = sv2.n_samples + svar1_hap = _timed(lambda: ds_hap[:len(regions), :n_s]) + svar1_var = _timed(lambda: ds_var[:len(regions), :n_s]) + + def du(path): + return subprocess.run(["du", "-sb", path], capture_output=True, + text=True).stdout.split()[0] + + print(f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " + f"regions={len(regions)} N={N}") + print(f" hap_latency_s svar1={svar1_hap:.4f} svar2={svar2_hap:.4f}") + print(f" var_latency_s svar1={svar1_var:.4f} svar2={svar2_var:.4f}") + print(f" store_bytes svar1={du(prefix + '.svar')} " + f"svar2={du(prefix + '.svar2')}") + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: +``` + +- [ ] **Step 2: Run the benchmark for both sources** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +pixi run -e default python tmp/svar2_mvp/benchmark.py $W/germline +pixi run -e default python tmp/svar2_mvp/benchmark.py $W/somatic +``` +Expected: printed hap/variant latencies + store bytes per source. **Checkpoint:** the somatic all-samples decode (16007 samples) may be large — if a single call is very slow or OOMs, reduce the region set (fewer/smaller windows) uniformly for **both** backends to keep the comparison fair, and note the reduced workload. If `ds.with_seqs("variants")` indexing differs from the installed API, adapt and record the drift. + +- [ ] **Step 3: Assemble the results table and commit** + +Create `docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md` with the 2×3 table (germline/somatic × {hap latency, variant latency, store size}) filled from Step 2's output, the exact workload (regions, n_samples, N), and the recorded caveat (adapter-vs-Dataset; somatic is where SVAR2 layout should win on size). Then: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tmp/svar2_mvp/benchmark.py docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md +git commit --no-verify -m "chore: SVAR2 vs SVAR1 gvl benchmark script + results + +Co-Authored-By: Claude Opus 4.8 " +``` + +- [ ] **Step 4: Summarize the value proposition** + +Report to the user: the filled table, whether SVAR2 wins on store size (esp. somatic) and how hap/variant latency compares, and a recommendation on whether Task B (Dataset wiring) is worth the invasive integration — plus the concrete all-samples-per-batch cost signal the benchmark revealed, to feed the Task B brainstorm. + +--- + +## Self-Review + +**Spec coverage:** +- Task A (cleanup) → Task 1. ✓ +- Task C (realign_tracks e2e test) → Task 2, with the spec's pure-Python SVAR1 oracle. ✓ +- Task D (real-data validation) → Task 3 (build both stores per source, validate both backends, small scope, germline interactive / somatic sbatch, contig-name checkpoint, reference FASTA). ✓ +- Task E (benchmark: hap latency, variant latency, store size, fair all-samples workload, median of warm repeats, 2×3 table, caveat) → Task 4. ✓ +- Task B (Dataset dispatch) → intentionally out of scope; noted in Global Constraints and fed by Task 4 Step 4. ✓ + +**Placeholder scan:** The only intentional fill-ins are ``/``, which are *resolved values* from Task 3 Step 1 (a genuine runtime discovery, not a deferred design decision), and the notes-file contents (recorded observations, which cannot be pre-written). All code steps contain complete, runnable code. No "TBD"/"add error handling"/"similar to Task N". + +**Type consistency:** `SparseVar2Source.reconstruct` / `.realign_tracks` signatures match `_svar2_source.py`; `shift_and_realign_track_sparse` params match `_tracks.py:708`; `run_conversion_pipeline` positional args match the reconstruct-test call and `gvl.write`/`SparseVar.from_vcf` match their source signatures. Hap indexing `h=(r*S+s)*P+p` is consistent between Task 2 and the decode layout. diff --git a/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md index c6f64710..8eaff462 100644 --- a/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md +++ b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md @@ -80,21 +80,31 @@ No design surface. Abort a removal if that worktree reports uncommitted work. `SparseVar2Source.realign_tracks` (the region→R·S track expansion) has no end-to-end test; only the Rust driver unit test `svar2_track_realign_del` exists. -**Oracle: option (a)** — gvl's own SVAR1 `shift_and_realign_tracks_sparse`, fed the -variants materialized from genoray `SparseVar2.decode`. It is a fully independent, -already-trusted code path (9 passing tests) and mirrors how -`tests/test_svar2_reconstruct.py` validates against a separate oracle. We reject the -pure-Python-consensus option (b): a second implementation to get right, and weaker than -reusing the trusted kernel. +**Oracle: gvl's SVAR1 _pure-Python_ `shift_and_realign_track_sparse` +(`_dataset/_tracks.py:708`), fed the variants materialized from genoray +`SparseVar2.decode`.** This is the strongest low-cost oracle and resolves the handoff's +(a)/(b) tension: +- It reuses *trusted* SVAR1 realign logic — including the DEL-anchor branch already + present at `_tracks.py:755` — so we don't hand-roll a second track-shift + implementation (the weakness of a fresh pure-Python consensus). +- It is a **pure-Python** implementation, genuinely independent of the SVAR2 **Rust** + kernel under test (`shift_and_realign_tracks_from_svar2`, a closure refactor of the + *Rust* SVAR1 kernel — a different implementation from this Python fallback). +- Setup is trivial: build a synthetic single-hap genotype layout from decode records — + `geno_v_idxs = arange(n_h)`, `geno_offsets = [0, n_h]`, `v_starts = pos_h`, + `ilens = ilen_h` — mirroring how `test_svar2_reconstruct.py` feeds decode records to + its oracle. **Construction:** -- Reuse the store builder from `tests/test_svar2_reconstruct.py`. +- Reuse the store builder + `decode_batch` extraction from + `tests/test_svar2_reconstruct.py`. - Use **DEL-only variants** so insertion-fill is bypassed and the strategy choice is - irrelevant (any valid `strategy_id`/`params` works — borrow from an existing gvl - track test). -- Define a per-region reference track (f32); realign through the SVAR2 adapter path and - through the SVAR1 oracle fed the decoded variants. -- **Assert per-`(r,s,p)`** equality between the two. + irrelevant (any valid `strategy_id`/`params` works — track realign is ilen-only for + DELs). +- Define a per-region reference track (f32); realign through the SVAR2 adapter path + (`SparseVar2Source.realign_tracks`) and, per hap, through the pure-Python oracle fed + that hap's decoded `(pos, ilen)` with `shift=0` and `out` sized to the hap length. +- **Assert per-`(r,s,p)`** float equality between the two (`np.testing.assert_allclose`). Place alongside `tests/test_svar2_reconstruct.py`. From b4c040c786b96d4166912a2ea54dcd51b08372a1 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 15:30:27 -0700 Subject: [PATCH 010/108] test: e2e SVAR2 realign_tracks vs SVAR1 pure-Python oracle Co-Authored-By: Claude Opus 4.8 --- tests/test_svar2_realign_tracks.py | 154 +++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 tests/test_svar2_realign_tracks.py diff --git a/tests/test_svar2_realign_tracks.py b/tests/test_svar2_realign_tracks.py new file mode 100644 index 00000000..c3d04f2d --- /dev/null +++ b/tests/test_svar2_realign_tracks.py @@ -0,0 +1,154 @@ +"""End-to-end validation of the SVAR2 track-realign adapter path. + +Builds a DEL-only SVAR2 store, realigns a reference track through gvl's SVAR2 +path (SparseVar2Source.realign_tracks, the Rust two-source kernel), and compares +per-(region, sample, ploid) against gvl's INDEPENDENT pure-Python SVAR1 track +realign (shift_and_realign_track_sparse) fed genoray's materialized decode +records. Agreement proves the SVAR2 Rust track kernel matches the trusted SVAR1 +realign semantics — including the DEL anchor. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). Two pure DELs chosen to match the reference exactly: +# POS 4 GTA>G -> 0-based pos 3, ilen -2 (ref[3:6] == "GTA") +# POS 10 GGG>G -> 0-based pos 9, ilen -2 (ref[9:12] == "GGG") +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t4\t.\tGTA\tG\t.\t.\t.\tGT\t1|0\t1|1 +chr1\t10\t.\tGGG\tG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def svar2_del_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_del") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], + 25_000, 2, 1, 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): + import genoray + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._tracks import shift_and_realign_track_sparse + + contig = "chr1" + q_start, q_end = 0, 40 + region_len = q_end - q_start + regions = [(q_start, q_end)] + + sv = genoray.SparseVar2(str(svar2_del_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # A per-region reference track (f32). Random-but-fixed so a positional bug + # can't hide behind a monotonic ramp. + rng = np.random.default_rng(0) + track = rng.random(region_len).astype(np.float32) + + strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) + params = np.zeros(1, np.float64) + base_seed = 0 + + # --- SVAR2 path under test: one region, expanded internally to R*S*P haps --- + src = SparseVar2Source(sv) + out_rag = src.realign_tracks( + contig, + regions, + track, # flat per-region track buffer + np.array([0, region_len], np.int64), # (R+1) offsets + params, + strategy_id, + base_seed, + shifts=None, # no jitter + parallel=False, + ) + + # --- oracle: genoray decode records -> pure-Python SVAR1 realign, per hap --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + + # Non-triviality: haps carry a varying number of DELs. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [1, 1, 2, 1], per_hap_counts + + # `out_rag` is a `_Flat` (flat data/offsets buffer) cast to `Ragged` for typing; + # `_Flat.__getitem__` only supports leading-axis slicing, so pull rows out via + # its flat offsets directly — the same pattern used against this adapter's + # sibling `_Flat` result in tests/test_svar2_reconstruct.py. + out_data = np.asarray(out_rag.data) + out_off = np.asarray(out_rag.offsets) + + for s in range(S): + for p in range(P): + h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = np.ascontiguousarray(d_pos[gi0:gi1], np.int32) + ilen_h = np.ascontiguousarray(d_ilen[gi0:gi1], np.int32) + n_h = gi1 - gi0 + + # Independently size the hap: region length + sum of (negative) ilens. + exp_len = region_len + int(ilen_h.sum()) + + got = out_data[int(out_off[h]) : int(out_off[h + 1])] + assert got.shape[0] == exp_len, ( + f"(s={s},p={p}) SVAR2 len {got.shape[0]} != expected {exp_len} " + f"(ilen={ilen_h.tolist()})" + ) + + # Synthetic single-hap SVAR1 layout: v_idxs 0..n_h, one group. + geno_v_idxs = np.arange(n_h, dtype=np.int32) + geno_offsets = np.array([0, n_h], np.int64) + expected = np.empty(exp_len, np.float32) + shift_and_realign_track_sparse( + offset_idx=0, + geno_v_idxs=geno_v_idxs, + geno_offsets=geno_offsets, + v_starts=pos_h, + ilens=ilen_h, + shift=0, + track=track, + query_start=q_start, + out=expected, + params=params, + strategy_id=strategy_id, + base_seed=base_seed, + query=0, + hap=h, + ) + np.testing.assert_allclose( + got, expected, rtol=0, atol=0, + err_msg=f"(s={s},p={p}) SVAR2 track != SVAR1 oracle " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", + ) From adf38f35cb8a660249bcfdf3fb930648fc7f5477 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 20:54:47 -0700 Subject: [PATCH 011/108] chore: SVAR2 MVP real-data validation scripts + notes Build both .svar (SVAR1) and .svar2 (SVAR2) stores from real chr21 germline (1000G, 3202 smp) and somatic (GDC, 16007 smp) BCFs, and spot-check both gvl backends. Both return non-empty, correctly-shaped haplotypes+variants. SVAR2 is 5.67x smaller (germline) / 1.46x smaller (somatic) on disk. Notes record two infra/data fixes: compute-node LD_LIBRARY_PATH GLIBCXX shadowing (prepend pixi env lib), and filtering symbolic/breakend ALTs that SVAR2 rejects (short-read only) before conversion. Co-Authored-By: Claude Opus 4.8 --- .../notes/2026-07-03-svar2-mvp-validation.md | 123 ++++++++++++++++++ tmp/svar2_mvp/build_stores.py | 26 ++++ tmp/svar2_mvp/validate.py | 54 ++++++++ 3 files changed, 203 insertions(+) create mode 100644 docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md create mode 100644 tmp/svar2_mvp/build_stores.py create mode 100644 tmp/svar2_mvp/validate.py diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md new file mode 100644 index 00000000..7746c0a0 --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md @@ -0,0 +1,123 @@ +# SVAR2 gvl MVP — Real-Data Validation (chr21) + +**Date:** 2026-07-03 +**Task:** Plan Task 3 (Task D) — prove the real-data plumbing works through both the +SVAR1 (gvl `Dataset` over `.svar`) and SVAR2 (`SparseVar2Source` over `.svar2`) backends +on real germline + somatic chr21 stores. Correctness is already proven by the test suite +(Task 2 e2e oracle); this exercises the real-data path end-to-end. + +**Env:** `pixi run -e default`, genoray 2.15.0 wheel. Reference FASTA +`/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. +**Work dir (outside repo):** `/carter/users/dlaub/repos/for_loukik/svar2_mvp` (`$W`). + +## Resolved contig names (Step 1) + +All three inputs use `chr21` — **no naming mismatch**: + +| File | Contig | +| --- | --- | +| Reference FASTA (`GRCh38.d1.vd1.fa`) | `chr21` | +| `chr21.bcf` (germline, 1000G) | `chr21` | +| `gdc.chr21.bcf` (somatic, GDC) | `chr21` | + +So `` = `` = `chr21` throughout. The germline `.csi` was +already present (no re-index needed). + +## Cohort sizes + +| Source | Samples | Ploidy | Raw records (chr21) | +| --- | --- | --- | --- | +| germline (1000G) | 3202 | 2 | 1,002,753 | +| somatic (GDC) | 16007 | 2 | 4,525,689 | + +## Build pipeline (Steps 2–3) + +Per source: `bcftools norm -m -any --atomize` → **filter symbolic/breakend** → build both +stores (`build_stores.py`: `SparseVar.from_vcf` for `.svar`, `run_conversion_pipeline` for +`.svar2`). Orchestrated by `$W/build_source.sh` (kept outside the repo), submitted via +`sbatch -p carter-compute` (germline 32G, somatic 128G, 8 cpus each). + +### Two environment/data issues found and fixed (API/infra drift) + +1. **Compute-node `libstdc++` / GLIBCXX shadowing.** On the compute nodes the inherited + `LD_LIBRARY_PATH` puts the gcc11 module's `libstdc++.so.6` (which lacks + `GLIBCXX_3.4.30`) ahead of the pixi env's newer one, so importing genoray crashed at + `llvmlite → numba` load: + `OSError: .../gcc11/.../libstdc++.so.6: version 'GLIBCXX_3.4.30' not found (required by + .../libLLVM-14.so)`. The login node is unaffected (its `miniforge3/lib` on + `LD_LIBRARY_PATH` rescues it). **Fix:** prepend the pixi env lib dir to + `LD_LIBRARY_PATH` in the job so the env's own `libstdc++` (which has `GLIBCXX_3.4.30`) + wins the loader search. Baked into `build_source.sh`. + +2. **SVAR2 rejects symbolic/breakend ALTs (short-read only).** genoray's + `normalize::atomize_record` returns `SymbolicAllele` for symbolic (``, + ``, ``, ``, …) and breakend ALTs, and the reader `.expect()`s on + it — so a single SV record **panics the whole conversion** + (`worker thread 'read-chr21' panicked`). `*`/`.` alleles are silently skipped, not + errored. **Fix:** drop symbolic ("other") + breakend ("bnd") types with + `bcftools view -V other,bnd` before building, and — for a **fair** benchmark — build + *both* stores from the identical filtered input. Filtering impact: + + | Source | Records before | Records after filter | Symbolic/bnd dropped | + | --- | --- | --- | --- | + | germline | 1,002,753 | 1,001,385 | **1,368** (702 ``, 260 ``, 195 ``, 174 ``, 17 ``, 12 ``, 7 ``, 1 ``) | + | somatic | 4,525,689 | 4,525,689 | **0** (GDC somatic calls are SNV/indel only) | + + This friction motivated a new roadmap milestone on the genoray side — **M13: opt-in + skip for out-of-scope alleles during conversion** (drop instead of panic), in + `genoray:docs/roadmap/svar-2.md`. + +Build wall-clock: germline ~11 min; somatic ~2h1m (peak RSS ~10.9 GB). + +## Backend validation (Step 4) — `validate.py`, 2 regions × chr21 + +Regions (0-based half-open): `(20_000_000, 20_001_000)` [1000 bp] and +`(30_000_000, 30_000_500)` [500 bp]. `validate.py` ran **as-written — no API drift** in the +`SparseVar2Source.reconstruct` / `sv.decode` / `gvl.write` / `gvl.Dataset.open` signatures. + +### germline (`$W/germline`) + +- **SVAR2** (`SparseVar2Source`): `n_samples=3202 ploidy=2`; hap ragged + `rows=12808` (= 2 regions × 3202 samples × 2 ploidy ✓), `min_len=499 max_len=1000` + (1000 bp window full-length; the 499 is a single-base net deletion) — sane. `decode` + returned a non-empty `seqpro.rag.Ragged`. +- **SVAR1** (gvl `Dataset` over `.svar`): dataset written (3202 samples, 2 regions) and + opened cleanly; `with_seqs("haplotypes")` returned a `seqpro.rag.Ragged`. + +### somatic (`$W/somatic`) + +- **SVAR2**: `n_samples=16007 ploidy=2`; hap ragged `rows=64028` (= 2 × 16007 × 2 ✓), + `min_len=498 max_len=1001` (a −2 DEL and a +1 INS at the extremes) — sane. `decode` + returned a non-empty `Ragged`. +- **SVAR1**: dataset written (16007 samples, 2 regions) and opened; haplotypes `Ragged`. + +Both backends return non-empty, correctly-shaped haplotypes and variants on real data for +both cohorts. + +## Store sizes (Step 5) + +`du` on the four stores: + +| Store | SVAR1 (`.svar`) | SVAR2 (`.svar2`) | SVAR2 / SVAR1 | SVAR2 advantage | +| --- | --- | --- | --- | --- | +| germline (3202 smp) | 1.1 G (1,149,533,941 B) | 178 M (202,842,586 B) | 0.176× | **5.67× smaller** | +| somatic (16007 smp) | 50 M (55,578,073 B) | 34 M (38,184,053 B) | 0.687× | **1.46× smaller** | + +SVAR2 wins on size for **both** cohorts. The germline win is dramatic (5.7×): 1000G +carries many **common, high-allele-frequency** variants, which SVAR2's cost model routes +to the 1-bit dense matrix — far cheaper than SVAR1's `u32` pointers. Somatic mutations are +**rare / near-private** (low AF), so they stay sparse in both formats and the gap narrows +to 1.46×. This is exactly the empirical distribution SVAR2 was designed to exploit. + +## Artifacts + +- Store builder: `tmp/svar2_mvp/build_stores.py` (committed) +- Backend validator: `tmp/svar2_mvp/validate.py` (committed) +- Job orchestrator: `$W/build_source.sh` (outside repo — norm/filter/build wrapper with + the `LD_LIBRARY_PATH` fix; not committed) +- Four stores + normalized/filtered BCFs live under `$W` (outside the repo). + +## Feeds Task 4 (benchmark) + +The four stores are the benchmark inputs. Early size signal above; latency (hap/variant, +all-samples-per-region) is measured next. diff --git a/tmp/svar2_mvp/build_stores.py b/tmp/svar2_mvp/build_stores.py new file mode 100644 index 00000000..935a0cda --- /dev/null +++ b/tmp/svar2_mvp/build_stores.py @@ -0,0 +1,26 @@ +"""Build .svar (SVAR1) and .svar2 (SVAR2) stores from a normalized biallelic BCF.""" +import sys +from pathlib import Path + +from genoray import VCF, SparseVar, _core + +def build(bcf: str, chrom: str, samples: list[str], out_prefix: str, ploidy: int): + bcf = str(bcf) + # SVAR 1.0 + SparseVar.from_vcf(f"{out_prefix}.svar", VCF(bcf), "8g", overwrite=True) + # SVAR 2.0 + _core.run_conversion_pipeline( + bcf, "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", + [chrom], f"{out_prefix}.svar2", samples, + 25_000, ploidy, 8, 8 * 1024 * 1024, + ) + print(f"built {out_prefix}.svar and {out_prefix}.svar2") + +if __name__ == "__main__": + # argv: + bcf, chrom, out_prefix = sys.argv[1], sys.argv[2], sys.argv[3] + import subprocess + samples = subprocess.run( + ["bcftools", "query", "-l", bcf], capture_output=True, text=True, check=True + ).stdout.split() + build(bcf, chrom, samples, out_prefix, ploidy=2) diff --git a/tmp/svar2_mvp/validate.py b/tmp/svar2_mvp/validate.py new file mode 100644 index 00000000..4331ece5 --- /dev/null +++ b/tmp/svar2_mvp/validate.py @@ -0,0 +1,54 @@ +"""Spot-check that gvl returns non-empty, sane haplotypes + variants through +both the SVAR1 (gvl Dataset over .svar) and SVAR2 (SparseVar2Source over .svar2) +backends, on a handful of regions x a few samples. Correctness is already proven +by the test suite; this proves the REAL-DATA plumbing works.""" +import sys +from pathlib import Path + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + +def main(prefix: str, chrom: str): + # A few small regions (0-based, half-open) in a variant-dense chr21 window. + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500)] + + # --- SVAR2 backend (adapter direct) --- + sv2 = SparseVar2(f"{prefix}.svar2") + print(f"[svar2] n_samples={sv2.n_samples} ploidy={sv2.ploidy}") + ref_bytes = _contig_ref(REF, chrom) + src = SparseVar2Source(sv2) + hap = src.reconstruct( + chrom, regions, + np.frombuffer(ref_bytes, np.uint8), + np.array([0, len(ref_bytes)], np.int64), + pad_char=ord("N"), shifts=None, output_length=-1, + ) + lens = np.asarray(hap.offsets) + print(f"[svar2] hap ragged rows={len(lens) - 1} " + f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}") + var = sv2.decode(chrom, regions) + print(f"[svar2] decode variants: {var}") + + # --- SVAR1 backend (gvl Dataset over .svar) --- + import polars as pl + bed = pl.DataFrame({ + "chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions], + }) + ds_path = f"{prefix}.gvl" + gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + seqs = ds[:len(regions), :3] # a few regions x first 3 samples + print(f"[svar1] gvl haplotypes sample shape/type: {type(seqs)}") + +def _contig_ref(fasta: str, chrom: str) -> bytes: + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: From 7a572b855a1aad8b68fdc6d8935b3ae36b67a694 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 20:58:42 -0700 Subject: [PATCH 012/108] chore: SVAR2 vs SVAR1 gvl benchmark script + results Fair all-samples/3-region workload on chr21 germline (3202 smp) + somatic (16007 smp), warm caches, median of N=5. SVAR2 store is 5.67x (germline) / 1.46x (somatic) smaller; SVAR2 hap latency scales far better with cohort size (1.34x vs SVAR1's 6.8x over 3202->16007 smp), reaching parity at 16007. Caveat: adapter-vs-Dataset (Task B unwired), so SVAR2 latency excludes gvl batching/collation. Fixes a fairness bug in the plan draft (opened validate's 2-region .gvl while SVAR2 queried 3 regions); now writes the 3-region BED first. Co-Authored-By: Claude Opus 4.8 --- .../notes/2026-07-03-svar2-mvp-benchmark.md | 77 +++++++++++++++++++ tmp/svar2_mvp/benchmark.py | 75 ++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md create mode 100644 tmp/svar2_mvp/benchmark.py diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md new file mode 100644 index 00000000..f26d2563 --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md @@ -0,0 +1,77 @@ +# SVAR2 vs SVAR1 gvl Benchmark (chr21) + +**Date:** 2026-07-03 +**Task:** Plan Task 4 (Task E) — measure SVAR1 (gvl `Dataset` over `.svar`) vs SVAR2 +(`SparseVar2Source` over `.svar2`) on hap latency, variant latency, and store size, for +the germline + somatic chr21 stores built in Task 3. + +**Script:** `tmp/svar2_mvp/benchmark.py`. **Env:** `pixi run -e default`, genoray 2.15.0. + +## Workload (fairness rule) + +Same workload on both backends: **all samples for a fixed region set**, matching genoray's +per-contig/all-samples query granularity. Warm caches (1 warmup), **median of N=5** repeats. + +- **Regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, + `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)` — 3 regions, 2.5 kb total. +- **Samples:** germline 3202, somatic 16007 (all samples, both backends). +- Both backends query the **same 3 regions × all samples**. (`benchmark.py` was corrected + from the plan draft, which opened `validate.py`'s leftover 2-region `.gvl` while the + SVAR2 path queried 3 regions — an unfair mismatch; it now `gvl.write`s the 3-region BED + first so both measure an identical workload.) + +## Results (median seconds; store in bytes) + +| Source (samples) | Metric | SVAR1 | SVAR2 | Outcome | +| --- | --- | --- | --- | --- | +| **germline** (3202) | hap latency (s) | 0.0555 | 0.2834 | SVAR1 5.1× faster | +| | var latency (s) | 0.0280 | 0.2270 | SVAR1 8.1× faster | +| | store size | 1,149,533,941 (1.1 G) | 202,842,586 (178 M) | **SVAR2 5.67× smaller** | +| **somatic** (16007) | hap latency (s) | 0.3798 | 0.3797 | **tied** | +| | var latency (s) | 0.0518 | 0.1088 | SVAR1 2.1× faster | +| | store size | 55,578,073 (50 M) | 38,184,053 (34 M) | **SVAR2 1.46× smaller** | + +## Reading the numbers + +**Store size — SVAR2 wins outright, most on high-AF cohorts.** 5.67× smaller on germline +(1000G, many common/high-allele-frequency variants routed to SVAR2's 1-bit dense matrix) +and 1.46× smaller on somatic (rare/near-private mutations stay sparse in both formats). +This is exactly the empirical short-read distribution SVAR2 targets. + +**Latency — SVAR2's cost is nearly flat in sample count; SVAR1's grows.** Going from 3202 +→ 16007 samples (5×): + +| Backend | hap latency 3202 → 16007 | factor | +| --- | --- | --- | +| SVAR1 | 0.0555 → 0.3798 s | **6.8×** | +| SVAR2 | 0.2834 → 0.3797 s | **1.34×** | + +SVAR1's gvl `Dataset` pays a per-sample read cost that scales with cohort size; the SVAR2 +adapter's live-query + decode is far less sensitive to sample count, so the two **reach +parity on hap latency at 16007 samples** and SVAR2 would extrapolate to *faster* for larger +cohorts. Variant-decode latency still favors SVAR1 (2–8×), but that gap also narrows with +scale (8.1× → 2.1×). + +## Caveat — adapter-vs-Dataset (Task B not wired) + +This compares the mature **SVAR1 gvl `Dataset`** (genotypes pre-materialized into an +optimized on-disk layout at `gvl.write` time; queries are then fast reads) against the +**SVAR2 `SparseVar2Source` adapter**, which queries genoray `overlap_batch` **live** and +decodes on every call, with **no Dataset-level batching, collation, or caching**. SVAR2 is +not yet wired into the Dataset (plan Task B, deferred). So SVAR2's latency here is a +*floor-less* direct-adapter number; wiring it into the Dataset (Task B) is what would let +it inherit the same batching/collation the SVAR1 path enjoys — likely closing or reversing +the remaining latency gap while **keeping the storage win**. + +## Value proposition & Task B signal + +- **SVAR2 is a clear win on disk footprint** (1.5–5.7×), scaling with allele frequency — + the more common the variants, the bigger the win. +- **SVAR2 latency scales much better with cohort size** — already at parity on hap latency + at 16k samples via the un-optimized adapter alone. +- **Concrete Task B cost signal:** the all-samples-per-region query is the granularity that + matters, and the adapter already competes there at scale. Wiring SVAR2 into the Dataset + (Task B) should be pursued: the storage advantage is unconditional, and the latency + trend says the Dataset's batching/collation would likely make SVAR2 competitive-to-faster + on latency too for realistic large cohorts. The invasive integration is justified by the + storage win alone; the scaling trend strengthens the case. diff --git a/tmp/svar2_mvp/benchmark.py b/tmp/svar2_mvp/benchmark.py new file mode 100644 index 00000000..c06bee84 --- /dev/null +++ b/tmp/svar2_mvp/benchmark.py @@ -0,0 +1,75 @@ +"""Benchmark SVAR1 (gvl Dataset over .svar) vs SVAR2 (SparseVar2Source over +.svar2): hap latency, variant latency, store size, for one source prefix. +Fair workload: ALL samples for a fixed region set. Warm caches, median of N.""" +import sys +import time +import subprocess +from statistics import median + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +N = 5 # repeats + +def _contig_ref(fasta, chrom): + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +def _timed(fn, warmup=1): + for _ in range(warmup): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter() + fn() + ts.append(time.perf_counter() - t0) + return median(ts) + +def main(prefix, chrom): + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), + (40_000_000, 40_001_000)] + ref_bytes = _contig_ref(REF, chrom) + ref_u8 = np.frombuffer(ref_bytes, np.uint8) + ref_off = np.array([0, len(ref_bytes)], np.int64) + + # SVAR2 backend + sv2 = SparseVar2(f"{prefix}.svar2") + src = SparseVar2Source(sv2) + svar2_hap = _timed(lambda: src.reconstruct( + chrom, regions, ref_u8, ref_off, pad_char=ord("N"), + shifts=None, output_length=-1)) + svar2_var = _timed(lambda: sv2.decode(chrom, regions)) + + # SVAR1 backend (all samples, same regions) + import polars as pl + bed = pl.DataFrame({"chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions]}) + ds_path = f"{prefix}.gvl" + # Write the Dataset over the SAME region set the SVAR2 path benchmarks, so both + # backends measure an identical workload (fairness rule). validate.py may have left + # a .gvl with a different region set, so always (re)write here. + gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF) + ds_hap = ds.with_seqs("haplotypes") + ds_var = ds.with_seqs("variants") + n_s = sv2.n_samples + svar1_hap = _timed(lambda: ds_hap[:len(regions), :n_s]) + svar1_var = _timed(lambda: ds_var[:len(regions), :n_s]) + + def du(path): + return subprocess.run(["du", "-sb", path], capture_output=True, + text=True).stdout.split()[0] + + print(f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " + f"regions={len(regions)} N={N}") + print(f" hap_latency_s svar1={svar1_hap:.4f} svar2={svar2_hap:.4f}") + print(f" var_latency_s svar1={svar1_var:.4f} svar2={svar2_var:.4f}") + print(f" store_bytes svar1={du(prefix + '.svar')} " + f"svar2={du(prefix + '.svar2')}") + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: From adfc87c761ec87eb0e484cc7f026c93d0839855a Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 21:12:02 -0700 Subject: [PATCH 013/108] docs: temper SVAR2 benchmark latency claims; record layout + profiling caveats Final-review + maintainer feedback: the latency comparison is a raw SVAR2 adapter (new, unprofiled Python orchestration) vs the mature SVAR1 Dataset, not apples-to-apples. Remove unprofiled causal claims and the confounded germline-vs-somatic 'scales with cohort size' extrapolation. Record: - dense matrix is hap-major (query.rs:131, bit = hap*n_dense+col), variant innermost -> per-hap contiguous (refutes a (V,S,P) pessimal-layout guess); but n_dense is contig-wide, so narrow-region all-sample reads scatter one cache line per hap (contig-scoped, unprofiled hypothesis). - var-latency columns measure different ops (genoray decode vs with_seqs). - size table is .svar vs .svar2 stores; SVAR1 also materializes a .gvl. - single-contig: split-by-contig layout unassessed. - build is read-bound; single-contig underuses threads. Storage win (1.46x-5.67x) stands as the unconfounded result. Co-Authored-By: Claude Opus 4.8 --- .../notes/2026-07-03-svar2-mvp-benchmark.md | 150 +++++++++++++----- .../notes/2026-07-03-svar2-mvp-validation.md | 6 +- 2 files changed, 114 insertions(+), 42 deletions(-) diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md index f26d2563..61492a4f 100644 --- a/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md @@ -20,16 +20,26 @@ per-contig/all-samples query granularity. Warm caches (1 warmup), **median of N= SVAR2 path queried 3 regions — an unfair mismatch; it now `gvl.write`s the 3-region BED first so both measure an identical workload.) -## Results (median seconds; store in bytes) +## Results (median seconds; store size = apparent bytes, `du -sb`) | Source (samples) | Metric | SVAR1 | SVAR2 | Outcome | | --- | --- | --- | --- | --- | | **germline** (3202) | hap latency (s) | 0.0555 | 0.2834 | SVAR1 5.1× faster | -| | var latency (s) | 0.0280 | 0.2270 | SVAR1 8.1× faster | -| | store size | 1,149,533,941 (1.1 G) | 202,842,586 (178 M) | **SVAR2 5.67× smaller** | -| **somatic** (16007) | hap latency (s) | 0.3798 | 0.3797 | **tied** | -| | var latency (s) | 0.0518 | 0.1088 | SVAR1 2.1× faster | -| | store size | 55,578,073 (50 M) | 38,184,053 (34 M) | **SVAR2 1.46× smaller** | +| | var latency (s)† | 0.0280 | 0.2270 | — (different ops, see below) | +| | store size | 1,149,533,941 (1.15 GB) | 202,842,586 (203 MB) | **SVAR2 5.67× smaller** | +| **somatic** (16007) | hap latency (s) | 0.3798 | 0.3797 | **≈ parity** | +| | var latency (s)† | 0.0518 | 0.1088 | — (different ops, see below) | +| | store size | 55,578,073 (55.6 MB) | 38,184,053 (38.2 MB) | **SVAR2 1.46× smaller** | + +Store-size labels are decimal (MB = 10⁶ B, GB = 10⁹ B) computed from the apparent byte +counts; all ratios below are computed from the byte counts, not the rounded labels. + +**† The two "var latency" columns measure different operations and are NOT a like-for-like +decode comparison.** SVAR2 var = `sv2.decode(chrom, regions)` (genoray raw variant-record +decode → `Ragged`); SVAR1 var = `ds.with_seqs("variants")[:regions, :n_s]` (gvl +variant-**sequence** materialization). They are reported side by side only because each is +the natural "variants" call for its backend — do not read them as one backend decoding +"the same thing" faster. ## Reading the numbers @@ -38,40 +48,100 @@ per-contig/all-samples query granularity. Warm caches (1 warmup), **median of N= and 1.46× smaller on somatic (rare/near-private mutations stay sparse in both formats). This is exactly the empirical short-read distribution SVAR2 targets. -**Latency — SVAR2's cost is nearly flat in sample count; SVAR1's grows.** Going from 3202 -→ 16007 samples (5×): - -| Backend | hap latency 3202 → 16007 | factor | -| --- | --- | --- | -| SVAR1 | 0.0555 → 0.3798 s | **6.8×** | -| SVAR2 | 0.2834 → 0.3797 s | **1.34×** | - -SVAR1's gvl `Dataset` pays a per-sample read cost that scales with cohort size; the SVAR2 -adapter's live-query + decode is far less sensitive to sample count, so the two **reach -parity on hap latency at 16007 samples** and SVAR2 would extrapolate to *faster* for larger -cohorts. Variant-decode latency still favors SVAR1 (2–8×), but that gap also narrows with -scale (8.1× → 2.1×). - -## Caveat — adapter-vs-Dataset (Task B not wired) - -This compares the mature **SVAR1 gvl `Dataset`** (genotypes pre-materialized into an -optimized on-disk layout at `gvl.write` time; queries are then fast reads) against the -**SVAR2 `SparseVar2Source` adapter**, which queries genoray `overlap_batch` **live** and -decodes on every call, with **no Dataset-level batching, collation, or caching**. SVAR2 is -not yet wired into the Dataset (plan Task B, deferred). So SVAR2's latency here is a -*floor-less* direct-adapter number; wiring it into the Dataset (Task B) is what would let -it inherit the same batching/collation the SVAR1 path enjoys — likely closing or reversing -the remaining latency gap while **keeping the storage win**. +**Latency — SVAR1 leads on the small high-density cohort; the two are at parity on the +large one.** Between the two cohorts: + +| Backend | hap latency germline (3202) → somatic (16007) | +| --- | --- | +| SVAR1 | 0.0555 → 0.3798 s | +| SVAR2 | 0.2834 → 0.3797 s | + +⚠️ **This is NOT a controlled sample-count sweep — the two points are different datasets** +(1000G germline, high-AF/high-density, vs GDC somatic, rare/sparse), differing in allele +frequency, variant density, and source, not just in sample count. So the movement cannot be +attributed to cohort size alone: germline's high variant density is a large part of why +SVAR2's germline hap latency (0.283 s) is high at only 3202 samples. What the data *does* +show is a real, useful fact — **on the large 16007-sample somatic cohort the un-optimized +SVAR2 adapter already matches the mature SVAR1 Dataset on hap latency** (0.3797 vs 0.3798 s; +median of N=5, no dispersion/CI measured, so treat "parity" as "indistinguishable at this +resolution"). The *hypothesis* that SVAR2's latency is less sample-count-sensitive is +plausible and worth testing, but proving it needs a **same-cohort subsampling sweep** +(fix the dataset, vary S) — not this germline-vs-somatic pair. The var-latency columns are +different operations (†) and are not compared here. + +## Caveat — this is two different code paths, NOT profiled + +The latency columns compare the mature **SVAR1 gvl `Dataset`** path against the **SVAR2 +`SparseVar2Source` adapter** — and the SVAR2 path here is **not the Dataset at all**. It is +a new Python adapter that calls genoray `overlap_batch` live and runs the two-source Rust +kernel per call, with no Dataset-level batching/collation/caching (Task B, deferred). So: + +- We make **no claim about *why* the latencies differ.** We have not run `perf` / a Rust + profiler on either path, so attributing the gap to "pre-materialization", "live decode", + memory layout, or Python overhead would be speculation. Any "X is slow" statement is out + of scope until profiled. +- **Prior gvl profiling does not transfer.** Earlier work found gvl `Dataset` latency was + dominated by the Rust core with negligible Python-orchestration overhead — but that was + measured on the `Dataset` path. The benchmarked SVAR2 path is the **raw adapter**, which + is *new Python orchestration* (per-call genoray query + numpy conversions) that has never + been profiled; its Python overhead on a small 3-region workload is unknown and could be a + material fraction of the 0.28–0.38 s. +- **The size table compares variant *stores* (`.svar` vs `.svar2`), not query artifacts.** + SVAR1's low latency is served from an additional pre-materialized `.gvl` Dataset that + `gvl.write` produces (here only the 3 benchmark regions: germline `.gvl` 292 KB, somatic + `.gvl` 1.6 MB — tiny because it holds just those regions, not the contig). The + `.svar`-vs-`.svar2` size comparison is the correct *variant-source* comparison; just note + SVAR1 additionally materializes a `.gvl` to reach its quoted latency, and a fully-wired + SVAR2 (Task B) would likewise gain a Dataset artifact. + +### What we *can* say about layout (checked in source, not profiled) + +The user hypothesized the SVAR2 dense matrix is variant-major `(V, S, P)` — pessimal for +gvl's access pattern (contiguous variant slices for random `(sample, ploid)`). **Checked and +refuted:** the dense genotype/presence matrix is **hap-major**, variant *innermost* — bit +index `hap * n_dense_variants + col` (`genoray:src/query.rs:131-134`, matching +`data-model.md`'s "hap-major `(sample, ploid, variant)`, variant fastest-varying"). During +build the in-memory chunk is variant-major (`BitGrid3(v,s,p)`) and `dense_merge` +bit-**transposes** it to hap-major on disk (roadmap M4). So a *single* hap's dense variants +are contiguous — the good case for per-hap reconstruction. + +A **related, real** concern remains (also unprofiled): `n_dense_variants` is the **whole +contig's** dense count, so a narrow-region × all-samples query reads one short contiguous +run per hap at stride `n_dense_variants` → a separate cache line per haplotype, scattered +across a contig-wide matrix; and presence is read **bit-by-bit** (`get_bit` per column), not +word-parallel. For germline (many common → dense variants → large `n_dense_variants`) that +scatter is worse, which is *directionally consistent* with germline's higher SVAR2 hap +latency — but this is a hypothesis for the profiler, not a conclusion, and it is +**contig-scoped**, so it ties into the split-by-contig question below. + +## Limitations / open questions (before strong claims or Task B) + +1. **Profile before concluding.** Run `perf` + a Rust profiler on both hap paths; separately + measure the SVAR2 adapter's Python overhead vs its Rust kernel. Only then attribute the + latency gap to a cause. +2. **Same-cohort sample sweep.** The germline↔somatic pair is confounded (different AF / + density / source). To claim SVAR2 scales better with S, subsample one cohort and vary S + with the dataset fixed. +3. **Single contig — split-by-contig layout unassessed.** All measurements are chr21 only. + The contig-wide dense stride above and genoray's per-contig on-disk partition (M3) can + only be evaluated multi-contig; defer that analysis. +4. **Build: reading dominates; single-contig underuses threads.** The conversion logs show + the pipeline reserving `1 concurrent chromosome | 3 HTSlib decompression threads (7 of 8 + active, 1 idle)` — for a single-contig job most cores sit idle and VCF read/decompress is + plainly the bottleneck (germline ~11 min, somatic ~2 h). Worth exploring a thread split + that dedicates ~1 thread to the executor+writer and the rest to VCF decompress+read when + few contigs are present (genoray conversion-pipeline tuning, not gvl). ## Value proposition & Task B signal -- **SVAR2 is a clear win on disk footprint** (1.5–5.7×), scaling with allele frequency — - the more common the variants, the bigger the win. -- **SVAR2 latency scales much better with cohort size** — already at parity on hap latency - at 16k samples via the un-optimized adapter alone. -- **Concrete Task B cost signal:** the all-samples-per-region query is the granularity that - matters, and the adapter already competes there at scale. Wiring SVAR2 into the Dataset - (Task B) should be pursued: the storage advantage is unconditional, and the latency - trend says the Dataset's batching/collation would likely make SVAR2 competitive-to-faster - on latency too for realistic large cohorts. The invasive integration is justified by the - storage win alone; the scaling trend strengthens the case. +- **SVAR2 is a clear, unconfounded win on disk footprint** (1.46× somatic, 5.67× germline; + larger where variants are common/high-AF). This alone is a strong reason to pursue Task B. +- **On the large 16k-sample cohort, the un-optimized SVAR2 adapter already matches SVAR1 on + hap latency** — so wiring SVAR2 into the Dataset (Task B) does not start from a latency + hole at scale. It is *not* established that SVAR2 "scales better with cohort size" (see + Limitation 2). +- **Recommendation:** pursue Task B for the storage win, but gate any latency claims on the + profiling in Limitation 1 first — a real chance the current SVAR2 adapter latency is + Python-adapter overhead the Dataset path would erase, and an equally real chance the + contig-wide dense access needs a layout fix. Profile, then decide how much layout work + Task B should include. diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md index 7746c0a0..343e7d18 100644 --- a/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md @@ -98,10 +98,12 @@ both cohorts. `du` on the four stores: +Apparent sizes (`du -sb`); decimal labels (MB = 10⁶ B, GB = 10⁹ B): + | Store | SVAR1 (`.svar`) | SVAR2 (`.svar2`) | SVAR2 / SVAR1 | SVAR2 advantage | | --- | --- | --- | --- | --- | -| germline (3202 smp) | 1.1 G (1,149,533,941 B) | 178 M (202,842,586 B) | 0.176× | **5.67× smaller** | -| somatic (16007 smp) | 50 M (55,578,073 B) | 34 M (38,184,053 B) | 0.687× | **1.46× smaller** | +| germline (3202 smp) | 1,149,533,941 B (1.15 GB) | 202,842,586 B (203 MB) | 0.176× | **5.67× smaller** | +| somatic (16007 smp) | 55,578,073 B (55.6 MB) | 38,184,053 B (38.2 MB) | 0.687× | **1.46× smaller** | SVAR2 wins on size for **both** cohorts. The germline win is dramatic (5.7×): 1000G carries many **common, high-allele-frequency** variants, which SVAR2's cost model routes From 495b193800ed2a7eabdc2967fa5eb7a9344d509c Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 21:21:39 -0700 Subject: [PATCH 014/108] docs: SVAR2 profiling follow-up spec (query latency attribution + build threads) Concrete measurement plan to de-confound the MVP benchmark's latency numbers: E1 py-spy Python-vs-native split then perf Rust detail (attribute the SVAR2 adapter gap to Python overhead vs Rust hot paths vs dense layout); E2 same-cohort sample sweep; E3 dense-access layout probe; E4 conversion thread-allocation. Notes the py3.10 perf caveat (Python frames unsymbolizable pre-3.12, so py-spy for Python-side + perf for Rust). Measurement only; feeds Task B + genoray conversion thread-policy decisions. Co-Authored-By: Claude Opus 4.8 --- .../2026-07-03-svar2-profiling-followup.md | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md diff --git a/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md new file mode 100644 index 00000000..eb051f75 --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md @@ -0,0 +1,153 @@ +# SVAR2 Profiling Follow-up — Spec + +> **Purpose:** turn the SVAR2 MVP benchmark's *unprofiled* latency observations into +> **concrete, attributed numbers** so we can decide (a) how much of the SVAR2 query-latency +> gap is Python-adapter overhead vs Rust hot paths vs memory layout, and (b) how to +> rebalance conversion threads for read-bound (few-contig) builds. **Measurement only — no +> optimization work is in scope here.** + +**Date:** 2026-07-03 · **Depends on:** the MVP stores + scripts from +`2026-07-03-svar2-gvl-mvp-validate-benchmark` (plan) and its notes +[`../notes/2026-07-03-svar2-mvp-benchmark.md`](../notes/2026-07-03-svar2-mvp-benchmark.md), +[`../notes/2026-07-03-svar2-mvp-validation.md`](../notes/2026-07-03-svar2-mvp-validation.md). + +## Why (background the next session needs) + +The MVP benchmark measured SVAR1 (gvl `Dataset` over `.svar`) vs SVAR2 (`SparseVar2Source` +adapter over `.svar2`), all-samples × 3 chr21 regions, warm, median N=5: + +| cohort | SVAR1 hap (s) | SVAR2 hap (s) | store SVAR1→SVAR2 | +| --- | --- | --- | --- | +| germline (3202 smp) | 0.0555 | 0.2834 | 5.67× smaller | +| somatic (16007 smp) | 0.3798 | 0.3797 | 1.46× smaller | + +**Established (unconfounded):** SVAR2's on-disk store is 1.46–5.67× smaller. + +**NOT established — the reasons this spec exists:** +1. The latency comparison is **not apples-to-apples**: the SVAR2 path is the *raw adapter* + (new Python orchestration — per-call genoray `overlap_batch`, numpy conversions, the + two-source Rust kernel), **not** the gvl `Dataset`. Prior gvl profiling (Dataset latency + is Rust-bound, Python-orchestration negligible) does **not** transfer to this adapter. +2. The "SVAR2 scales better with cohort size" reading is **confounded** — germline vs + somatic differ in AF/density/source, not just sample count. +3. The dense presence matrix is **hap-major** (`genoray:src/query.rs:131-134`, bit + `hap * n_dense_variants + col`; refutes a `(V,S,P)` guess) — per-hap contiguous — **but** + `n_dense_variants` is **contig-wide**, so a narrow-region × all-samples read scatters one + cache line per hap, and presence is read **bit-by-bit** (`get_bit`), not word-parallel. + Plausible but unmeasured. +4. Conversion is **read-bound**: single-contig builds show `1 concurrent chromosome | 3 + HTSlib decompression threads (7/8 active)`, VCF read/decompress dominating (chr21: + germline ~11 min, somatic ~2 h). Thread rebalance is untested. + +## Environment & tooling (read before profiling) + +- **Env:** `pixi run -e default` (only env installed). **Python 3.10.20.** +- **CRITICAL perf/Python caveat:** on **Python < 3.12**, `perf` **cannot** symbolize Python + stack frames — the `-X perf` / `PYTHONPERFSUPPORT` trampoline that emits `perf`-readable + Python symbols only exists in **3.12+** + (see ). This env is 3.10, so `perf` + alone shows resolved **Rust/native** symbols but **opaque** Python frames + (`_PyEval_EvalFrameDefault` …). Therefore: + - **py-spy** (`.pixi/envs/default/bin/py-spy`, confirmed present) — sampling profiler that + resolves **Python** frames on any version and, with `--native`, also unwinds into the + **Rust** extension. Use it for the **Python-vs-native split** (the "total Rust %") and + Python-side hotspots. + - **perf** (`/carter/users/dlaub/.pixi/bin/perf`, confirmed present) — use for + **fine-grained Rust symbol-level** profiling once py-spy says Rust dominates. +- **Rust symbolization for perf/py-spy --native:** rebuild the genoray + gvl extensions + with frame pointers + line-table debuginfo. In the relevant `Cargo.toml` profile (or via + env): `RUSTFLAGS="-C force-frame-pointers=yes"` and `debug = "line-tables-only"` (or + `debug = true`); then `pixi run -e default maturin develop` for gvl, and rebuild the + genoray wheel (its build is `pixi run -e py310 maturin build --release` in the genoray + repo — note genoray is a **frozen wheel** in gvl's env, so a debug-enabled rebuild + wheel + swap is needed to symbolize genoray's `query.rs`/`svar2-codec` frames). Prefer + `perf record -g --call-graph dwarf` if frame pointers are unavailable. +- **Noise control:** run on a dedicated compute node (`sbatch -p carter-compute + --exclusive` or an interactive alloc), warm caches, median of N≥5, pin threads, record + CPU governor / turbo state. Keep the MVP workload (same 3 regions, all-samples) for + continuity. +- **Stores (already built):** `$W = /carter/users/dlaub/repos/for_loukik/svar2_mvp`: + `{germline,somatic}.{svar,svar2,gvl}` + `{chr21,gdc.chr21}.norm.filt.bcf`. Reuse the MVP + `benchmark.py` harness (`GenVarLoader worktree: tmp/svar2_mvp/benchmark.py`). + +## Experiments + +### E1 — Query-latency attribution (the primary experiment) + +For each **(backend × cohort)** — SVAR2 adapter `reconstruct` and SVAR1 `ds_hap[:R,:S]`, +germline + somatic — on the MVP workload (3 regions, all samples): + +1. **py-spy split.** `py-spy record --native --rate 500 -o .svg -- python ` + where `` loops the single path K times (warm first). Also capture + `py-spy record --format speedscope`. Extract: **% wall-clock in Python vs % in native + (Rust)**, and the top Python frames (numpy conversions, `overlap_batch` marshalling, FFI + boundary). This directly answers *"is SVAR2 adapter latency Python overhead?"*. +2. **perf Rust detail** (only where py-spy shows native dominates): `perf record -g + --call-graph dwarf -- python `; `perf report`. Read the top **Rust** symbols — + e.g. `svar2_codec` decode, the dense presence `get_bit` gather, `overlap_range` search, + memcpy/alloc. (Python frames will be opaque — that's expected; py-spy covered them.) + +**Deliverable E1:** a table per (backend × cohort) of `{Python %, native %, wall-clock}` + +the top ~10 Rust symbols for the SVAR2 path. **Decision output:** classify the SVAR2 gap as +(A) Python-adapter overhead → Task B Dataset wiring likely erases it; (B) Rust hot-path / +layout → Task B must include a kernel/layout fix; or (C) mix (quantify). + +### E2 — Same-cohort sample sweep (de-confound "scales with S") + +Hold the **dataset fixed** and vary sample count S. Pick the **somatic** cohort (widest +range, up to 16007). Subsample the filtered BCF at **S ∈ {1000, 2000, 4000, 8000, 16007}** +(`bcftools view -S --force-samples`), and for each S build **both** `.svar` and +`.svar2` (reuse `build_stores.py`; SLURM). Benchmark hap latency (warm, median N=5, same 3 +regions, all S samples) for both backends at each S. + +> Note the mild caveat: subsampling drops sites monomorphic in the subset, so variant count +> shrinks slightly with S; the AF *spectrum* is approximately preserved. Record per-S +> variant counts so the curves are interpretable. + +**Deliverable E2:** hap-latency-vs-S curves for both backends on one cohort, with slopes. +**Decision output:** confirm/refute that SVAR2 latency is less S-sensitive than SVAR1 — +the claim the MVP notes explicitly declined to make. + +### E3 — Dense-access layout probe (conditional on E1 showing dense gather hot) + +If E1 attributes significant Rust time to the dense presence gather: measure SVAR2 hap +latency vs **(a) region width** and **(b) `n_dense_variants`** (contig-wide dense count). +Optionally a Rust microbench of the `get_bit`-per-column gather vs a word-parallel variant. +**Deliverable E3:** evidence for/against the "contig-wide stride + bit-by-bit read" cost, +and a rough size of the win from a region-local / word-parallel presence layout — input to +how much layout work Task B should carry. + +### E4 — Conversion thread-allocation profiling (build side) + +1. **Confirm the bottleneck.** Profile a single-contig `run_conversion_pipeline` on a + compute node: py-spy `--native` (or perf on the Rust threads) to quantify time in + **htslib read/decompress** vs **encode** vs **Phase-2 merge**. (genoray already exposes a + sampler via `GENORAY_SAMPLE_INTERVAL` — capture its channel-fill / per-thread CPU% + alongside.) +2. **Sweep the split.** For a fixed single-contig input, vary the thread policy — htslib + decompression threads vs executor/writer threads (and total cores) — and measure build + wall-clock. Find the split minimizing single-contig time. + +**Deliverable E4:** build wall-clock vs thread-split table + a concrete recommended policy +for few-contig jobs, feeding +`genoray:docs/roadmap/architecture.md` → Open questions → *read-bound conversion / thread +allocation*. + +## Deliverables & out of scope + +- **Deliverable:** a notes file + `docs/superpowers/notes/2026-07-XX-svar2-profiling-results.md` with the E1–E4 tables and + two concrete recommendations: (1) where SVAR2 query latency actually goes (→ what Task B + must include), and (2) the conversion thread-split policy. Profiling driver scripts under + `tmp/svar2_mvp/`. +- **Out of scope (deliberately):** implementing Task B (Dataset wiring), any dense-layout + change, and the conversion thread rebalance. This spec produces the numbers that decide + *whether and how* to do those. + +## Suggested execution + +Small enough for a single focused session; a step-by-step plan can be derived from E1–E4 and +run via superpowers:subagent-driven-development. Start with **E1** (highest decision value: +it tells us if the whole latency story is just un-wired Python) and **E4** (independent, +build-side) — they need no new stores. E2 needs the SLURM sub-cohort builds; E3 is +conditional on E1. From 11c396312b39e457d9bae34291c4726974fbb3ed Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 21:48:30 -0700 Subject: [PATCH 015/108] docs: SVAR2 profiling follow-up implementation plan (E1-E4) Co-Authored-By: Claude Opus 4.8 --- .../2026-07-03-svar2-profiling-followup.md | 971 ++++++++++++++++++ 1 file changed, 971 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md diff --git a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md new file mode 100644 index 00000000..f058e4a8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md @@ -0,0 +1,971 @@ +# SVAR2 Profiling Follow-up Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Attribute the SVAR2 adapter's query-latency gap to Python overhead vs Rust hot paths vs memory layout, and find the conversion thread-split policy for read-bound (few-contig) builds — measurement only, no optimization. + +**Architecture:** Add small parametrized *profiling driver* scripts under `tmp/svar2_mvp/` that exercise a single code path (SVAR1 `Dataset` hap query, SVAR2 adapter `reconstruct`, or genoray `run_conversion_pipeline`) in a warm loop. Profile them with py-spy (Python-vs-native split, any Python version) and perf (Rust symbol detail, needs debug-symbolized extensions). Accumulate every result table into one notes file. All timed runs go on a dedicated `carter-compute` node. + +**Tech Stack:** Python 3.10.20 via `pixi run -e default`; genoray 2.15.0 (Rust/PyO3, source at `/carter/users/dlaub/projects/genoray`); gvl Rust extension in this repo's `src/`; py-spy 0.x (sampling profiler); Linux `perf`; SLURM (`carter-compute`); bcftools. + +## Global Constraints + +Copied verbatim from the spec — every task's requirements implicitly include these. + +- **This is measurement only.** No optimization, no Dataset wiring (Task B), no dense-layout change, no conversion rebalance. The output is *numbers that decide whether/how to do those*. +- **Env:** `pixi run -e default` (the only installed env). **Python 3.10.20.** +- **perf cannot symbolize Python frames on Python < 3.12.** This env is 3.10, so `perf` shows resolved **Rust/native** symbols but **opaque** Python frames (`_PyEval_EvalFrameDefault`). Use **py-spy** for the Python-vs-native split and Python hotspots; use **perf** only for Rust symbol detail once py-spy says native dominates. +- **Profilers (confirmed present):** py-spy at `.pixi/envs/default/bin/py-spy`; perf at `/carter/users/dlaub/.pixi/bin/perf`. +- **Noise control:** compute node via `srun`/`sbatch -p carter-compute` (do **NOT** use `--exclusive` — Carter has only 3 compute nodes, always shared, so `--exclusive` won't schedule). The node is shared and noisy, so **absolute wall-clock is not comparable across allocations** — only *relative* comparisons **within one short allocation on the same hardware** are valid (matches the prior perf-gate lesson: gate on same-session before/after, not absolute time). Therefore: **run both backends of any comparison inside a single `srun`** (back-to-back on the same node), warm caches, **median of N≥5**, record CPU governor / turbo. CPU governor on the target nodes is `performance` — record the actual value observed on the node you land on. Keep the MVP workload (same 3 chr21 regions, all samples) for continuity. +- **MVP workload regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)`. +- **Stores (already built), `$W`:** `/carter/users/dlaub/repos/for_loukik/svar2_mvp` — `{germline,somatic}.{svar,svar2,gvl}` + `{chr21,gdc.chr21}.norm.filt.bcf`. germline = 3202 samples (1000G, high-AF/dense); somatic = 16007 samples (GDC, rare/sparse). +- **Reference FASTA:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. +- **Contig:** both cohorts are chr21-only. germline chrom label and somatic chrom label are both `chr21`. +- **Working dir:** the gvl worktree `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`. Scripts live in its `tmp/svar2_mvp/`. +- **Deliverable notes file:** `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md`, holding the E1–E4 tables + two recommendations. +- **RTK:** prefix git/build shell commands with `rtk` per project CLAUDE.md. +- **Commit style:** conventional commits (this is a `chore`/`docs` profiling branch — driver scripts are `chore`, notes are `docs`). + +--- + +## File Structure + +All new files land under `tmp/svar2_mvp/` (profiling drivers, throwaway — not gold-plated) except the notes deliverable. + +- `tmp/svar2_mvp/prof_driver.py` — **E1.** Parametrized single-path driver: `python prof_driver.py `. Warms once, then loops the chosen path K times. For SVAR1 it writes the 3-region `.gvl` once *outside* the loop (so we profile the query, not `gvl.write`). Prints `per_call_s=`. +- `tmp/svar2_mvp/split_folded.py` — **E1/E4.** Post-processes a py-spy folded (`--format raw`) stack file into a **leaf-attributed Python% vs native%** split, plus top-N leaf frames per class. +- `tmp/svar2_mvp/e1_profile.sh` — **E1.** Orchestrates py-spy (flamegraph SVG + speedscope + folded raw) and perf (dwarf call-graph) captures for each `(backend × cohort)` on the compute node; drops artifacts in `tmp/svar2_mvp/prof_out/e1/`. +- `tmp/svar2_mvp/e2_subsample.sh` — **E2.** Builds per-S sample lists and subsampled BCFs (`bcftools view -S`). +- `tmp/svar2_mvp/e2_build.sbatch` — **E2.** SLURM array: for each S, build `.svar` + `.svar2` (reuses `build_stores.py`). +- `tmp/svar2_mvp/e2_bench.py` — **E2.** Hap-latency-vs-S sweep for both backends (warm, median N=5), emits a TSV. +- `tmp/svar2_mvp/e3_probe.py` — **E3 (conditional).** Sweeps SVAR2 hap latency vs (a) region width and (b) `n_dense_variants`. +- `tmp/svar2_mvp/e4_convert_driver.py` — **E4.** Runs one single-contig `run_conversion_pipeline` with a given `max_threads`, under optional `GENORAY_SAMPLE_INTERVAL`. Prints build wall-clock. +- `tmp/svar2_mvp/e4_sweep.sbatch` — **E4.** SLURM: sweep `max_threads` on a fixed single-contig input; capture wall-clock + genoray's reported thread split. +- `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` — **the deliverable.** E1–E4 tables + two recommendations. + +--- + +## Task 0: Symbolized-build setup + notes skeleton + node baseline + +**Files:** +- Create: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` +- Create: `tmp/svar2_mvp/env_baseline.txt` (recorded node/env state) +- No source edits — we inject debug info via Cargo env vars, not Cargo.toml edits. + +**Interfaces:** +- Consumes: nothing. +- Produces: a gvl extension and a genoray wheel both built `--release` **with** `debug = line-tables-only` + frame pointers, installed into the `default` env; a notes file with empty E1–E4 section headers; a recorded env baseline that later tasks cite. The symbol `reconstruct_haplotypes_from_svar2` and genoray's `overlap_batch`/query symbols must resolve in perf after this task. + +- [ ] **Step 1: Grab an interactive compute node and record the baseline** + +Run (from the worktree root): +```bash +srun -p carter-compute --cpus-per-task=16 --pty bash +# then, on the node: +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +{ echo "host=$(hostname)"; echo "date=$(date -Iseconds)"; + echo "governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)"; + echo "turbo_no_turbo=$(cat /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null)"; + echo "nproc=$(nproc)"; + echo "paranoid=$(cat /proc/sys/kernel/perf_event_paranoid)"; } | tee tmp/svar2_mvp/env_baseline.txt +``` +Expected: `governor=performance`, `paranoid` ≤ 2 (memory: paranoid=2 works, no sudo). If `paranoid` > 2, perf call-graph capture will be restricted — note it; py-spy still works. + +> If interactive `srun` isn't practical for the agent, wrap each later timed command as `srun -p carter-compute --cpus-per-task=16 ` instead. Every timed/profiled run in this plan MUST be on a compute node, never the login node. + +- [ ] **Step 2: Confirm profilers and store layout on the node** + +Run: +```bash +.pixi/envs/default/bin/py-spy --version +/carter/users/dlaub/.pixi/bin/perf --version +ls /carter/users/dlaub/repos/for_loukik/svar2_mvp/{germline,somatic}.svar2 +``` +Expected: py-spy prints a version; perf prints a version; both `.svar2` dirs list. + +- [ ] **Step 3: Rebuild the gvl extension with release + line-table debug + frame pointers** + +Run: +```bash +CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ +RUSTFLAGS="-C force-frame-pointers=yes" \ +rtk pixi run -e default maturin develop --release +``` +Expected: `maturin` finishes; the compiled `.so` under `.pixi/envs/default/.../genvarloader/` is rebuilt. (This does NOT edit `Cargo.toml` — the `CARGO_PROFILE_RELEASE_DEBUG` env var injects debug info into the release profile for this build only.) + +- [ ] **Step 4: Rebuild + swap the genoray wheel with the same debug flags** + +genoray source is at `/carter/users/dlaub/projects/genoray` (pyproject version 2.15.0 — matches the installed wheel). Build a debug-symbolized release wheel and force-install it into gvl's `default` env: +```bash +cd /carter/users/dlaub/projects/genoray +CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ +RUSTFLAGS="-C force-frame-pointers=yes" \ +rtk pixi run -e py310 maturin build --release +# install the freshly built wheel into gvl's default env (py310 ABI == default's 3.10.20): +WHEEL=$(ls -t target/wheels/genoray-2.15.0-*.whl | head -1) +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +rtk pixi run -e default pip install --force-reinstall --no-deps "/carter/users/dlaub/projects/genoray/$WHEEL" +``` +Expected: a wheel is produced under `genoray/target/wheels/`; `pip install` reports genoray 2.15.0 reinstalled. Verify version unchanged: +```bash +rtk pixi run -e default python -c "import genoray; print(genoray.__version__)" +``` +Expected: `2.15.0`. + +> **Risk / degrade-gracefully:** if the genoray wheel rebuild fails (toolchain/htslib build issue) or its version drifts from 2.15.0, do NOT block the whole plan. gvl's own kernel (`reconstruct_haplotypes_from_svar2`, in *this* repo's `src/`) was rebuilt in Step 3 and will symbolize regardless. Record in the notes that genoray-internal frames (`overlap_batch`, `query.rs`, `svar2-codec`) may appear as raw addresses in perf, and lean on py-spy `--native` (which still attributes them to the genoray `.so` module even without line tables) for those. Keep going. + +- [ ] **Step 5: Verify Rust symbolization end-to-end** + +Run a 3-second perf capture over the SVAR2 driver (driver written in Task 1 — for now smoke with a one-liner) and grep for a known gvl symbol: +```bash +/carter/users/dlaub/.pixi/bin/perf record -g --call-graph dwarf -o /tmp/sym_check.data -- \ + rtk pixi run -e default python -c " +import numpy as np +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source +import pysam +REF='/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa' +rb=pysam.FastaFile(REF).fetch('chr21').encode() +ru=np.frombuffer(rb,np.uint8); ro=np.array([0,len(rb)],np.int64) +regs=[(20_000_000,20_001_000),(30_000_000,30_000_500),(40_000_000,40_001_000)] +src=SparseVar2Source(SparseVar2('/carter/users/dlaub/repos/for_loukik/svar2_mvp/germline.svar2')) +for _ in range(20): src.reconstruct('chr21',regs,ru,ro,pad_char=ord('N'),shifts=None,output_length=-1) +" +/carter/users/dlaub/.pixi/bin/perf report --stdio -i /tmp/sym_check.data 2>/dev/null | grep -iE "reconstruct_haplotypes_from_svar2|svar2|overlap|query" | head +``` +Expected: at least `reconstruct_haplotypes_from_svar2` (and ideally genoray `overlap`/`query` symbols) appear as named frames, not bare `0x...` addresses. If only gvl symbols resolve, that matches the Step 4 degrade note — acceptable. + +- [ ] **Step 6: Create the results-notes skeleton** + +Write `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md`: +```markdown +# SVAR2 Profiling Results (E1–E4) + +**Date:** 2026-07-03 · **Spec:** `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` +**Node/env baseline:** see `tmp/svar2_mvp/env_baseline.txt`. +**Symbolization:** gvl + genoray rebuilt `--release` with `debug=line-tables-only` + frame pointers. + +## E1 — Query-latency attribution +_(filled by Task 3)_ + +## E2 — Same-cohort sample sweep +_(filled by Task 5)_ + +## E3 — Dense-access layout probe +_(filled by Task 6; conditional on E1)_ + +## E4 — Conversion thread-allocation +_(filled by Task 8)_ + +## Recommendations +_(filled by Task 9)_ +``` + +- [ ] **Step 7: Commit** + +```bash +rtk git add tmp/svar2_mvp/env_baseline.txt docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: SVAR2 profiling env baseline + symbolized builds + notes skeleton" +``` + +--- + +## Task 1: E1 profiling driver (`prof_driver.py`) + +**Files:** +- Create: `tmp/svar2_mvp/prof_driver.py` + +**Interfaces:** +- Consumes: `$W` stores; `SparseVar2Source.reconstruct(contig, regions, ref_, ref_offsets, pad_char, shifts, output_length)` (signature in `python/genvarloader/_dataset/_svar2_source.py:51`); `gvl.Dataset.open(path, reference).with_seqs("haplotypes")[:R, :n_s]`. +- Produces: CLI `python prof_driver.py ` where `backend ∈ {svar1, svar2}`, `cohort ∈ {germline, somatic}`, `K` = loop count. Prints exactly one line `per_call_s=`. For `svar1` it writes the 3-region `.gvl` **once before the loop** so profiling captures the query, not `gvl.write`. + +- [ ] **Step 1: Write the driver** + +Create `tmp/svar2_mvp/prof_driver.py`: +```python +"""E1 single-path profiling driver. Exercises ONE code path in a warm loop so +py-spy/perf attribute time to that path only. + + python prof_driver.py + +Prints: per_call_s= +For svar1, the 3-region .gvl is written ONCE before the loop (we profile the +query, not gvl.write).""" +import sys +import time + +import numpy as np + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] + + +def _ref(): + import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + return np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + + +def make_svar2(cohort): + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + src = SparseVar2Source(SparseVar2(f"{W}/{cohort}.svar2")) + ru, ro = _ref() + + def call(): + src.reconstruct(CHROM, REGIONS, ru, ro, pad_char=ord("N"), + shifts=None, output_length=-1) + return call + + +def make_svar1(cohort): + import polars as pl + import genvarloader as gvl + from genoray import SparseVar2 + n_s = SparseVar2(f"{W}/{cohort}.svar2").n_samples + bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS]}) + ds_path = f"{W}/{cohort}.gvl" + gvl.write(ds_path, bed, variants=f"{W}/{cohort}.svar", overwrite=True) # ONCE + ds_hap = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + + def call(): + ds_hap[:len(REGIONS), :n_s] + return call + + +def main(backend, cohort, K): + call = {"svar1": make_svar1, "svar2": make_svar2}[backend](cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + dt = time.perf_counter() - t0 + print(f"per_call_s={dt / K:.4f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 2: Smoke-test all four (backend × cohort) combos on the node** + +Run (short K to keep it fast): +```bash +for b in svar2 svar1; do for c in germline somatic; do + echo -n "$b $c -> "; rtk pixi run -e default python tmp/svar2_mvp/prof_driver.py $b $c 5 +done; done +``` +Expected: four `per_call_s=` lines, all `>0`, roughly consistent with the MVP table (svar2 germline ~0.28, svar1 germline ~0.06, both somatic ~0.38). If svar1 numbers are ~10× higher than the MVP, the `.gvl` write leaked into timing — verify `gvl.write` is outside the loop. + +- [ ] **Step 3: Commit** + +```bash +rtk git add tmp/svar2_mvp/prof_driver.py +rtk git commit -m "chore: E1 single-path profiling driver for SVAR1/SVAR2 hap query" +``` + +--- + +## Task 2: Python-vs-native split post-processor (`split_folded.py`) + +**Files:** +- Create: `tmp/svar2_mvp/split_folded.py` + +**Interfaces:** +- Consumes: a py-spy `--format raw` folded-stack file (one `frame1;frame2;...;leaf ` line per unique stack). py-spy marks Python frames as `path.py:func:line` and native frames (with `--native`) as demangled symbols or `` without a `.py:` token. +- Produces: CLI `python split_folded.py ` printing `python_pct= native_pct= total_samples=` and the top-15 **leaf** frames with per-frame sample counts and class. Leaf-attribution = self-time. A frame is Python iff its text contains `.py:`. + +- [ ] **Step 1: Write the post-processor** + +Create `tmp/svar2_mvp/split_folded.py`: +```python +"""Split a py-spy --format raw (folded) stack file into Python vs native +self-time by LEAF frame. A leaf frame is Python iff it contains '.py:'. + + python split_folded.py +""" +import sys +from collections import Counter + + +def is_python(frame: str) -> bool: + return ".py:" in frame or frame.endswith(".py") + + +def main(path): + py = nat = 0 + leaves = Counter() + classed = {} + with open(path) as fh: + for line in fh: + line = line.rstrip("\n") + if not line: + continue + stack, _, cnt = line.rpartition(" ") + try: + n = int(cnt) + except ValueError: + continue + leaf = stack.split(";")[-1] + leaves[leaf] += n + classed[leaf] = "python" if is_python(leaf) else "native" + if is_python(leaf): + py += n + else: + nat += n + tot = py + nat + if tot == 0: + print("no samples parsed"); return + print(f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}") + print("top-15 leaf frames (self-time):") + for leaf, n in leaves.most_common(15): + print(f" {100 * n / tot:5.1f}% [{classed[leaf]:6s}] {leaf}") + + +if __name__ == "__main__": + main(sys.argv[1]) +``` + +- [ ] **Step 2: Unit-smoke it on a synthetic folded file** + +Run: +```bash +printf 'a.py:f:1;b.py:g:2 10\nc.py:h:3;native_symbol 30\n' > /tmp/folded_smoke.txt +rtk pixi run -e default python tmp/svar2_mvp/split_folded.py /tmp/folded_smoke.txt +``` +Expected: `python_pct=25.0 native_pct=75.0 total_samples=40`, then two leaf lines (`b.py:g:2` python 25%, `native_symbol` native 75%). + +- [ ] **Step 3: Commit** + +```bash +rtk git add tmp/svar2_mvp/split_folded.py +rtk git commit -m "chore: py-spy folded-stack Python/native self-time splitter" +``` + +--- + +## Task 3: E1 — capture profiles + fill the attribution table + +**Files:** +- Create: `tmp/svar2_mvp/e1_profile.sh` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E1 section) +- Artifacts: `tmp/svar2_mvp/prof_out/e1/` (flamegraphs, speedscope, folded, perf data) — gitignored via not committing binaries. + +**Interfaces:** +- Consumes: `prof_driver.py` (Task 1), `split_folded.py` (Task 2), the symbolized builds (Task 0). +- Produces: E1 table `{backend, cohort, python_pct, native_pct, per_call_s}` for all four combos + top-~10 Rust symbols for the two SVAR2 paths, and a written classification **A / B / C** (Python-adapter overhead / Rust hot-path / mix). + +- [ ] **Step 1: Write the capture orchestrator** + +Create `tmp/svar2_mvp/e1_profile.sh`: +```bash +#!/usr/bin/env bash +# E1: py-spy split (all 4) + perf Rust detail (svar2 only). Run ON a compute node. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/e1 +mkdir -p "$OUT" +PYSPY=.pixi/envs/default/bin/py-spy +PERF=/carter/users/dlaub/.pixi/bin/perf +K=200 # warm loops per capture; adjust so each capture runs >=15s of wall time + +run_pyspy () { # backend cohort + local b=$1 c=$2 tag="${1}_${2}" + # native flamegraph (visual) + folded raw (for the split) + speedscope + $PYSPY record --native --rate 500 --format flamegraph -o "$OUT/${tag}.svg" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + $PYSPY record --native --rate 500 --format raw -o "$OUT/${tag}.folded" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + $PYSPY record --native --rate 500 --format speedscope -o "$OUT/${tag}.speedscope.json" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + echo "== split $tag ==" | tee -a "$OUT/splits.txt" + pixi run -e default python tmp/svar2_mvp/split_folded.py "$OUT/${tag}.folded" | tee -a "$OUT/splits.txt" +} + +run_perf () { # backend cohort (svar2 only) + local b=$1 c=$2 tag="${1}_${2}" + $PERF record -g --call-graph dwarf -o "$OUT/${tag}.perf.data" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + echo "== perf top symbols $tag ==" | tee -a "$OUT/perf_top.txt" + $PERF report --stdio -i "$OUT/${tag}.perf.data" --sort=overhead,symbol -g none 2>/dev/null \ + | grep -vE '^\s*#' | head -25 | tee -a "$OUT/perf_top.txt" +} + +for b in svar2 svar1; do for c in germline somatic; do run_pyspy "$b" "$c"; done; done +for c in germline somatic; do run_perf svar2 "$c"; done +echo "DONE. splits -> $OUT/splits.txt ; perf -> $OUT/perf_top.txt" +``` + +- [ ] **Step 2: Run it on the compute node** + +Run: +```bash +chmod +x tmp/svar2_mvp/e1_profile.sh +srun -p carter-compute --cpus-per-task=16 bash tmp/svar2_mvp/e1_profile.sh +``` +Expected: prints four `python_pct=… native_pct=…` blocks and two perf top-symbol blocks; `tmp/svar2_mvp/prof_out/e1/splits.txt` and `perf_top.txt` populated. Sanity: the SVAR1 path (known Rust-bound from prior gvl profiling) should show high `native_pct`; if SVAR1 shows high `python_pct`, py-spy failed to unwind native — check that `--native` is active and the build has frame pointers (Task 0 Step 3). + +- [ ] **Step 3: Extract per-call wall-clock for the table** + +Run (clean timing, no profiler attached — profilers inflate wall time). **One allocation for all four** so the numbers are comparable (shared node — see Global Constraints): +```bash +srun -p carter-compute --cpus-per-task=16 bash -c ' +for b in svar1 svar2; do for c in germline somatic; do + echo -n "$b $c "; pixi run -e default python tmp/svar2_mvp/prof_driver.py $b $c 50 +done; done' +``` +Expected: four `per_call_s=` values; record them as the wall-clock column. (Absolute values are node-dependent; only their *ratios within this run* are load-bearing.) + +- [ ] **Step 4: Read the top Rust symbols for the SVAR2 paths** + +Run: +```bash +cat tmp/svar2_mvp/prof_out/e1/perf_top.txt +``` +Read the top ~10 named Rust symbols (expect candidates: `reconstruct_haplotypes_from_svar2`, genoray `overlap`/`query` decode, dense presence `get_bit` gather, `svar2_codec` decode, memcpy/alloc). Note any that are bare addresses (genoray un-symbolized per Task 0 degrade note). + +- [ ] **Step 5: Fill the E1 section of the notes file** + +Replace the `## E1` placeholder in `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` with a table of the real captured numbers, in this exact shape (fill `<...>` from Steps 2–4): +```markdown +## E1 — Query-latency attribution + +Workload: 3 chr21 regions × all samples, warm, K-loop. py-spy `--native` rate 500; +leaf-attributed Python/native self-time via `split_folded.py`. perf dwarf call-graph. + +| backend | cohort | per_call_s | python_pct | native_pct | +| ------- | -------- | ---------- | ---------- | ---------- | +| svar1 | germline | <...> | <...> | <...> | +| svar1 | somatic | <...> | <...> | <...> | +| svar2 | germline | <...> | <...> | <...> | +| svar2 | somatic | <...> | <...> | <...> | + +**Top SVAR2 Rust symbols (perf, self-overhead):** +1. — + ... (~10) + +**Top SVAR2 Python leaf frames (py-spy):** +- — (e.g. `np.ascontiguousarray` conversions, `overlap_batch` marshalling, FFI boundary) + +**Classification:** — : +- **A** = Python-adapter overhead dominates the SVAR2 gap → Task B Dataset wiring likely erases it. +- **B** = Rust hot-path / dense layout dominates → Task B must carry a kernel/layout fix. +- **C** = mix → quantify each share. +``` + +> Decision hint for classifying: the SVAR2 adapter does a long chain of `np.ascontiguousarray` copies + `overlap_batch` marshalling in `_svar2_source.py:_query`/`reconstruct` (lines 36–88). If those Python leaf frames dominate → A. If the dense-presence gather / codec decode dominates native → B. Report the split, don't force a single bucket. + +- [ ] **Step 6: Commit (scripts + notes; not the binary artifacts)** + +```bash +rtk git add tmp/svar2_mvp/e1_profile.sh docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E1 query-latency attribution — py-spy split + perf Rust detail + table" +``` + +> Do NOT commit `tmp/svar2_mvp/prof_out/` (flamegraph SVGs, speedscope JSON, perf.data are large binaries). If `git status` shows them, add `tmp/svar2_mvp/prof_out/` to `.gitignore` in this commit. + +--- + +## Task 4: E2 — subsample somatic + build per-S stores + +**Files:** +- Create: `tmp/svar2_mvp/e2_subsample.sh` +- Create: `tmp/svar2_mvp/e2_build.sbatch` +- Artifacts: `$W/somatic_s{1000,2000,4000,8000,16007}.{svar,svar2}` + sample lists + per-S variant counts. + +**Interfaces:** +- Consumes: `$W/gdc.chr21.norm.filt.bcf` (the somatic filtered BCF); `build_stores.py build(bcf, chrom, samples, out_prefix, ploidy)` (existing, `tmp/svar2_mvp/build_stores.py`). +- Produces: for each `S ∈ {1000, 2000, 4000, 8000, 16007}`, a subsampled BCF `$W/gdc.chr21.s${S}.bcf`, its `.svar`+`.svar2` at prefix `$W/somatic_s${S}`, and a recorded per-S variant count (subsampling drops sites monomorphic in the subset — record counts so curves are interpretable). S=16007 = the full cohort (reuse existing `somatic.*` — do not rebuild). + +- [ ] **Step 1: Write the subsample script** + +Create `tmp/svar2_mvp/e2_subsample.sh`: +```bash +#!/usr/bin/env bash +# E2: fixed-cohort (somatic) sample-count sweep. Build per-S subsampled BCFs. +set -euo pipefail +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SRC=$W/gdc.chr21.norm.filt.bcf +CHROM=chr21 +bcftools query -l "$SRC" > "$W/somatic.samples.txt" +TOTAL=$(wc -l < "$W/somatic.samples.txt") +echo "total somatic samples=$TOTAL" # expect 16007 +for S in 1000 2000 4000 8000; do + head -n "$S" "$W/somatic.samples.txt" > "$W/somatic.s${S}.list" + bcftools view -S "$W/somatic.s${S}.list" --force-samples -Ob \ + -o "$W/gdc.chr21.s${S}.bcf" "$SRC" + bcftools index -f "$W/gdc.chr21.s${S}.bcf" + N=$(bcftools view -H "$W/gdc.chr21.s${S}.bcf" | wc -l) + echo "S=$S variants=$N" | tee -a "$W/e2_variant_counts.txt" +done +# S=16007 is the full cohort: reuse existing somatic.* store; just record its count. +N=$(bcftools view -H "$SRC" | wc -l) +echo "S=16007 variants=$N" | tee -a "$W/e2_variant_counts.txt" +``` + +- [ ] **Step 2: Run subsampling on the node** + +Run: +```bash +chmod +x tmp/svar2_mvp/e2_subsample.sh +srun -p carter-compute --cpus-per-task=8 bash tmp/svar2_mvp/e2_subsample.sh +cat /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_variant_counts.txt +``` +Expected: `total somatic samples=16007`; five `S=... variants=...` lines with variant counts monotonically non-decreasing in S (mild — see spec caveat). Sub-BCFs and `.csi` indexes present. + +- [ ] **Step 3: Write the build array job** + +Create `tmp/svar2_mvp/e2_build.sbatch`: +```bash +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH --array=0-3 +#SBATCH -J e2build +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_build_%a.log +set -euo pipefail +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SIZES=(1000 2000 4000 8000) +S=${SIZES[$SLURM_ARRAY_TASK_ID]} +pixi run -e default python tmp/svar2_mvp/build_stores.py \ + "$W/gdc.chr21.s${S}.bcf" chr21 "$W/somatic_s${S}" +``` +> `build_stores.py` reads its own sample list via `bcftools query -l `, so the subsampled BCF fully determines the cohort. S=16007 is NOT in the array — it reuses the existing `$W/somatic.*` store. + +- [ ] **Step 4: Submit and wait for builds** + +Run: +```bash +rtk pixi run -e default sbatch tmp/svar2_mvp/e2_build.sbatch # note: sbatch itself needs no env, but rtk-wrapping is harmless +# poll: +squeue -u "$USER" -n e2build +``` +Expected: four array tasks queue; on completion each `e2_build_.log` ends with `built .../somatic_s.svar and .../somatic_s.svar2`. Verify: +```bash +ls -d /carter/users/dlaub/repos/for_loukik/svar2_mvp/somatic_s{1000,2000,4000,8000}.{svar,svar2} +``` +Expected: eight store dirs exist. + +- [ ] **Step 5: Commit the scripts + variant counts** + +```bash +cp /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_variant_counts.txt tmp/svar2_mvp/e2_variant_counts.txt +rtk git add tmp/svar2_mvp/e2_subsample.sh tmp/svar2_mvp/e2_build.sbatch tmp/svar2_mvp/e2_variant_counts.txt +rtk git commit -m "chore: E2 somatic sample-count sweep — subsample + per-S store builds" +``` + +--- + +## Task 5: E2 — hap-latency-vs-S benchmark + curves + +**Files:** +- Create: `tmp/svar2_mvp/e2_bench.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E2 section) + +**Interfaces:** +- Consumes: the per-S stores from Task 4 (prefix `$W/somatic_s${S}` for S<16007, `$W/somatic` for S=16007); the same warm-median timing shape as `prof_driver.py`. +- Produces: a TSV `tmp/svar2_mvp/prof_out/e2_curve.tsv` with columns `S variants svar1_hap_s svar2_hap_s`, and slopes (Δlatency/ΔS) for both backends written into the notes. + +- [ ] **Step 1: Write the sweep benchmark** + +Create `tmp/svar2_mvp/e2_bench.py`: +```python +"""E2: hap-latency vs sample-count S for both backends on the SOMATIC cohort. +Fixed dataset family (subsampled somatic), same 3 regions, warm, median N=5. + + python e2_bench.py > tmp/svar2_mvp/prof_out/e2_curve.tsv +""" +import time +from statistics import median + +import numpy as np +import polars as pl +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] +N = 5 +SIZES = [1000, 2000, 4000, 8000, 16007] + + +def prefix(S): + return f"{W}/somatic" if S == 16007 else f"{W}/somatic_s{S}" + + +def timed(fn, warmup=1): + for _ in range(warmup): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter() + fn() + ts.append(time.perf_counter() - t0) + return median(ts) + + +def variants_at(S): + # read the recorded count file written by e2_subsample.sh + for line in open(f"{W}/e2_variant_counts.txt"): + if line.startswith(f"S={S} "): + return int(line.split("variants=")[1]) + return -1 + + +def main(): + import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + ru, ro = np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + print("S\tvariants\tsvar1_hap_s\tsvar2_hap_s") + for S in SIZES: + p = prefix(S) + sv2 = SparseVar2(f"{p}.svar2") + src = SparseVar2Source(sv2) + n_s = sv2.n_samples + svar2 = timed(lambda: src.reconstruct(CHROM, REGIONS, ru, ro, + pad_char=ord("N"), shifts=None, output_length=-1)) + bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS]}) + gvl.write(f"{p}.gvl", bed, variants=f"{p}.svar", overwrite=True) + ds_hap = gvl.Dataset.open(f"{p}.gvl", reference=REF).with_seqs("haplotypes") + svar1 = timed(lambda: ds_hap[:len(REGIONS), :n_s]) + print(f"{S}\t{variants_at(S)}\t{svar1:.4f}\t{svar2:.4f}", flush=True) + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 2: Run the sweep on the node** + +Run: +```bash +mkdir -p tmp/svar2_mvp/prof_out +srun -p carter-compute --cpus-per-task=16 \ + pixi run -e default python tmp/svar2_mvp/e2_bench.py | tee tmp/svar2_mvp/prof_out/e2_curve.tsv +``` +Expected: a 5-row TSV; `svar2_hap_s` should track the full-cohort somatic value (~0.38) at S=16007 and both columns rise with S. Sanity: at S=16007 the two backends should be near the MVP parity point (~0.38 each). + +- [ ] **Step 3: Compute slopes and fill the E2 notes section** + +Compute least-squares slope per backend: +```bash +rtk pixi run -e default python -c " +import numpy as np +rows=[l.split() for l in open('tmp/svar2_mvp/prof_out/e2_curve.tsv').read().splitlines()[1:]] +S=np.array([int(r[0]) for r in rows]); a=np.array([float(r[2]) for r in rows]); b=np.array([float(r[3]) for r in rows]) +print('svar1 slope s/sample=%.3e'%np.polyfit(S,a,1)[0]) +print('svar2 slope s/sample=%.3e'%np.polyfit(S,b,1)[0]) +" +``` +Replace the `## E2` placeholder with the TSV rendered as a markdown table plus both slopes, and a one-line **decision output**: does SVAR2 latency rise *less steeply* with S than SVAR1 (confirming the MVP's declined hypothesis) or not? Cite the per-S variant counts as the interpretability caveat (subsampling drops monomorphic sites). + +- [ ] **Step 4: Commit** + +```bash +rtk git add tmp/svar2_mvp/e2_bench.py tmp/svar2_mvp/prof_out/e2_curve.tsv docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E2 same-cohort hap-latency-vs-S sweep + slopes" +``` + +--- + +## Task 6: E3 — dense-access layout probe (conditional) + +**Files:** +- Create: `tmp/svar2_mvp/e3_probe.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E3 section) + +**Interfaces:** +- Consumes: E1's classification (Task 3). **Gate:** run this task ONLY if E1 attributed significant native time to the dense-presence gather (classification B or C with the dense gather in the top Rust symbols). Otherwise, write "E3 skipped — E1 showed the SVAR2 gap is " and commit that. +- Consumes: `SparseVar2` (`n_samples`, `overlap_batch`), `SparseVar2Source.reconstruct`. +- Produces: two curves — SVAR2 hap latency vs (a) **region width** (fixed all-samples) and (b) proxy for **`n_dense_variants`** (dense count touched by widening the region toward the whole contig) — into `tmp/svar2_mvp/prof_out/e3.tsv`, and an estimated size of the "contig-wide stride + bit-by-bit read" cost. + +- [ ] **Step 1: Decide the gate** + +Read the E1 classification from the notes file. If it is **A** (Python-adapter overhead) or **native-but-not-dense-gather**, skip to Step 4 (write the skip note). If **B/C with dense gather hot**, proceed to Step 2. + +- [ ] **Step 2: Write the layout probe** + +Create `tmp/svar2_mvp/e3_probe.py`: +```python +"""E3: SVAR2 hap-latency sensitivity to region width and dense count. +Germline (high-AF -> large n_dense_variants) is the stress cohort. + + python e3_probe.py > tmp/svar2_mvp/prof_out/e3.tsv +""" +import time +from statistics import median + +import numpy as np +import pysam +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +N = 5 +START = 20_000_000 +WIDTHS = [200, 1_000, 5_000, 25_000, 100_000, 500_000] + + +def timed(fn): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter(); fn(); ts.append(time.perf_counter() - t0) + return median(ts) + + +def main(): + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + ru, ro = np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + sv2 = SparseVar2(f"{W}/germline.svar2") + src = SparseVar2Source(sv2) + print("width\tn_dense_in_region\thap_s") + for w in WIDTHS: + regs = [(START, START + w)] + d = sv2.overlap_batch(CHROM, [(START, START + w)]) + # dense variants actually spanned by this region (dense_range gives [lo,hi) per region) + dr = np.asarray(d["dense_range"]).reshape(-1, 2) + n_dense = int((dr[:, 1] - dr[:, 0]).sum()) + t = timed(lambda: src.reconstruct(CHROM, regs, ru, ro, + pad_char=ord("N"), shifts=None, output_length=-1)) + print(f"{w}\t{n_dense}\t{t:.4f}", flush=True) + + +if __name__ == "__main__": + main() +``` +> The dense-presence *stride* is contig-wide (`n_dense_variants`), but the *count read* per query scales with the dense variants spanned by the region — widening the region increases both the gather work and cache-line scatter. This probe measures latency vs that count. A full Rust microbench of `get_bit`-per-column vs word-parallel is out of scope for measurement; note it as the follow-up if the curve is steep. + +- [ ] **Step 3: Run the probe** + +Run: +```bash +srun -p carter-compute --cpus-per-task=16 \ + pixi run -e default python tmp/svar2_mvp/e3_probe.py | tee tmp/svar2_mvp/prof_out/e3.tsv +``` +Expected: latency rising with width and `n_dense_in_region`. A super-linear rise vs `n_dense_in_region` is evidence for the scatter/bit-by-bit cost. + +- [ ] **Step 4: Fill the E3 notes section (probe result OR skip note)** + +If run: replace `## E3` with the TSV as a table + a sentence on whether latency scales with `n_dense_in_region` (evidence for/against the contig-wide-stride + bit-by-bit cost) and a rough magnitude of the potential win from a region-local / word-parallel layout — input to how much layout work Task B carries. +If skipped: replace `## E3` with the one-line skip note citing E1's classification. + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e3_probe.py docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +# include the tsv only if the probe ran: +[ -f tmp/svar2_mvp/prof_out/e3.tsv ] && rtk git add tmp/svar2_mvp/prof_out/e3.tsv +rtk git commit -m "chore: E3 dense-access layout probe (or documented skip per E1)" +``` + +--- + +## Task 7: E4 — conversion phase breakdown (build side) + +**Files:** +- Create: `tmp/svar2_mvp/e4_convert_driver.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E4 section, part 1) + +**Interfaces:** +- Consumes: `genoray._core.run_conversion_pipeline(vcf_path, reference_path, chroms, output_dir, samples, chunk_size=25000, ploidy=2, max_threads=None, long_allele_capacity=8388608)` (confirmed signature); `GENORAY_SAMPLE_INTERVAL` env (genoray's built-in channel-fill/per-thread sampler — the only genoray env knob present); `split_folded.py` (Task 2). +- Produces: CLI `python e4_convert_driver.py ` that runs one conversion and prints `build_wall_s=`; plus a py-spy `--native` phase breakdown (htslib read/decompress vs encode vs Phase-2 merge) for a single build, written to the E4 notes section. + +- [ ] **Step 1: Write the conversion driver** + +Create `tmp/svar2_mvp/e4_convert_driver.py`: +```python +"""E4: run ONE single-contig svar2 conversion with a chosen max_threads. + python e4_convert_driver.py +Prints: build_wall_s= +Set GENORAY_SAMPLE_INTERVAL in the environment to enable genoray's sampler.""" +import sys +import time +import subprocess + +from genoray import _core + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + + +def main(bcf, chrom, out_prefix, max_threads): + samples = subprocess.run(["bcftools", "query", "-l", bcf], + capture_output=True, text=True, check=True).stdout.split() + t0 = time.perf_counter() + _core.run_conversion_pipeline( + bcf, REF, [chrom], f"{out_prefix}.svar2", samples, + 25_000, 2, int(max_threads), 8 * 1024 * 1024, + ) + print(f"build_wall_s={time.perf_counter() - t0:.2f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) +``` + +- [ ] **Step 2: Capture the phase breakdown with py-spy --native + genoray sampler** + +Use the germline single-contig BCF (~11 min full; use a subsampled BCF from Task 4 if you want a faster capture — e.g. `gdc.chr21.s1000.bcf` builds in a few minutes and exercises the same phases). Run on the node: +```bash +mkdir -p tmp/svar2_mvp/prof_out/e4 +GENORAY_SAMPLE_INTERVAL=1000 \ +srun -p carter-compute --cpus-per-task=16 \ + .pixi/envs/default/bin/py-spy record --native --rate 250 --format raw \ + -o tmp/svar2_mvp/prof_out/e4/convert.folded -- \ + pixi run -e default python tmp/svar2_mvp/e4_convert_driver.py \ + /carter/users/dlaub/repos/for_loukik/svar2_mvp/chr21.norm.filt.bcf chr21 \ + /tmp/e4_probe 8 +``` +Expected: a `.folded` file; py-spy sampled all Rust worker threads (htslib/encode/merge). genoray's sampler prints channel-fill / per-thread CPU% to stderr — capture it too (append `2> tmp/svar2_mvp/prof_out/e4/genoray_sampler.txt` if desired). + +- [ ] **Step 3: Bucket native frames into phases** + +Run the split, then bucket by symbol substring (approximate — regex over demangled names): +```bash +rtk pixi run -e default python tmp/svar2_mvp/split_folded.py tmp/svar2_mvp/prof_out/e4/convert.folded | tee tmp/svar2_mvp/prof_out/e4/split.txt +# rough phase buckets from the folded leaves: +rtk pixi run -e default python -c " +from collections import Counter +buckets=Counter() +for line in open('tmp/svar2_mvp/prof_out/e4/convert.folded'): + stack,_,cnt=line.rstrip().rpartition(' ') + try: n=int(cnt) + except ValueError: continue + leaf=stack.split(';')[-1].lower() + if any(k in leaf for k in ('bgzf','inflate','zlib','htslib','decompress','read')): b='read/decompress' + elif any(k in leaf for k in ('encode','bitgrid','codec','pack')): b='encode' + elif any(k in leaf for k in ('merge','dense_merge','transpose')): b='phase2-merge' + else: b='other' + buckets[b]+=n +tot=sum(buckets.values()) +for b,n in buckets.most_common(): print(f'{100*n/tot:5.1f}% {b}') +" | tee -a tmp/svar2_mvp/prof_out/e4/split.txt +``` +Expected: a percentage per phase; the MVP log's claim ("VCF read/decompress dominating") predicts `read/decompress` is the largest bucket. The `other` bucket catches un-symbolized/misc frames — if it dominates, note that symbol names didn't match the regex (genoray un-symbolized per Task 0) and lean on the genoray sampler output instead. + +- [ ] **Step 4: Write the E4 part-1 notes** + +Replace the `## E4` placeholder's first half with: the phase-bucket table, the genoray sampler's reported thread split (e.g. `1 concurrent chromosome | N HTSlib decompression threads`), and a one-line confirmation/refutation of "conversion is read-bound." + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e4_convert_driver.py docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E4.1 conversion phase breakdown (read/decompress vs encode vs merge)" +``` + +--- + +## Task 8: E4 — thread-split sweep + recommended policy + +**Files:** +- Create: `tmp/svar2_mvp/e4_sweep.sbatch` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E4 section, part 2) + +**Interfaces:** +- Consumes: `e4_convert_driver.py` (Task 7). The Python-exposed knob is `max_threads` (total pipeline threads); genoray internally derives the decompression-vs-executor/writer split from it and reports it in its log line. There is **no** finer env knob exposed (only `GENORAY_SAMPLE_INTERVAL`) — so this sweep varies `max_threads` (and total cores) and records genoray's *reported* internal split at each point. +- Produces: a table `max_threads → build_wall_s` (+ genoray's reported decompress/executor split) for a fixed single-contig input, and a recommended few-contig thread policy. + +- [ ] **Step 1: Pick the sweep input** + +Use a single-contig input that builds in a few minutes so the sweep is affordable: the somatic `S=1000` subsample (`$W/gdc.chr21.s1000.bcf`, from Task 4) OR full germline (`$W/chr21.norm.filt.bcf`, ~11 min each). Prefer the smaller input for a denser sweep; note the chosen input in the notes. Sweep `max_threads ∈ {2, 4, 8, 16}` (cap at node `nproc`). + +- [ ] **Step 2: Write the sweep job** + +Create `tmp/svar2_mvp/e4_sweep.sbatch`: +```bash +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH -J e4sweep +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e4_sweep.log +set -euo pipefail +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +BCF=$W/gdc.chr21.s1000.bcf # single-contig, fast; swap to chr21.norm.filt.bcf for germline +for T in 2 4 8 16; do + echo "=== max_threads=$T ===" + GENORAY_SAMPLE_INTERVAL=1000 pixi run -e default python tmp/svar2_mvp/e4_convert_driver.py \ + "$BCF" chr21 "/tmp/e4_sweep_t${T}" "$T" 2>&1 | grep -E "build_wall_s|concurrent chromosome|decompression" + rm -rf "/tmp/e4_sweep_t${T}.svar2" +done +``` + +- [ ] **Step 3: Submit and collect** + +Run: +```bash +rtk pixi run -e default sbatch tmp/svar2_mvp/e4_sweep.sbatch +squeue -u "$USER" -n e4sweep # wait for completion +cat /carter/users/dlaub/repos/for_loukik/svar2_mvp/e4_sweep.log +``` +Expected: four `max_threads=T` blocks, each with a `build_wall_s=` and genoray's reported thread-split line. Wall-clock should fall then plateau (or regress) as threads rise — the minimum identifies the read-bound sweet spot. The sweep runs sequentially in **one** allocation so all points share the same node, but a shared node still adds noise — if a point looks anomalous (non-monotonic in a way threads don't explain), re-run the whole `.sbatch` and prefer the run whose curve is smooth; note contention in the E4 section. + +- [ ] **Step 4: Fill the E4 part-2 notes + recommended policy** + +Replace the `## E4` placeholder's second half with the `max_threads → build_wall_s (+ reported split)` table and a concrete recommendation for **few-contig** jobs (e.g. "for single-contig input, max_threads=N minimizes wall-clock; the read-bound phase wants most threads on htslib decompress"). Add the cross-reference line: this feeds `genoray:docs/roadmap/architecture.md` → Open questions → *read-bound conversion / thread allocation*. + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e4_sweep.sbatch docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E4.2 conversion thread-split sweep + recommended few-contig policy" +``` + +--- + +## Task 9: Synthesis — two recommendations + cross-links + +**Files:** +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (Recommendations section) + +**Interfaces:** +- Consumes: the filled E1–E4 sections. +- Produces: the two concrete recommendations the spec requires, plus a decision on E3's layout-work sizing, with pointers back to the spec's Task B and the genoray architecture open question. + +- [ ] **Step 1: Write the two recommendations** + +Replace the `## Recommendations` placeholder with exactly two headed recommendations, each grounded in the tables above: + +1. **Where SVAR2 query latency actually goes → what Task B must include.** State the E1 classification (A/B/C) with the Python% vs native% numbers. If A: "Task B Dataset wiring likely erases the gap — no kernel/layout work needed." If B/C: name the hot Rust symbol(s) and cite E3 for the layout-win size, i.e. "Task B must carry a layout change worth ~." Reference the E2 slopes for the scales-with-S question (confirmed/refuted). +2. **Conversion thread-split policy.** State the E4 read-bound confirmation and the recommended `max_threads` / split for few-contig jobs, with the wall-clock delta between default and recommended. + +- [ ] **Step 2: Add cross-links** + +Add a short "Feeds" list: (a) the spec's Task B (Dataset wiring) — this notes file is the gate that decides its scope; (b) `genoray:docs/roadmap/architecture.md` → Open questions → read-bound conversion / thread allocation (E4). Note that split-by-contig layout remains unassessed (single-contig only) per the MVP notes' open question 3. + +- [ ] **Step 3: Verify the notes file is complete and self-consistent** + +Run (each alternative is a literal leftover-placeholder marker): +```bash +rtk grep -nF -e "filled by" -e "<...>" -e "TBD" docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +``` +Expected: **no matches** — every placeholder replaced with real numbers. If any remain, the corresponding experiment task's Step didn't write its result; go back and fill it. + +- [ ] **Step 4: Commit** + +```bash +rtk git add docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "docs: SVAR2 profiling results synthesis — latency attribution + thread-split recommendations" +``` + +--- + +## Notes on scope & honesty (carry into every task) + +- **No optimization.** If a hot path is obvious, record it — do not fix it. Task B and the thread rebalance are explicitly out of scope (spec "Out of scope"). +- **Profilers inflate wall-clock.** Always take the reported latency numbers from *unprofiled* runs (`prof_driver.py`/`e2_bench.py` timing), and use py-spy/perf only for *attribution* (the %-split and symbol ranks). +- **Degrade gracefully on genoray symbols.** If the genoray wheel rebuild (Task 0 Step 4) didn't land, say so in the notes and rely on py-spy `--native` module attribution + the genoray sampler for genoray-internal phases; gvl's own kernel symbolizes regardless. +- **Record dispersion honestly.** Median of N=5 with no CI — call parity "indistinguishable at this resolution," not "equal," matching the MVP notes' framing. From e587e2c0e3b586a981f42b6a099801ced0454fd8 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 21:56:47 -0700 Subject: [PATCH 016/108] chore: py-spy folded-stack Python/native self-time splitter Co-Authored-By: Claude Opus 4.8 --- tmp/svar2_mvp/split_folded.py | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tmp/svar2_mvp/split_folded.py diff --git a/tmp/svar2_mvp/split_folded.py b/tmp/svar2_mvp/split_folded.py new file mode 100644 index 00000000..3fd1f621 --- /dev/null +++ b/tmp/svar2_mvp/split_folded.py @@ -0,0 +1,45 @@ +"""Split a py-spy --format raw (folded) stack file into Python vs native +self-time by LEAF frame. A leaf frame is Python iff it contains '.py:'. + + python split_folded.py +""" +import sys +from collections import Counter + + +def is_python(frame: str) -> bool: + return ".py:" in frame or frame.endswith(".py") + + +def main(path): + py = nat = 0 + leaves = Counter() + classed = {} + with open(path) as fh: + for line in fh: + line = line.rstrip("\n") + if not line: + continue + stack, _, cnt = line.rpartition(" ") + try: + n = int(cnt) + except ValueError: + continue + leaf = stack.split(";")[-1] + leaves[leaf] += n + classed[leaf] = "python" if is_python(leaf) else "native" + if is_python(leaf): + py += n + else: + nat += n + tot = py + nat + if tot == 0: + print("no samples parsed"); return + print(f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}") + print("top-15 leaf frames (self-time):") + for leaf, n in leaves.most_common(15): + print(f" {100 * n / tot:5.1f}% [{classed[leaf]:6s}] {leaf}") + + +if __name__ == "__main__": + main(sys.argv[1]) From 3f0ef4392400adb7b1a507079801e113d0ecdeb3 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:07:03 -0700 Subject: [PATCH 017/108] =?UTF-8?q?chore:=20SVAR2=20profiling=20Task=200?= =?UTF-8?q?=20=E2=80=94=20env=20baseline,=20symbolized=20builds,=20notes?= =?UTF-8?q?=20skeleton?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session runs in a 2cpu/8GB carter-cn-04 alloc: light profiling runs on-node, heavy work via sbatch. gvl+genoray rebuilt --release with line-tables debug + frame pointers; perf --call-graph fp resolves genoray_core::query::overlap_batch to source-level Rust symbols. Plan: E1 perf switched dwarf->fp (dwarf overloaded the node). Co-Authored-By: Claude Opus 4.8 --- .../2026-07-03-svar2-profiling-results.md | 20 +++++++++++++++++++ .../2026-07-03-svar2-profiling-followup.md | 8 ++++++-- tmp/svar2_mvp/env_baseline.txt | 17 ++++++++++++++++ tmp/svar2_mvp/genoray_debug_build.sbatch | 13 ++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/notes/2026-07-03-svar2-profiling-results.md create mode 100644 tmp/svar2_mvp/env_baseline.txt create mode 100644 tmp/svar2_mvp/genoray_debug_build.sbatch diff --git a/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md b/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md new file mode 100644 index 00000000..6bde899d --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md @@ -0,0 +1,20 @@ +# SVAR2 Profiling Results (E1–E4) + +**Date:** 2026-07-03 · **Spec:** `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` +**Node/env baseline:** see `tmp/svar2_mvp/env_baseline.txt`. +**Symbolization:** gvl + genoray rebuilt `--release` with `debug=line-tables-only` + frame pointers. + +## E1 — Query-latency attribution +_(filled by Task 3)_ + +## E2 — Same-cohort sample sweep +_(filled by Task 5)_ + +## E3 — Dense-access layout probe +_(filled by Task 6; conditional on E1)_ + +## E4 — Conversion thread-allocation +_(filled by Task 8)_ + +## Recommendations +_(filled by Task 9)_ diff --git a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md index f058e4a8..61f3fe6f 100644 --- a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md +++ b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md @@ -16,7 +16,8 @@ Copied verbatim from the spec — every task's requirements implicitly include t - **Env:** `pixi run -e default` (the only installed env). **Python 3.10.20.** - **perf cannot symbolize Python frames on Python < 3.12.** This env is 3.10, so `perf` shows resolved **Rust/native** symbols but **opaque** Python frames (`_PyEval_EvalFrameDefault`). Use **py-spy** for the Python-vs-native split and Python hotspots; use **perf** only for Rust symbol detail once py-spy says native dominates. - **Profilers (confirmed present):** py-spy at `.pixi/envs/default/bin/py-spy`; perf at `/carter/users/dlaub/.pixi/bin/perf`. -- **Noise control:** compute node via `srun`/`sbatch -p carter-compute` (do **NOT** use `--exclusive` — Carter has only 3 compute nodes, always shared, so `--exclusive` won't schedule). The node is shared and noisy, so **absolute wall-clock is not comparable across allocations** — only *relative* comparisons **within one short allocation on the same hardware** are valid (matches the prior perf-gate lesson: gate on same-session before/after, not absolute time). Therefore: **run both backends of any comparison inside a single `srun`** (back-to-back on the same node), warm caches, **median of N≥5**, record CPU governor / turbo. CPU governor on the target nodes is `performance` — record the actual value observed on the node you land on. Keep the MVP workload (same 3 chr21 regions, all samples) for continuity. +- **Compute execution reality (discovered at run start):** this session already runs **inside** a small interactive SLURM allocation on `carter-cn-04` (**2 CPUs, 8 GB RAM**, 14-day walltime, governor=performance, paranoid=2). Therefore: **run light single-threaded profiling directly on this node** — do **NOT** wrap it in `srun` (an in-allocation `srun` makes a constrained 2-CPU *step* and fails with "More processors requested than permitted" at >2 cpus). Route **heavy multi-core / large-RAM work through fresh `sbatch` jobs** (independent of this allocation) sized to the big partition nodes (carter-cn-02/03/04 = 96–128 CPU, 476–953 GB). Everywhere a task below says `srun … `, execute `` **directly on the node** instead (the 3-region E1/E2-bench/E3 workloads are single-threaded and fit in 8 GB). Keep `sbatch` for E2 store builds and the E4 thread sweep, adding explicit `--cpus-per-task` and `--mem`. +- **Noise control:** compute node via `sbatch -p carter-compute` for heavy jobs; do **NOT** use `--exclusive` — Carter has only 3 compute nodes, always shared, so `--exclusive` won't schedule. The node is shared and noisy, so **absolute wall-clock is not comparable across allocations** — only *relative* comparisons **within one short allocation on the same hardware** are valid (matches the prior perf-gate lesson: gate on same-session before/after, not absolute time). Therefore: **run both backends of any comparison inside a single `srun`** (back-to-back on the same node), warm caches, **median of N≥5**, record CPU governor / turbo. CPU governor on the target nodes is `performance` — record the actual value observed on the node you land on. Keep the MVP workload (same 3 chr21 regions, all samples) for continuity. - **MVP workload regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)`. - **Stores (already built), `$W`:** `/carter/users/dlaub/repos/for_loukik/svar2_mvp` — `{germline,somatic}.{svar,svar2,gvl}` + `{chr21,gdc.chr21}.norm.filt.bcf`. germline = 3202 samples (1000G, high-AF/dense); somatic = 16007 samples (GDC, rare/sparse). - **Reference FASTA:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. @@ -388,7 +389,10 @@ run_pyspy () { # backend cohort run_perf () { # backend cohort (svar2 only) local b=$1 c=$2 tag="${1}_${2}" - $PERF record -g --call-graph dwarf -o "$OUT/${tag}.perf.data" -- \ + # Use frame-pointer call graph (extensions were built with -C force-frame-pointers=yes). + # DWARF call-graph on this workload produced a 1GB perf.data and overloaded the shared node; + # fp gives clean source-level Rust symbols at ~1MB. Lower -F to keep it light on the shared node. + $PERF record -g --call-graph fp -F 199 -o "$OUT/${tag}.perf.data" -- \ pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" echo "== perf top symbols $tag ==" | tee -a "$OUT/perf_top.txt" $PERF report --stdio -i "$OUT/${tag}.perf.data" --sort=overhead,symbol -g none 2>/dev/null \ diff --git a/tmp/svar2_mvp/env_baseline.txt b/tmp/svar2_mvp/env_baseline.txt new file mode 100644 index 00000000..10945a84 --- /dev/null +++ b/tmp/svar2_mvp/env_baseline.txt @@ -0,0 +1,17 @@ +# Captured 2026-07-03T21:54:02-07:00 +# This Claude session runs INSIDE an interactive SLURM allocation: +host=carter-cn-04 +node_total_cpus=128 +node_total_mem_MB=953674 +alloc_cpus=2 # SLURM_CPUS_ON_NODE (this session's allocation) +alloc_mem_MB=8192 # SLURM_MEM_PER_NODE +alloc_walltime=14-00:00:00 +partition=carter-compute +qos=normal # no MaxTRESPU cap +governor=performance +turbo_no_turbo=NA +paranoid=2 # perf call-graph works, no sudo +# CONSEQUENCE: light single-threaded profiling (E1 3-region workload) runs DIRECTLY on +# this node (no srun — srun makes a constrained 2-cpu step and fails at >2 cpus). +# Heavy multi-core / large-RAM work (E2 builds, E4 thread sweep) goes via fresh sbatch +# jobs sized to the big nodes (96-128 cpu, 476-953 GB). diff --git a/tmp/svar2_mvp/genoray_debug_build.sbatch b/tmp/svar2_mvp/genoray_debug_build.sbatch new file mode 100644 index 00000000..8984e29c --- /dev/null +++ b/tmp/svar2_mvp/genoray_debug_build.sbatch @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH --mem=32G +#SBATCH -J genoray_dbg +#SBATCH -o /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/genoray_debug_build.log +set -euo pipefail +cd /carter/users/dlaub/projects/genoray +CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ +RUSTFLAGS="-C force-frame-pointers=yes" \ +pixi run -e py310 maturin build --release +echo "WHEEL_BUILD_DONE" +ls -t target/wheels/genoray-2.15.0-*.whl | head -1 From f82198256578ea0663b96843bea6c4737f97aeb7 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:09:08 -0700 Subject: [PATCH 018/108] chore: E2 somatic sample-sweep subsample+build sbatch chain (controller-driven) Submitted subsample(12064665)->build array(12064666, afterok). Builds .svar+.svar2 for S in {1000,2000,4000,8000}; S=16007 reuses existing somatic.* store. Run off-node (fresh sbatch, 16cpu/64G) to keep the 2-cpu profiling node free. Co-Authored-By: Claude Opus 4.8 --- tmp/svar2_mvp/e2_build.sbatch | 16 ++++++++++++++++ tmp/svar2_mvp/e2_subsample.sbatch | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tmp/svar2_mvp/e2_build.sbatch create mode 100644 tmp/svar2_mvp/e2_subsample.sbatch diff --git a/tmp/svar2_mvp/e2_build.sbatch b/tmp/svar2_mvp/e2_build.sbatch new file mode 100644 index 00000000..b9179d31 --- /dev/null +++ b/tmp/svar2_mvp/e2_build.sbatch @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G +#SBATCH --array=0-3 +#SBATCH -J e2build +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_build_%a.log +set -euo pipefail +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SIZES=(1000 2000 4000 8000) +S=${SIZES[$SLURM_ARRAY_TASK_ID]} +echo "building S=$S host=$(hostname) cpus=$SLURM_CPUS_ON_NODE" +pixi run -e default python tmp/svar2_mvp/build_stores.py \ + "$W/gdc.chr21.s${S}.bcf" chr21 "$W/somatic_s${S}" +echo "E2_BUILD_DONE S=$S" diff --git a/tmp/svar2_mvp/e2_subsample.sbatch b/tmp/svar2_mvp/e2_subsample.sbatch new file mode 100644 index 00000000..74f7e137 --- /dev/null +++ b/tmp/svar2_mvp/e2_subsample.sbatch @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=8 +#SBATCH --mem=16G +#SBATCH -J e2_subsample +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_subsample.log +set -euo pipefail +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SRC=$W/gdc.chr21.norm.filt.bcf +bcftools query -l "$SRC" > "$W/somatic.samples.txt" +TOTAL=$(wc -l < "$W/somatic.samples.txt") +echo "total somatic samples=$TOTAL" +: > "$W/e2_variant_counts.txt" +for S in 1000 2000 4000 8000; do + head -n "$S" "$W/somatic.samples.txt" > "$W/somatic.s${S}.list" + bcftools view -S "$W/somatic.s${S}.list" --force-samples --threads 8 -Ob \ + -o "$W/gdc.chr21.s${S}.bcf" "$SRC" + bcftools index -f "$W/gdc.chr21.s${S}.bcf" + N=$(bcftools view -H "$W/gdc.chr21.s${S}.bcf" | wc -l) + echo "S=$S variants=$N" | tee -a "$W/e2_variant_counts.txt" +done +N=$(bcftools view -H "$SRC" | wc -l) +echo "S=16007 variants=$N" | tee -a "$W/e2_variant_counts.txt" +echo "E2_SUBSAMPLE_DONE" From e41ec71f43e9f77ac41498c562224e83c4d97c34 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:12:10 -0700 Subject: [PATCH 019/108] chore: E1 single-path profiling driver for SVAR1/SVAR2 hap query Small warm-loop CLI (tmp/svar2_mvp/prof_driver.py) that exercises exactly one hap-reconstruction path per invocation so perf/py-spy attribute time to that path alone: SVAR1 via gvl.Dataset.open(...).with_seqs("haplotypes") or SVAR2 via SparseVar2Source.reconstruct. For svar1 the 3-region .gvl is written once before the timed loop. Co-Authored-By: Claude Opus 4.8 --- tmp/svar2_mvp/prof_driver.py | 66 ++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tmp/svar2_mvp/prof_driver.py diff --git a/tmp/svar2_mvp/prof_driver.py b/tmp/svar2_mvp/prof_driver.py new file mode 100644 index 00000000..060526a0 --- /dev/null +++ b/tmp/svar2_mvp/prof_driver.py @@ -0,0 +1,66 @@ +"""E1 single-path profiling driver. Exercises ONE code path in a warm loop so +py-spy/perf attribute time to that path only. + + python prof_driver.py + +Prints: per_call_s= +For svar1, the 3-region .gvl is written ONCE before the loop (we profile the +query, not gvl.write).""" +import sys +import time + +import numpy as np + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] + + +def _ref(): + import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + return np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + + +def make_svar2(cohort): + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + src = SparseVar2Source(SparseVar2(f"{W}/{cohort}.svar2")) + ru, ro = _ref() + + def call(): + src.reconstruct(CHROM, REGIONS, ru, ro, pad_char=ord("N"), + shifts=None, output_length=-1) + return call + + +def make_svar1(cohort): + import polars as pl + import genvarloader as gvl + from genoray import SparseVar2 + n_s = SparseVar2(f"{W}/{cohort}.svar2").n_samples + bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS]}) + ds_path = f"{W}/{cohort}.gvl" + gvl.write(ds_path, bed, variants=f"{W}/{cohort}.svar", overwrite=True) # ONCE + ds_hap = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + + def call(): + ds_hap[:len(REGIONS), :n_s] + return call + + +def main(backend, cohort, K): + call = {"svar1": make_svar1, "svar2": make_svar2}[backend](cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + dt = time.perf_counter() - t0 + print(f"per_call_s={dt / K:.4f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) From 595b952f8648d0f06f9cdd03e8dd2f003f4044c3 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:48:25 -0700 Subject: [PATCH 020/108] docs: SVAR2 Dataset wiring design spec (cache overlap results like SVAR1) Grounded by E1 profiling: SVAR2 adapter is ~80% genoray SearchTree::build (rebuilt per overlap_batch). Design mirrors SVAR1's write-time search cache: genoray gains find_ranges/gather_ranges/read_ranges (samples=-aware) split; gvl caches compact ranges + Svar2Link, gathers tree-free at read. Haps+tracks, byte-identical parity. Co-Authored-By: Claude Opus 4.8 --- .../2026-07-03-svar2-dataset-wiring-design.md | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md diff --git a/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md b/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md new file mode 100644 index 00000000..63b07b9e --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md @@ -0,0 +1,198 @@ +# SVAR2 Dataset Wiring — Design Spec + +> **Purpose:** wire the SVAR2 format into the gvl `Dataset` the way SVAR1 already is — +> by caching genoray's interval-search result at `gvl.write` time and replaying it at read +> time — so SVAR2 haplotype/track reads stop paying a per-query search-tree rebuild. This is +> the deferred **Task B** (`TODO(svar2-dataset-dispatch)` in `_svar2_source.py`), now +> justified and shaped by the E1 profiling result below. + +**Date:** 2026-07-03 · **Depends on:** the M6b SVAR2 kernels (`reconstruct_haplotypes_from_svar2`, +`shift_and_realign_tracks_from_svar2`, already built + parity-validated vs the genoray `decode` +oracle), the `SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`), and the +E1 profiling result (`docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md`). **Two +deliverables:** a genoray PR (search/gather split) then a gvl PR (write + read wiring). + +## Problem & evidence (E1) + +The benchmarked SVAR2 path is the **raw `SparseVar2Source` adapter**, which calls genoray +`SparseVar2.overlap_batch` **live on every query**. E1 profiled it with `perf --call-graph fp` +(py-spy is unusable on Carter compute nodes — `ptrace_scope=2`), attributing time by DSO on the +3-region × all-samples chr21 workload: + +| path | native genoray `_core.so` | gvl kernel | numpy conv | python interp | +| --- | --- | --- | --- | --- | +| **svar2 germline** (3202) | **88.3%** (68.6% `SearchTree::build` + 14.3% `overlap_batch`) | 1.1% | 0.1% | 3.9% | +| **svar2 somatic** (16007) | **78.5%** (60.3% `SearchTree::build`) | 0.7% | 0.2% | 4.8% | +| svar1 germline | 0% | 56.9% (`get_diffs_sparse` + `reconstruct…_from_sparse`) | 20.4% | 7.3% | +| svar1 somatic | 0% | 10.2% (rest: genotype IO / ZSTD decompress + page faults) | 24.2% | 12.1% | + +**Conclusion:** SVAR2 adapter latency is **neither Python-adapter overhead (≤5%) nor the gvl +reconstruct kernel (~1%) nor numpy conversions (~0.1%)** — it is genoray **rebuilding interval +search trees on every `overlap_batch` call**. In genoray's source: `DenseUnion::overlap` builds a +fresh `SearchTree::new(&self.positions)` **per region** (`src/query.rs:288`, comment "one per region +in a batch") over the contig-wide dense union, and `vk_slice → spine::gather_keys` builds one **per +hap** (`src/spine.rs:48`). SVAR1 does not pay this: `gvl.write` calls genoray's **search-only** +`_find_starts_ends` once and caches a compact `(2, R, S, P)` offsets memmap +(`_write_from_svar` in `python/genvarloader/_dataset/_write.py:961`); reads then **slice the shared +`.svar` store directly** by those offsets, with no search. **This spec gives SVAR2 the same +write-time cache** — the fix E1 points to. (The earlier "dense-gather layout" hypothesis, E3, is +**moot**: the gvl kernel where that gather lives is ~1% of time; the cost is the tree build.) + +## Approach — mirror SVAR1 + +Split genoray's fused `overlap_batch` into a **search-only** step and a **tree-free gather** step, +mirroring SparseVar's `_find_starts_ends` / `read_ranges` pair. `gvl.write` runs the search once and +caches the compact result into `.gvl`; `Dataset` reads replay the cache through the gather step and +the existing SVAR2 kernels. The bulk variant data stays in the compressed `.svar2`, referenced by a +`Svar2Link` — so the cache is O(offsets) and **SVAR2's on-disk size advantage (1.46–5.67× smaller +than `.svar`, established in the MVP benchmark) is preserved**, not materialized away. + +Unlike SVAR1 (whose per-hap variants are a contiguous slice, so reads slice the shared store with no +genoray call), SVAR2's dense-presence bits are **computed** from the class tables (the `carried` +tests), not sliceable. So SVAR2's read path makes one genoray gather call per read — but with **no +interval search**, which is the entire cost E1 identified. + +## Component A — genoray API (dependency; separate genoray PR) + +Refactor `overlap_batch` into three public methods on `SparseVar2`, mirroring SparseVar's naming and +all accepting a `samples=` subset (like every SparseVar range method). Signatures follow SparseVar's +`_find_starts_ends` / `read_ranges` exactly where analogous: + +- **`find_ranges(contig, starts, ends, samples=None, out=None) → ranges`** — *search-only*, the + analog of `SparseVar._find_starts_ends`. Builds each search tree **≤ once per contig** and returns + the compact bundle needed to gather later: per-region `dense_range` `(R, 2)` and per-hap var_key + column offsets (no gathered `KeyRef`s, no presence bits). `samples=` restricts which samples' + offsets are computed; `out=` writes into a preallocated memmap so `gvl.write` streams the cache + straight to disk (exactly as `_find_starts_ends(..., out=out)` does for SVAR1). + +- **`gather_ranges(contig, ranges, samples=None) → payload`** — the *gather* step. Consumes a + `find_ranges` bundle (the **cached ranges**, not `starts/ends`) and therefore performs **no + interval search**: it slices var_key from the shared `.svar2` store and computes dense-presence + bits tree-free, returning the full reconstruct payload dict (`vk_pos`, `vk_key`, `vk_off`, + `dense_pos`, `dense_key`, `dense_range`, `dense_present`, `dense_present_off`, `lut_bytes`, + `lut_off`) — the exact input shape the SVAR2 kernels already accept. `samples=` restricts which + samples are materialized (cache all at write, read any subset). + +- **`read_ranges(contig, starts, ends, samples=None) → payload`** — the public **fused** API, exactly + like `SparseVar.read_ranges`. A thin wrapper: + `gather_ranges(contig, find_ranges(contig, starts, ends, samples), samples)`. It replaces + `overlap_batch`'s role for live/uncached queries and serves as the parity oracle. (`overlap_batch` + may be kept as a deprecated alias or removed — genoray maintainer's call.) + +**Contract (byte-identical parity):** for any `contig, starts, ends, samples`, +`reconstruct(read_ranges(...))` ≡ `reconstruct(gather_ranges(find_ranges(...)))` ≡ +`reconstruct(overlap_batch(...))` ≡ genoray `decode` oracle, byte-for-byte. gvl treats the +`find_ranges` bundle as an **opaque** array bundle — it persists and replays it, with no gvl coupling +to genoray's internal layout (the same way the adapter treats the `overlap_batch` dict today). + +## Component B — gvl write (`_write.py`, new `_write_from_svar2`, new `_svar2_link.py`) + +Mirror the SVAR1 write path: + +- **Detect `.svar2`.** In `write()` variant-source dispatch (`_write.py:~225`, alongside the existing + `.svar` branch), a directory with suffix `.svar2` → `SparseVar2(dir)`. Add a `SparseVar2` branch to + the genotype-writing dispatch (`_write.py:~325`, next to `isinstance(variants, SparseVar)`). + +- **`_write_from_svar2(path, bed, svar2, samples, extend_to_length)`** mirrors `_write_from_svar` + (`_write.py:961`): allocate a memmap in `/genotypes/`, and per contig call + `svar2.find_ranges(c, df["chromStart"], df["chromEnd"], samples=samples, out=out)` to stream the + compact ranges cache to disk. Write `/genotypes/svar2_meta.json` (bundle shapes/dtypes) and + a `Svar2Link` into `metadata.json`. Set `metadata["ploidy"] = svar2.ploidy`. + +- **`_svar2_link.py`** mirrors `_svar_link.py`: a `Svar2Link` pydantic model + (`relative_path`, `absolute_path`, `fingerprint`) + `_resolve_svar2(gvl_path, link, override)` and + `_verify_fingerprint(...)`. Fingerprint the `.svar2` on its stable identity (e.g. `n_variants` from + its index + a byte count of a canonical store file), analogous to `SvarFingerprint`. + +- **Reject unsupported variants** (symbolic/breakend ALTs) as SVAR1 does; upstream normalization + (genoray roadmap M13, `-V other,bnd`) already filters these in the MVP build scripts. + +## Component C — gvl read (`_open.py`, `_haps.py`, `_reconstruct.py`, `_tracks.py`; refactor `_svar2_source.py`) + +- **`Dataset.open`** resolves + fingerprints the `Svar2Link` (mirroring `_resolve_svar` / + `_verify_fingerprint`, with a `svar2=` override on `open` paralleling `svar=`), and holds a + `SparseVar2` handle plus the memmapped cached-ranges bundle. + +- **Haplotypes.** The `Haps` reconstruction path routes SVAR2 datasets to: load the cached ranges for + the requested `(region, sample)` block → `svar2.gather_ranges(contig, ranges_block, samples=block)` + → `reconstruct_haplotypes_from_svar2(...)` → `Ragged` haps. This retires + `TODO(svar2-dataset-dispatch)`. A dataset carries a source discriminant (svar2_link present) that + selects the SVAR2 reconstructor over the SVAR1 one. + +- **Tracks.** `with_tracks` / the track re-aligner routes to `shift_and_realign_tracks_from_svar2` + via the **same** cached ranges + `gather_ranges` — the cache is written once and serves both + haplotype and track reconstruction. + +- **Refactor `_svar2_source.py`.** Replace the adapter's live `overlap_batch` call with the + cached-ranges + `gather_ranges` path (or fold its kernel-marshalling into `Haps`). The + region→(R·S) expansion and `ascontiguousarray` marshalling already there stay valid; only the data + source changes from live query to cache+gather. + +## Cache format (`.gvl/genotypes/`) + +- The compact `find_ranges` bundle, region/sample-sharded to match the dataset layout (per-region + `dense_range` + per-hap var_key offsets), stored as memmaps written in place via `out=`. + `svar2_meta.json` records shapes/dtypes (mirrors SVAR1's `svar_meta.json`). `Svar2Link` → + shared `.svar2`. +- **Size:** O(offsets) — same order as SVAR1's `offsets.npy`, **not** the ~1.8 MB/3-region + `overlap_batch` payload. The bulk (var_key positions/keys, dense class tables, LUT) stays in the + compressed `.svar2`. + +## Data flow + +- **WRITE:** `bed + .svar2` → per contig `find_ranges(..., out=memmap)` (search once) → compact ranges + cache in `.gvl/genotypes/` + `Svar2Link` + `svar2_meta.json`. +- **READ:** `dataset[regions, samples]` → load cached ranges block → `gather_ranges` (tree-free) → + `reconstruct_haplotypes_from_svar2` / `shift_and_realign_tracks_from_svar2` → haps / tracks. **No + interval search at read.** + +## Parity & testing (the contract) + +- **Byte-identical:** cached-path reconstruct ≡ live `read_ranges`/`overlap_batch` reconstruct ≡ + `decode` oracle, on the M6b matrix (SNP / INS / DEL × samples × ploids) + real chr21 germline & + somatic stores. Track re-alignment matched the same way. +- **Additive guarantee:** the SVAR1 path is byte-unchanged — full SVAR1 regression suite green + (`pixi run -e dev pytest tests -q`; and `cargo test` for the kernels). Follows the rust-migration + byte-identical parity contract and the numba-oracle-bug policy (if the cached path and a numba + oracle disagree, check whether numba is the buggy one before "fixing" the new path). +- **Perf verification (same-session, shared-node caveat):** a warm SVAR2 `Dataset` read no longer + shows `SearchTree::build` — the perf DSO split flips from ~80% genoray to gvl-kernel-bound, like + SVAR1. Report as a relative before/after within one allocation (absolute wall-clock is not + comparable across allocations on the shared Carter nodes). +- **Docs/roadmaps:** update the genoray roadmap (search/gather split + read-bound conversion open + question) and `docs/roadmaps/rust-migration.md`; update user-facing docs for `.svar2` as a `write` + variant source — `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, + `README.md` (per the repo's docs-audit + skill-maintenance gates), and keep `api.md` in sync with + any new `__all__` symbols. + +## Out of scope + +- SVAR2 **variants** and **annotated** output modes (this spec covers haplotypes + tracks only; the + same cache extends to them later). +- The remaining profiling experiments **E2** (same-cohort sample-count sweep) and **E4** (conversion + build-thread allocation) — independent of this fix; parked, resumable from the profiling plan. + **E3** (dense-access layout probe) is **dropped** — E1 established the gvl dense-gather is ~1% of + time, not the hot path. +- Any change to the on-disk `.svar2` format itself (that is genoray's; this spec only adds the + search/gather split and the gvl-side cache). + +## Deliverables & sequencing + +1. **genoray PR** — `find_ranges` / `gather_ranges` / `read_ranges` split with `samples=`, parity + tests vs the current `overlap_batch`, release a wheel. (Crate/wheel release gate per the existing + SVAR2 dev-wiring notes.) +2. **gvl PR** — `_write_from_svar2` + `_svar2_link.py` (write) and the `_open.py` / `_haps.py` / + `_reconstruct.py` / `_tracks.py` dispatch + `_svar2_source.py` refactor (read), targeting the + genoray API from (1); byte-identical parity + docs/roadmap updates. + +## Open questions / risks + +- **`find_ranges` bundle contents:** the exact compact fields (are per-hap var_key offsets one range + per hap, or separate SNP + indel channel ranges?) are a genoray-internal detail settled in the + genoray PR; the gvl side stays agnostic (opaque persist/replay), so this does not block the gvl + design. +- **`gather_ranges` presence-bit cost at read:** the per-hap `carried` tests over `dense[ds..de]` + remain at read (they are cheap, no tree) — confirm in the perf verification they don't become the + new hot path (E1 already shows this work is a small fraction of the gvl-side once the tree is gone). +- **Format version:** adding `svar2_link` to `metadata.json` is additive; confirm + `_check_dataset_format_version` tolerates it and that a bump (if any) is handled by `_migrate.py`. From c7266d66c08ed82e151cc62f977a2258f6ec843a Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:50:15 -0700 Subject: [PATCH 021/108] chore: E1 perf attribution tooling (perf-only DSO split) + park profiling plan py-spy unusable on Carter (ptrace_scope=2); E1 done via perf --call-graph fp DSO buckets. E2/E4 cancelled per user pivot to the SVAR2 Dataset-wiring design. prof_out/ binaries gitignored. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + tmp/svar2_mvp/e1_bucket_dso.py | 36 +++++++++++++++++++++++ tmp/svar2_mvp/e1_profile.sh | 53 ++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tmp/svar2_mvp/e1_bucket_dso.py create mode 100755 tmp/svar2_mvp/e1_profile.sh diff --git a/.gitignore b/.gitignore index 2e7ef6bd..a6ea3ea9 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,4 @@ tests/benchmarks/profiling/*.speedscope.json tests/benchmarks/profiling/*.memray.bin tests/benchmarks/profiling/*.flamegraph.html tests/benchmarks/profiling/*.perf.data +tmp/svar2_mvp/prof_out/ diff --git a/tmp/svar2_mvp/e1_bucket_dso.py b/tmp/svar2_mvp/e1_bucket_dso.py new file mode 100644 index 00000000..3d9ce852 --- /dev/null +++ b/tmp/svar2_mvp/e1_bucket_dso.py @@ -0,0 +1,36 @@ +"""Bucket a `perf report --sort=dso --no-children --stdio` self-time dump into the +E1 attribution classes. Reads that dump on stdin. + + report --stdio --sort=dso --no-children -i data.perf | python e1_bucket_dso.py +""" +import re +import sys + +buckets = {"native-gvl": 0.0, "native-genoray": 0.0, "numpy-conv": 0.0, + "python-interp": 0.0, "other": 0.0} +for line in sys.stdin: + if line.lstrip().startswith("#") or not line.strip(): + continue + m = re.match(r"\s*([0-9.]+)%\s+(.*)", line) + if not m: + continue + pct = float(m.group(1)) + dso = m.group(2).strip().lower() + if "genvarloader" in dso: + b = "native-gvl" + elif "genoray" in dso or "_core.cpython" in dso: + b = "native-genoray" + elif "multiarray" in dso or "umath" in dso or "/numpy" in dso: + b = "numpy-conv" + elif dso.startswith("python") or "libpython" in dso: + b = "python-interp" + else: + b = "other" + buckets[b] += pct + +tot = sum(buckets.values()) or 1.0 +for k in ("native-gvl", "native-genoray", "numpy-conv", "python-interp", "other"): + print(f" {buckets[k]:6.1f}% {k}") +nat = buckets["native-gvl"] + buckets["native-genoray"] +print(f" ---- native(gvl+genoray)={nat:.1f}% numpy-conv={buckets['numpy-conv']:.1f}% " + f"python-interp={buckets['python-interp']:.1f}% (sum {tot:.1f}%)") diff --git a/tmp/svar2_mvp/e1_profile.sh b/tmp/svar2_mvp/e1_profile.sh new file mode 100755 index 00000000..3478b6a4 --- /dev/null +++ b/tmp/svar2_mvp/e1_profile.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# E1: per-(backend x cohort) query-latency attribution via perf ONLY. +# py-spy is unusable here (ptrace_scope=2, no sudo). perf works (paranoid=2, uses perf_event). +# +# Profile the env's python DIRECTLY (.pixi/envs/default/bin/python) — NOT via `pixi run`: +# the pixi launcher otherwise eats ~60% of samples, and the extensions import fine standalone +# (RPATH handles deps). Large K so the steady-state reconstruct loop drowns import/startup +# (genvarloader pulls in torch — a heavy one-time import). +# +# Python frames are opaque on 3.10, so the split is at the DSO level (self-time): +# gvl genvarloader.abi3.so + genoray _core.so = native Rust hot path +# numpy _multiarray_umath*.so = conversion/ascontiguousarray overhead +# python3.10 / libpython = interpreter/orchestration +# Frame-pointer call graph (built with -C force-frame-pointers=yes); DWARF overloads the node. +# Runs DIRECTLY on the current 2-cpu carter-cn-04 allocation (no srun). +# NOTE: no `pipefail` — the `| head` truncations send SIGPIPE (141) upstream to perf report, +# which under pipefail+set-e would abort the whole sweep after the first combo. +set -eu +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/e1 +mkdir -p "$OUT" +PERF=/carter/users/dlaub/.pixi/bin/perf +PY=.pixi/envs/default/bin/python +TARGET_S=60 # ~60s hot loop per capture so startup is negligible +FREQ=199 + +probe_K () { # backend cohort -> K sized to ~TARGET_S + local per + per=$("$PY" tmp/svar2_mvp/prof_driver.py "$1" "$2" 5 | sed 's/per_call_s=//') + "$PY" -c "import math; print(max(20, math.ceil($TARGET_S/max(float('$per'),1e-4))))" +} + +: > "$OUT/dso_split.txt"; : > "$OUT/perf_top.txt"; : > "$OUT/callgraph.txt"; : > "$OUT/K_used.txt" + +for b in svar2 svar1; do for c in germline somatic; do + tag="${b}_${c}" + K=$(probe_K "$b" "$c"); echo "$tag K=$K" | tee -a "$OUT/K_used.txt" + $PERF record -g --call-graph fp -F $FREQ -o "$OUT/${tag}.perf.data" -- \ + "$PY" tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" >/dev/null 2>&1 + echo "== DSO split $tag ==" | tee -a "$OUT/dso_split.txt" + $PERF report --stdio --sort=dso --no-children -g none -i "$OUT/${tag}.perf.data" 2>/dev/null \ + | "$PY" tmp/svar2_mvp/e1_bucket_dso.py | tee -a "$OUT/dso_split.txt" + echo "== top self-time symbols $tag ==" | tee -a "$OUT/perf_top.txt" + $PERF report --stdio --sort=symbol --no-children -g none -i "$OUT/${tag}.perf.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -15 | tee -a "$OUT/perf_top.txt" + # call-graph for the svar2 paths (shows e.g. SearchTree::build <- overlap_batch) + if [ "$b" = svar2 ]; then + echo "== call graph $tag (top) ==" >> "$OUT/callgraph.txt" + $PERF report --stdio --sort=overhead,symbol -i "$OUT/${tag}.perf.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -40 >> "$OUT/callgraph.txt" + fi +done; done +echo "E1_PROFILE_DONE dso -> $OUT/dso_split.txt symbols -> $OUT/perf_top.txt" From 03961eb647b4397cfbf35bfbeda894c2806473de Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 3 Jul 2026 22:50:26 -0700 Subject: [PATCH 022/108] docs: fold perf-only/ptrace + compute-reality findings into parked profiling plan Co-Authored-By: Claude Opus 4.8 --- .../superpowers/plans/2026-07-03-svar2-profiling-followup.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md index 61f3fe6f..8ccd96a6 100644 --- a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md +++ b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md @@ -14,8 +14,9 @@ Copied verbatim from the spec — every task's requirements implicitly include t - **This is measurement only.** No optimization, no Dataset wiring (Task B), no dense-layout change, no conversion rebalance. The output is *numbers that decide whether/how to do those*. - **Env:** `pixi run -e default` (the only installed env). **Python 3.10.20.** -- **perf cannot symbolize Python frames on Python < 3.12.** This env is 3.10, so `perf` shows resolved **Rust/native** symbols but **opaque** Python frames (`_PyEval_EvalFrameDefault`). Use **py-spy** for the Python-vs-native split and Python hotspots; use **perf** only for Rust symbol detail once py-spy says native dominates. -- **Profilers (confirmed present):** py-spy at `.pixi/envs/default/bin/py-spy`; perf at `/carter/users/dlaub/.pixi/bin/perf`. +- **py-spy is UNUSABLE on this node (discovered at run start).** `/proc/sys/kernel/yama/ptrace_scope = 2` (admin-only ptrace) and there is no sudo, so py-spy — native *and* non-native — dies with "Permission Denied". **perf is the only profiler.** perf uses `perf_event_open` (paranoid=2 allows own-process sampling), not ptrace, so it works. +- **perf cannot symbolize Python frames on Python < 3.12** (opaque `_PyEval_EvalFrameDefault`). Since py-spy is unavailable, the **Python-vs-native split is done at the DSO level with perf** (`perf report --sort=dso` / `--sort=dso,symbol`): bucket samples by shared object — gvl `genvarloader.abi3.so` + genoray `_core.so` = **native Rust hot path**; numpy `_multiarray_umath*.so` = **conversion/`ascontiguousarray` overhead**; the `python3.10` binary / `libpython` = **interpreter/orchestration**. This DSO split is *finer* than py-spy's leaf python/native split and directly drives the A/B/C classification. Use `--call-graph fp` (frame pointers were compiled in), never `dwarf` (1 GB file, overloads the shared node). +- **Profiler:** perf at `/carter/users/dlaub/.pixi/bin/perf`. (py-spy at `.pixi/envs/default/bin/py-spy` exists but is blocked by ptrace_scope — do not use.) `split_folded.py` (Task 2) was written for py-spy folded output and is therefore **unused**; left in place, harmless. - **Compute execution reality (discovered at run start):** this session already runs **inside** a small interactive SLURM allocation on `carter-cn-04` (**2 CPUs, 8 GB RAM**, 14-day walltime, governor=performance, paranoid=2). Therefore: **run light single-threaded profiling directly on this node** — do **NOT** wrap it in `srun` (an in-allocation `srun` makes a constrained 2-CPU *step* and fails with "More processors requested than permitted" at >2 cpus). Route **heavy multi-core / large-RAM work through fresh `sbatch` jobs** (independent of this allocation) sized to the big partition nodes (carter-cn-02/03/04 = 96–128 CPU, 476–953 GB). Everywhere a task below says `srun … `, execute `` **directly on the node** instead (the 3-region E1/E2-bench/E3 workloads are single-threaded and fit in 8 GB). Keep `sbatch` for E2 store builds and the E4 thread sweep, adding explicit `--cpus-per-task` and `--mem`. - **Noise control:** compute node via `sbatch -p carter-compute` for heavy jobs; do **NOT** use `--exclusive` — Carter has only 3 compute nodes, always shared, so `--exclusive` won't schedule. The node is shared and noisy, so **absolute wall-clock is not comparable across allocations** — only *relative* comparisons **within one short allocation on the same hardware** are valid (matches the prior perf-gate lesson: gate on same-session before/after, not absolute time). Therefore: **run both backends of any comparison inside a single `srun`** (back-to-back on the same node), warm caches, **median of N≥5**, record CPU governor / turbo. CPU governor on the target nodes is `performance` — record the actual value observed on the node you land on. Keep the MVP workload (same 3 chr21 regions, all samples) for continuity. - **MVP workload regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)`. From a527fe151d0e80c19e16b2affafc878ecaaccc2b Mon Sep 17 00:00:00 2001 From: d-laub Date: Sat, 4 Jul 2026 09:47:56 -0700 Subject: [PATCH 023/108] docs: SVAR2 gvl read-bound dataset wiring design spec Wire .svar2 into gvl.write() + Dataset.__getitem__ with all four output modes (haplotypes, tracks, variants, variant-windows) reconstructing in Rust off a write-time offsets cache. Read is all-Rust (genoray_core path-dep, no Python gather_ranges), with a per-class read-bound gather that eliminates both the interval-search rebuild and the per-read contig-wide dense-union rebuild (4 offset arrays into the already-split 2-bit/u32 dense tables). Write caches only the dataset's samples. genoray gets a query-only `conversion` feature so gvl links an htslib-free core. Local-only; relocate+rerun the MVP benchmark. Co-Authored-By: Claude Opus 4.8 --- ...07-04-svar2-gvl-readbound-wiring-design.md | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md diff --git a/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md b/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md new file mode 100644 index 00000000..416659d4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md @@ -0,0 +1,233 @@ +# SVAR2 gvl Read-Bound Dataset Wiring — Design Spec + +> **Purpose:** wire the SVAR2 format into `gvl.write()` and `gvl.Dataset.__getitem__` so all +> four output modes (haplotypes, tracks, variants, variant-windows) reconstruct **in Rust** off a +> write-time offsets cache, with **no interval-search tree rebuild and no dense-union rebuild at +> read**. This supersedes the gvl-side (Components B/C) of +> `2026-07-03-svar2-dataset-wiring-design.md`, now that genoray's search/gather split has shipped, +> and elects the deferred "fully read-bound" follow-up. + +**Date:** 2026-07-04 · **Status:** design, not yet implemented · **Ship:** **local-only, not +shipped** (genoray path-dep + local wheel; no crates.io / PyPI release in this spec). + +## Background & what changed + +The prior spec (`2026-07-03-svar2-dataset-wiring-design.md`) split genoray's fused `overlap_batch` +into search-only `find_ranges` + tree-free `gather_ranges` + fused `read_ranges`. **That split is +shipped** (merged to genoray `svar-2`): the interval-search trees are built once at write and not +rebuilt at read. Two things drive this follow-up spec: + +1. **The read path should be all-Rust.** `gvl.Dataset.__getitem__`'s hot loop is a single Rust FFI + call for SVAR1. SVAR2 reconstruction must match that — reconstruct in Rust, **not** via the + Python `SparseVar2.gather_ranges` API, and with no per-read numpy round-trip. +2. **The shipped `gather_ranges` still rebuilds a per-read dense union.** It merges the two on-disk + dense class tables (`dense/snp` 2-bit, `dense/indel` u32) into a transient position-sorted + `DenseUnion` **over the entire contig on every read** (genoray `src/query.rs` `dense_union()`) — + the O(N_contig) residual the prior spec flagged as "presence-bit cost at read." At 16k-sample + scale this is real. This spec eliminates it with a **per-class read-bound gather**. + +The `.svar2` dense store is **already** split and 2-bit-packed on disk (`src/dense.rs` +`DENSE_REGISTRY`: `dense/snp` `pack_snp: true`, `dense/indel` u32) — so **no genoray writer/format +change is needed**. The change is a new *gather* that slices those split tables' per-region windows +directly instead of unioning them. + +## Global constraints + +- **Byte-identical parity contract.** For any `contig, starts, ends, samples`: + `reconstruct(read-bound BatchResult)` ≡ `reconstruct(gather_ranges(find_ranges(...)))` (the shipped + union path) ≡ `reconstruct(overlap_batch(...))` ≡ genoray `decode` oracle — field-for-field / + byte-for-byte, for every output mode. +- **Additive.** The shipped union-based `find_ranges`/`gather_ranges`/`read_ranges`/`overlap_batch` + stay byte-unchanged and serve as the parity oracle and live-query API. The SVAR1 gvl path is + byte-unchanged (full SVAR1 regression green). Follows the rust-migration byte-identical parity + contract and the numba-oracle-bug policy. +- **Write caches only the dataset's samples.** `gvl.write` already selects which samples enter a + dataset (requested samples, plus any track-overlap resolution); the cache is sized to that + selection `S'`, not the full `.svar2` cohort — mirroring `_write_from_svar`. +- **Local-only.** gvl links `genoray_core` as a **path** dependency (`default-features = false`); + the Python genoray dep stays a **local wheel**. No crates.io/PyPI release. The path-dep and the + wheel MUST be built from the same genoray commit (the `RangesBundle` field layout is the contract). + +## Architecture — two deliverables + +**(A) genoray** — a second genoray PR, additive to the shipped split: +1. A **`conversion` cargo feature** (default-on) gating the htslib-tainted modules so the query core + builds htslib-free. +2. A **per-class read-bound path**: `find_ranges` emits per-class dense ranges; a read-bound gather + returns a **split-dense `BatchResult`** without building the contig-wide union. + +**(B) gvl** — the write cache + all-Rust read wiring targeting (A). + +Linkage: gvl adds `genoray_core = { path = "../genoray", default-features = false }` to `Cargo.toml` +(alongside the existing `svar2-codec` path-dep). Only `src/vcf_reader.rs` references `rust-htslib`, +so `default-features = false` yields a query-only, htslib-free core — gvl's build stays +toolchain-light. gvl's Rust opens `genoray_core::query::ContigReader` and calls the read-bound +gather directly; the Python `gather_ranges` API is **not** on gvl's read path. + +## Component A — genoray query-only feature + read-bound gather (dependency PR) + +### A1. `conversion` cargo feature (query-only build) +- `Cargo.toml`: make `rust-htslib` **optional**; add `[features] conversion = ["rust-htslib", ...]`, + `default = ["conversion", "extension-module"]` (keep `extension-module` behavior for the wheel). +- `src/lib.rs`: `#[cfg(feature = "conversion")]` on `vcf_reader` and its conversion-only dependents + (`writer`, `orchestrator`, `normalize` pipeline, and the `py_*` conversion entry points — the set + that transitively needs htslib). The query core (`query`, `search`, `spine`, `bits`, `nrvk`, + `rvk`, `layout`, `dense`, `types`, `py_query*`) builds with `default-features = false`. +- **Verify** the query core compiles with `--no-default-features` and that the Python wheel (built + with defaults) is byte-behavior-unchanged. The full genoray test suite stays green. + +### A2. Per-class read-bound `find_ranges` + gather + split-dense `BatchResult` +- **`find_ranges`** additionally emits **`dense_snp_range (R,2)`** and **`dense_indel_range (R,2)`** + — each a per-region `[ds,de]` into the corresponding on-disk dense table, computed by a per-class + overlap search (a `SearchTree` per class per region, at **write**, cold). These join the existing + `vk_snp_range`, `vk_indel_range` (per-hap) — the **4 offset arrays**. (The shipped union + `dense_range` may remain for the oracle path; the read-bound path uses the two per-class ranges.) +- **Read-bound gather** (`gather_ranges_readbound` or a bundle mode — genoray maintainer's naming): + slices `dense/snp[ds..de]` (2-bit) and `dense/indel[ds..de]` (u32) directly, applies the + per-element `q_start < v_end` left-overlap re-check, and computes **per-class presence bits** + (`DenseView::carried(hap, col)` over each class's window). Returns a **read-bound `BatchResult`**: + the var_key channel as today (snp+indel merged per hap via `merge_keys`, cheap over a small window) + **plus split `dense_snp` and `dense_indel` channels** (positions, keys, presence bits, and range), + **never** the contig-wide union. Exact channel factoring is a genoray-internal detail; gvl consumes + whatever the read-bound `BatchResult` exposes. +- **Parity:** a genoray test asserts `read-bound BatchResult` reconstructs identically to the shipped + union `BatchResult` and to the `decode` oracle (SNP/INS/DEL × samples × ploids, plus real chr21). + +## Component B — gvl write (`_write.py`, `_write_from_svar2`, `_svar2_link.py`) + +Mirror the SVAR1 write path (`_write_from_svar`, `_write.py:961`): +- **Detect `.svar2`.** Add a `.svar2` arm to variant-source path coercion (`_write.py:~217`) and a + `SparseVar2` arm to genotype-writing dispatch (`_write.py:~315`), alongside the `.svar`/`SparseVar` + branches. +- **`_write_from_svar2(path, bed, svar2, samples, extend_to_length)`.** Per contig, call + `svar2.find_ranges(c, df["chromStart"], df["chromEnd"], samples=samples, out=memmaps)` — for the + **dataset's samples only** — streaming the 4 offset arrays plus `region_starts`/`sample_cols` into + memmaps under `/genotypes/svar2_ranges/`. Write `svar2_meta.json` (shapes/dtypes) and a + `Svar2Link` into `metadata.json`; set `metadata["ploidy"] = svar2.ploidy`. +- **`_svar2_link.py`** mirrors `_svar_link.py`: a `Svar2Link` pydantic model + (`relative_path`, `absolute_path`, `fingerprint`) + `_resolve_svar2(gvl_path, link, override)` + + `_verify_fingerprint`. Fingerprint the `.svar2` on stable identity (n_variants from its index + a + canonical store-file byte count), analogous to `SvarFingerprint`. +- **Reject unsupported variants** (symbolic/breakend) as SVAR1 does; upstream `-V other,bnd` / + genoray M13 already filters these in the MVP build scripts. + +## Component C — gvl read (all Rust; `_open.py`, `_impl.py`/`_query.py`, `_haps.py`, kernels) + +- **`Svar2Store` (gvl pyclass).** Wraps a `genoray_core::query::ContigReader` per contig, opened + **once** at `Dataset.open` from the resolved `.svar2` path (the analog of SVAR1's once-built + `ffi_static` global table). Holds the memmapped cached 4-array ranges bundle. `Dataset.open` + resolves + fingerprints the `Svar2Link` (with a `svar2=` override paralleling `svar=`). +- **One FFI call per read.** `Dataset.__getitem__` → `_query.py` → a gvl Rust pyfunction that, for the + requested `(region, sample)` block: reconstructs a `genoray_core::RangesBundle` from the cached + memmap slice → calls the genoray **read-bound gather** → gets the split-dense `BatchResult` → + feeds the reconstruct core **in Rust** (no numpy round-trip, no Python `gather_ranges`). +- **Reconstruct kernel (read-bound variant).** A read-bound form of + `reconstruct_haplotypes_from_svar2` consumes the split-dense `BatchResult` and **merges + `var_key ⋈ dense_snp ⋈ dense_indel` by position** during assembly (extending the current + two-source splice), decoding keys inline via `svar2-codec`. `LongAlleleReader` lookups resolve + through the `ContigReader`'s LUT (no `lut_bytes`/`lut_off` marshalling). Tracks route + `shift_and_realign_tracks_from_svar2` off the **same** `BatchResult`. +- **Dispatch discriminant.** A dataset carries a source discriminant (`svar2_link` present) selecting + the SVAR2 reconstructor over the SVAR1 one; retires `TODO(svar2-dataset-dispatch)` in + `_svar2_source.py` (its live `overlap_batch` marshalling is removed for the cached path). + +## Output modes — all four, all Rust (Phase 1) + +All four modes read the **same** cached ranges and the **same** read-bound `BatchResult`; they differ +only in the final assembly kernel: +- **Haplotypes** (`with_seqs("haplotypes")` / default) — `reconstruct_haplotypes_from_svar2` + (read-bound). +- **Tracks** (`with_tracks`) — `shift_and_realign_tracks_from_svar2` (needs only `ilen`/`deletion_len`, + no alleles). +- **Variants** and **variant-windows** (`with_seqs("variants"/"variant-windows")`) — a gvl Rust + kernel decodes the per-hap overlapping keys from the read-bound `BatchResult` into `RaggedVariants` + (via `svar2-codec` `decode_key`: `Inline`/`PureDel`/`Lookup`, LUT via the `ContigReader`), mirroring + genoray's `decode_hap`. **No** Python `gather_ranges`/`decode` path. Same static-table Rust route + gvl already uses for SVAR1 variants. + +## Cache format (`.gvl/genotypes/svar2_ranges/`) + +The 4 offset arrays + 2 index vectors from `find_ranges(samples=dataset)`, as in-place memmaps +(`out=`), region/sample-sharded to the dataset layout; `svar2_meta.json` records shapes/dtypes +(mirrors SVAR1's `svar_meta.json`); `Svar2Link` → shared `.svar2`. + +``` +vk_snp_range (R, S', P, 2) -> vk_snp packed positions/keys (2-bit) +vk_indel_range (R, S', P, 2) -> vk_indel packed positions/keys (u32) +dense_snp_range (R, 2) -> dense/snp window (2-bit) +dense_indel_range (R, 2) -> dense/indel window (u32) +region_starts (R,) -> q_start per region (left-overlap re-check) +sample_cols (S',) -> selected slot -> original sample index +``` +**Size:** O(offsets) — same order as SVAR1's `offsets.npy`. The bulk (var_key + dense positions/keys, +genotype bitmatrices, LUT) stays in the compressed `.svar2`; SVAR2's on-disk size advantage +(1.46–5.67× smaller than `.svar`, per the MVP benchmark) is preserved. + +## Data flow + +- **WRITE:** `bed + .svar2` → per contig `find_ranges(..., samples=dataset, out=memmaps)` (search + once, per-class) → 4-array ranges cache + `Svar2Link` + `svar2_meta.json`. +- **READ:** `dataset[regions, samples]` → slice cached 4-array block → **read-bound gather** + (tree-free, union-free, per-class windows only) → split-dense `BatchResult` → Rust reconstruct + (haps / tracks / variants / variant-windows). **No interval search and no contig-wide union at + read.** + +## Parity & testing + +- **Byte-identical:** read-bound reconstruct ≡ shipped union reconstruct ≡ `decode` oracle, for all + four output modes, on the M6b matrix (SNP/INS/DEL × samples × ploids) + real chr21 germline & + somatic stores. Track re-alignment matched the same way. +- **Additive guarantee:** SVAR1 path byte-unchanged — `pixi run -e dev pytest tests -q` + + `cargo test`. genoray: query-only build compiles; full genoray suite green; the shipped union path + unchanged. +- **Perf verification (same-session, shared-node caveat):** a warm SVAR2 `Dataset` read shows + **neither** `SearchTree::build` **nor** the dense-union rebuild — the perf DSO split flips from + ~80% genoray to gvl-kernel-bound, like SVAR1. Report as a relative before/after within one + allocation (absolute wall-clock is not comparable across allocations on shared Carter nodes). +- **Docs/roadmaps:** update the genoray roadmap (read-bound per-class gather; conversion feature) and + `docs/roadmaps/rust-migration.md`; update user-facing docs for `.svar2` as a `write` variant source + — `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, `README.md` + (docs-audit + skill-maintenance gates), and keep `api.md` in sync with any new `__all__` symbols. + +## Benchmark — relocate & re-run + +- **Relocate:** `mv /carter/users/dlaub/repos/for_loukik/svar2_mvp /carter/users/dlaub/projects/svar2_mvp` + (4.1 GB, not a git repo — plain move; update any absolute paths in `build_source.sh` / + benchmark driver). +- **Re-run** the SVAR1-vs-SVAR2 `gvl.Dataset.__getitem__` benchmark on chr21 germline (3202) + + somatic (16007) after wiring: latency (same-session before/after) + store size. **Success:** the + SVAR2 read is gvl-kernel-bound (no `SearchTree::build`, no union rebuild), and SVAR2's store-size + advantage holds. + +## Out of scope + +- **Annotated** output mode (extends from the same cache later). +- Shipping / release (crates.io / PyPI) — local-only path-dep + wheel; the release gate is a later, + separate step. +- Any change to the on-disk `.svar2` format itself (already split + 2-bit-packed; genoray-owned). +- Profiling experiments **E2** (same-cohort sample sweep) and **E4** (conversion build-thread + allocation) — parked; **E3** (dense-access probe) dropped (gvl dense-gather ≈1%). + +## Deliverables & sequencing + +1. **genoray PR** — `conversion` query-only feature + per-class read-bound `find_ranges`/gather + + split-dense `BatchResult` + parity (vs union & `decode`); build local wheel + crate. +2. **gvl PR** — write: `_write_from_svar2` + `_svar2_link.py` (4-array cache, dataset samples); + read: `Svar2Store` + read-bound reconstruct kernels for all four output modes + + `Dataset.open`/`__getitem__` dispatch + `_svar2_source.py` retirement; `genoray_core` path-dep. +3. **Relocate + re-run** the MVP benchmark. +4. **Docs/roadmap** updates. + +## Open questions / risks + +- **read-bound `BatchResult` channel factoring** (keep var_key merged + dense split, vs. fully + 4-channel unmerged) is a genoray-internal detail settled in the genoray PR; gvl consumes the + payload the read-bound gather exposes, staying agnostic. +- **wheel ↔ path-dep sync:** the Python wheel that writes the cache and the Rust path-dep that reads + it must be the same genoray commit; document the local-dev sync discipline (a fingerprint/version + check at `Dataset.open` is a possible guard). +- **format version:** adding `svar2_link` to `metadata.json` is additive; confirm + `_check_dataset_format_version` tolerates it and any bump is handled by `_migrate.py`. +- **`conversion` feature partition:** confirm the exact module set that transitively needs + `rust-htslib` (only `vcf_reader.rs` references it directly; verify `writer`/`orchestrator`/ + `normalize` and their py-entry points are the complete gated set). From 0b7c6b6f17a2701194d1b3b07b73cb5f67c0723d Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 00:33:19 -0700 Subject: [PATCH 024/108] feat(dataset): Svar2Link resolution/fingerprint + Metadata.svar2_link Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_link.py | 118 ++++++++++++++++++++ python/genvarloader/_dataset/_write.py | 2 + tests/unit/dataset/test_svar2_link.py | 102 +++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 python/genvarloader/_dataset/_svar2_link.py create mode 100644 tests/unit/dataset/test_svar2_link.py diff --git a/python/genvarloader/_dataset/_svar2_link.py b/python/genvarloader/_dataset/_svar2_link.py new file mode 100644 index 00000000..e551a36e --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_link.py @@ -0,0 +1,118 @@ +"""Resolution and integrity for the GVL dataset -> .svar2 back-reference. + +Mirrors _svar_link.py; the fingerprint keys on the .svar2 store's stable +identity (file count + summed byte size of its data files) rather than +SVAR1's variant_idxs.npy / index.arrow, neither of which .svar2 has. +SparseVar2 exposes no cheap variant-count accessor, so a semantic +n_variants field is deliberately not part of this fingerprint -- deriving +one would require contig lengths plus a full-span decode, which is +over-engineering for an integrity check. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_files: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +def _svar2_store_fingerprint(svar2_path: Path) -> tuple[int, int]: + """Deterministic (file count, total bytes) over the .svar2 store's data files. + + Walks the store for ``.bin``/``.npy`` files (dense + var_key + long-allele + payloads across all contigs). Changes iff the store's data files change -- + that is this fingerprint's only contract. + """ + files = sorted( + p for p in svar2_path.rglob("*") if p.is_file() and p.suffix in {".bin", ".npy"} + ) + return len(files), sum(p.stat().st_size for p in files) + + +def _resolve_svar2( + gvl_path: Path, + link: Svar2Link | None, + override: Path | str | None, +) -> Path: + """Resolve the .svar2 directory referenced by a GVL dataset. + + Order: override -> link.relative_path -> link.absolute_path -> sibling *.svar2. + Raises FileNotFoundError if none resolve to a directory. + """ + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_svar2_fingerprint(svar2_path: Path, link: Svar2Link | None) -> None: + """Compare the recorded fingerprint against the resolved svar2 store. + + No-op when ``link`` is None (legacy dataset, or one without a svar2 link). + Raises ValueError on mismatch. + """ + if link is None: + return + + n_files_observed, bytes_observed = _svar2_store_fingerprint(svar2_path) + + exp = link.fingerprint + mismatches: list[str] = [] + if n_files_observed != exp.n_files: + mismatches.append( + f"n_files: expected {exp.n_files}, observed {n_files_observed}" + ) + if bytes_observed != exp.store_bytes: + mismatches.append( + f"store_bytes: expected {exp.store_bytes}, observed {bytes_observed}" + ) + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) + + +def make_svar2_link(gvl_path: Path, svar2_path: Path) -> Svar2Link: + svar2_resolved = svar2_path.resolve() + n_files, store_bytes = _svar2_store_fingerprint(svar2_resolved) + return Svar2Link( + relative_path=os.path.relpath(svar2_resolved, start=gvl_path).replace( + os.sep, "/" + ), + absolute_path=str(svar2_resolved), + fingerprint=Svar2Fingerprint(n_files=n_files, store_bytes=store_bytes), + ) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index f3587430..29a92248 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -37,6 +37,7 @@ from .._ragged import INTERVAL_DTYPE # noqa: F401 # Task 3 migration reader imports this from .._utils import lengths_to_offsets, normalize_contig_name from .._variants._utils import path_is_pgen, path_is_vcf +from ._svar2_link import Svar2Link from ._svar_link import SvarLink from ._utils import bed_to_regions, regions_to_bed @@ -92,6 +93,7 @@ class Metadata(BaseModel, arbitrary_types_allowed=True): version: SemanticVersion | None = None format_version: SemanticVersion | None = None svar_link: SvarLink | None = None + svar2_link: Svar2Link | None = None @property def n_samples(self) -> int: diff --git a/tests/unit/dataset/test_svar2_link.py b/tests/unit/dataset/test_svar2_link.py new file mode 100644 index 00000000..fb0c2950 --- /dev/null +++ b/tests/unit/dataset/test_svar2_link.py @@ -0,0 +1,102 @@ +"""Unit tests for ``Svar2Link`` resolution + fingerprint integrity. + +Mirrors ``test_svar_link_models.py`` but for the ``.svar2`` back-reference +(``_svar2_link.py``). Three pure/tmp_path tests exercise the override/no-op +error paths; one integration-flavored test builds a real ``.svar2`` store +(via genoray's conversion pipeline, same fixture recipe as +``tests/test_svar2_reconstruct.py``) to prove the fingerprint actually +detects a mutated store. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from genvarloader._dataset._svar2_link import ( + Svar2Fingerprint, + Svar2Link, + _resolve_svar2, + _verify_svar2_fingerprint, + make_svar2_link, +) + +# Same tiny fixture recipe as tests/test_svar2_reconstruct.py::svar2_store. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +def test_resolve_prefers_override(tmp_path: Path): + real = tmp_path / "cohort.svar2" + real.mkdir() + link = Svar2Link( + relative_path="nope.svar2", + absolute_path="/nope.svar2", + fingerprint=Svar2Fingerprint(n_files=1, store_bytes=1), + ) + assert _resolve_svar2(tmp_path, link, real) == real + + +def test_resolve_missing_override_raises(tmp_path: Path): + with pytest.raises(FileNotFoundError): + _resolve_svar2(tmp_path, None, tmp_path / "absent.svar2") + + +def test_verify_none_link_is_noop(tmp_path: Path): + _verify_svar2_fingerprint(tmp_path, None) # must not raise + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_link") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_fingerprint_detects_mutated_store(svar2_store: Path, tmp_path: Path): + gvl_path = tmp_path / "ds.gvl" + gvl_path.mkdir() + + link = make_svar2_link(gvl_path, svar2_store) + _verify_svar2_fingerprint(svar2_store, link) # must not raise + + bin_files = sorted(svar2_store.rglob("*.bin")) + assert bin_files, "expected at least one .bin file in a real .svar2 store" + with open(bin_files[0], "ab") as f: + f.write(b"\x00") + + with pytest.raises(ValueError): + _verify_svar2_fingerprint(svar2_store, link) From 40c5e34252ef301b01edd99569ab3d9f79f6e233 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 01:20:25 -0700 Subject: [PATCH 025/108] feat(write): _write_from_svar2 6-array ranges cache + dispatch Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_write.py | 145 +++++++++++++++++++++- tests/dataset/test_write_svar2.py | 159 +++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 tests/dataset/test_write_svar2.py diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 29a92248..4d95b98b 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -16,7 +16,7 @@ import numpy as np import polars as pl import seqpro as sp -from genoray import PGEN, VCF, Reader, SparseVar +from genoray import PGEN, VCF, Reader, SparseVar, SparseVar2 from genoray import exprs as _gexprs from genoray._svar import dense2sparse from genoray._svar import _dense2sparse_with_length # type: ignore[missing-module-attribute] @@ -226,13 +226,18 @@ def write( variants = VCF(variants) elif variants.is_dir() and variants.suffix == ".svar": variants = SparseVar(variants) + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) else: raise ValueError( f"File {variants} has an unrecognized file extension. Please provide either a VCF or PGEN file.`" ) if available_samples is None: - available_samples = set(variants.available_samples) + if isinstance(variants, SparseVar2): + available_samples = set(variants.samples) + else: + available_samples = set(variants.available_samples) # Eagerly load the variant index so max_mem accounting is honest. # VCF and PGEN both support lazy-index construction; without this, @@ -329,6 +334,11 @@ def write( path, gvl_bed, variants, samples, extend_to_length ) metadata["svar_link"] = _svar_link + elif isinstance(variants, SparseVar2): + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link metadata["ploidy"] = variants.ploidy # free memory del variants @@ -1070,6 +1080,137 @@ def _write_from_svar( ), svar_link +def _svar2_region_max_ends( + svar2: SparseVar2, + contig: str, + starts: NDArray[np.integer], + ends: NDArray[np.integer], + samples: list[str], +) -> NDArray[np.int32]: + """SVAR1 parity: per region, the end (``pos - min(ilen, 0)``) of the + highest-position variant over the SELECTED samples' haplotypes. Regions with + no variants keep their original ``chromEnd``. + + ``SparseVar2.decode`` reports 0-based ``pos`` (unlike ``SparseVar.index``'s + 1-based VCF ``POS``, which SVAR1's ``v_ends`` formula is written against), so + ``pos`` is converted to 1-based here before applying the same formula -- + otherwise every extension would be off by one (masked in most regions + because the un-extended ``chromEnd`` already dominates the max). + + This is O(R * len(samples) * ploidy) Python iteration over decoded records -- + acceptable for correctness/tests; vectorize for large-cohort write perf as a + follow-up. + """ + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = [svar2.samples.index(s) for s in samples] + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = dec.data["pos"] + ilen_arr = dec.data["ilen"] + off = np.asarray(dec.offsets) + out = np.asarray(ends, np.int64).copy() # default = chromEnd + for r in range(R): + best_pos, best_end = -1, -1 + for s in sel: + for p in range(P): + h = (r * S_all + s) * P + p + a, b = int(off[h]), int(off[h + 1]) + if a == b: + continue + seg_pos = pos_arr[a:b] + seg_ilen = ilen_arr[a:b] + j = int(np.argmax(seg_pos)) # highest-position variant in this hap + p_pos = int(seg_pos[j]) + p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) # +1: 0-based -> 1-based + if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): + best_pos, best_end = p_pos, p_end + if best_pos >= 0: + out[r] = best_end + return out.astype(np.int32) + + +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: SparseVar2, + samples: list[str], + extend_to_length: bool, +) -> tuple[pl.DataFrame, Svar2Link]: + # symbolic/breakend variants are rejected upstream at .svar2 conversion; the + # store cannot represent them, and SparseVar2 exposes no index to re-check. + + out_dir = path / "genotypes" / "svar2_ranges" + out_dir.mkdir(parents=True, exist_ok=True) + + R, S, P = bed.height, len(samples), svar2.ploidy + vk_snp = np.memmap(out_dir / "vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap( + out_dir / "vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2) + ) + dense_snp = np.memmap(out_dir / "dense_snp_range.npy", np.int64, "w+", shape=(R, 2)) + dense_indel = np.memmap( + out_dir / "dense_indel_range.npy", np.int64, "w+", shape=(R, 2) + ) + region_starts = np.memmap(out_dir / "region_starts.npy", np.int64, "w+", shape=(R,)) + # sample_cols: selected slot -> original sample index (same for every contig). + sample_cols = np.asarray([svar2.samples.index(s) for s in samples], np.int64) + np.save(out_dir / "sample_cols.npy", sample_cols) + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "vk_snp_range": {"shape": [R, S, P, 2], "dtype": " 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +# Mirrors tests/test_svar2_reconstruct.py's svar2_store fixture exactly, so the +# matched .svar (SVAR1) store built from the same VCF is a valid parity oracle. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def vcf_and_ref(tmp_path_factory) -> tuple[Path, Path]: + """A bgzipped/indexed BCF + FASTA shared by the .svar2 and .svar fixtures.""" + d = tmp_path_factory.mktemp("svar2_write_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar2_store(vcf_and_ref, tmp_path_factory) -> Path: + bcf, ref = vcf_and_ref + from genoray import _core + + out = tmp_path_factory.mktemp("svar2_write") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +@pytest.fixture(scope="module") +def svar1_store(vcf_and_ref, tmp_path_factory) -> Path: + bcf, _ref = vcf_and_ref + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1_write") / "store.svar" + SparseVar.from_vcf(out, VCF(bcf), max_mem="64m", overwrite=True) + return out + + +def test_write_svar2_emits_cache(svar2_store: Path, tmp_path: Path): + from genoray import SparseVar2 + + svar2 = SparseVar2(svar2_store) + bed = pl.DataFrame( + { + "chrom": ["chr1", "chr1"], + "chromStart": [0, 5], + "chromEnd": [20, 15], + } + ) + out = tmp_path / "ds.gvl" + gvl.write(out, bed, variants=svar2, samples=None, overwrite=True) + + rd = out / "genotypes" / "svar2_ranges" + meta = json.loads((rd / "svar2_meta.json").read_text()) + assert set(meta) >= { + "vk_snp_range", + "vk_indel_range", + "dense_snp_range", + "dense_indel_range", + "region_starts", + "sample_cols", + } + assert meta["ploidy"] == svar2.ploidy + + md = json.loads((out / "metadata.json").read_text()) + assert md["svar2_link"] is not None + assert md["ploidy"] == svar2.ploidy + Svar2Link.model_validate(md["svar2_link"]) # shape check + + region_starts_shape = tuple(meta["region_starts"]["shape"]) + region_starts = np.memmap( + rd / "region_starts.npy", dtype=np.int64, mode="r", shape=region_starts_shape + ) + assert region_starts.shape == (bed.height,) + + +def test_write_svar2_max_ends_matches_svar1( + svar2_store: Path, svar1_store: Path, tmp_path: Path +): + """SVAR1 parity gate: end-extension semantics must match exactly. + + Regions are chosen to overlap the DEL at (0-based) POS 11 with varying + windows, so the extension is non-trivial and exercises the "no variants" + (keep chromEnd) branch too. + """ + from genoray import SparseVar, SparseVar2 + + svar2 = SparseVar2(svar2_store) + svar1 = SparseVar(svar1_store) + + bed = pl.DataFrame( + { + "chrom": ["chr1"] * 5, + "chromStart": [0, 0, 5, 12, 20], + "chromEnd": [15, 20, 10, 13, 30], + } + ) + + out2 = tmp_path / "ds_svar2.gvl" + gvl.write(out2, bed, variants=svar2, samples=None, overwrite=True) + + out1 = tmp_path / "ds_svar1.gvl" + gvl.write(out1, bed, variants=svar1, samples=None, overwrite=True) + + regions2 = np.load(out2 / "regions.npy") + regions1 = np.load(out1 / "regions.npy") + + # columns: chrom_idx, chromStart, chromEnd, strand + chrom_end_2 = regions2[:, 2] + chrom_end_1 = regions1[:, 2] + + assert chrom_end_2.tolist() == chrom_end_1.tolist(), ( + f"svar2 max_ends {chrom_end_2.tolist()} != svar1 max_ends {chrom_end_1.tolist()}" + ) From ccad4eec1d52c5ad393b514af60b92a2dd944598 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 01:44:00 -0700 Subject: [PATCH 026/108] test(write): lock svar2 cache contents + same-POS tie max_ends parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix 1: assert the 6-array ranges cache CONTENTS (not just shapes) against a direct find_ranges call, locking the row-major (R,S,P) reshape. Fix 2: same-POS SNP+DEL tie chromEnd parity vs a matched .svar (passes for SNP-first store order). NOTE: DEL-first store order provably diverges (svar1 under-extends via max-v_idx order; svar2's max-end is the correct coverage) — a latent SVAR1 max_ends bug; policy pending. Co-Authored-By: Claude Opus 4.8 --- tests/dataset/test_write_svar2.py | 152 ++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py index 9d1ce3e4..cfef154f 100644 --- a/tests/dataset/test_write_svar2.py +++ b/tests/dataset/test_write_svar2.py @@ -118,6 +118,67 @@ def test_write_svar2_emits_cache(svar2_store: Path, tmp_path: Path): ) assert region_starts.shape == (bed.height,) + # ---- FIX 1: verify cache CONTENTS (not just shapes/keys) against a direct + # find_ranges call over the same regions. gvl sorts the written samples, so + # replay find_ranges with the sorted sample list to match slot ordering. + # This LOCKS the row-major (R, S, P) reshape and per-contig layout: a + # scrambled / mis-transposed cache would fail loudly here. + sorted_samples = sorted(svar2.samples) # what gvl.write wrote (samples.sort()) + S, P = len(sorted_samples), svar2.ploidy + + def mm(name: str) -> np.ndarray: + # raw memmaps are written as ".npy" (no .npy header); the meta key + # is the bare name. Read via np.memmap with the recorded shape/dtype. + shape = tuple(meta[name]["shape"]) + return np.array( + np.memmap(rd / f"{name}.npy", dtype=np.int64, mode="r", shape=shape) + ) + + vk_snp = mm("vk_snp_range") # (R, S, P, 2) + vk_indel = mm("vk_indel_range") # (R, S, P, 2) + dense_snp = mm("dense_snp_range") # (R, 2) + dense_indel = mm("dense_indel_range") # (R, 2) + region_starts_full = mm("region_starts") # (R,) + + # sample_cols is written with np.save (has a .npy header): read with np.load. + sample_cols = np.load(rd / "sample_cols.npy") + assert sample_cols.tolist() == [svar2.samples.index(s) for s in sorted_samples] + + contig_offset = 0 + for (c,), df in bed.partition_by( + "chrom", as_dict=True, maintain_order=True + ).items(): + rc = df.height + lo, hi = contig_offset, contig_offset + rc + d = svar2.find_ranges( + c, + df["chromStart"].to_numpy(), + df["chromEnd"].to_numpy(), + samples=sorted_samples, + ) + # region_starts: exact per-contig match (upcast int32 -> int64). + np.testing.assert_array_equal( + region_starts_full[lo:hi], np.asarray(d["region_starts"], np.int64) + ) + # vk ranges: reshape (rc, S, P, 2) -> (rc*S*P, 2) must equal find_ranges' + # row-major (R*S*P, 2). This pins the reshape done in _write_from_svar2. + np.testing.assert_array_equal( + vk_snp[lo:hi].reshape(rc * S * P, 2), + np.asarray(d["vk_snp_range"], np.int64), + ) + np.testing.assert_array_equal( + vk_indel[lo:hi].reshape(rc * S * P, 2), + np.asarray(d["vk_indel_range"], np.int64), + ) + # dense ranges: per-region (rc, 2), upcast int32 -> int64. + np.testing.assert_array_equal( + dense_snp[lo:hi], np.asarray(d["dense_snp_range"], np.int64) + ) + np.testing.assert_array_equal( + dense_indel[lo:hi], np.asarray(d["dense_indel_range"], np.int64) + ) + contig_offset += rc + def test_write_svar2_max_ends_matches_svar1( svar2_store: Path, svar1_store: Path, tmp_path: Path @@ -157,3 +218,94 @@ def test_write_svar2_max_ends_matches_svar1( assert chrom_end_2.tolist() == chrom_end_1.tolist(), ( f"svar2 max_ends {chrom_end_2.tolist()} != svar1 max_ends {chrom_end_1.tolist()}" ) + + +# Same-POS tie fixture (FIX 2): two records at POS 12 (0-based 11) with different +# ends -- a SNP (G>A, end=12) and a DEL (GTA>G, ILEN -2, end=14) -- placed on +# DIFFERENT haplotypes of S0 (SNP on hap0, DEL on hap1). A single haplotype +# cannot carry both an overlapping SNP and DEL, so putting them on the same hap +# would make the svar2 encoder drop one; different haps keeps both variants +# present and reachable. Ordering is the coordinator's exact example (SNP record +# first, DEL record second), so in store order the DEL gets the higher v_idx. +# SVAR1's max_ends picks the max-v_idx variant's end; svar2 picks the max-end +# variant on a POS tie -- here both rules select the DEL (end 14), so the paths +# agree. See the task-2 report for the reverse store order (DEL-first), where +# the two rules provably diverge. +_TIE_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t12\t.\tG\tA\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t0|1\t0|0 +""" + + +@pytest.fixture(scope="module") +def tie_stores(tmp_path_factory) -> tuple[Path, Path]: + """Matched .svar2 and .svar stores from the same two-same-POS-records VCF.""" + from genoray import VCF, SparseVar, _core + + d = tmp_path_factory.mktemp("svar2_tie") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_TIE_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + svar2_out = d / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(svar2_out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (svar2_out / "meta.json").exists(), "svar2 conversion did not finish" + + svar1_out = d / "store.svar" + SparseVar.from_vcf( + svar1_out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return svar2_out, svar1_out + + +def test_write_svar2_max_ends_same_pos_tie( + tie_stores: tuple[Path, Path], tmp_path: Path +): + """SVAR1 parity on a same-POS tie: a SNP and a DEL at the same position. + + The bed region ends 1bp short of the DEL's footprint so the extension is + variant-driven (not masked by the region's own chromEnd). Both paths must + agree on the extended chromEnd. + """ + from genoray import SparseVar, SparseVar2 + + svar2_out, svar1_out = tie_stores + svar2 = SparseVar2(svar2_out) + svar1 = SparseVar(svar1_out) + + # POS 12 -> 0-based 11. Region [11, 13) overlaps it; region chromEnd 13 is + # below the DEL end (14), so the max_ends extension is variant-driven. + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [11], "chromEnd": [13]}) + + out2 = tmp_path / "tie_svar2.gvl" + gvl.write(out2, bed, variants=svar2, samples=None, overwrite=True) + out1 = tmp_path / "tie_svar1.gvl" + gvl.write(out1, bed, variants=svar1, samples=None, overwrite=True) + + chrom_end_2 = np.load(out2 / "regions.npy")[:, 2] + chrom_end_1 = np.load(out1 / "regions.npy")[:, 2] + + assert chrom_end_2.tolist() == chrom_end_1.tolist(), ( + f"same-POS tie: svar2 max_ends {chrom_end_2.tolist()} != " + f"svar1 max_ends {chrom_end_1.tolist()}" + ) From 6397ecf4679563c6f1919ead80dc63aebb74687e Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 09:04:33 -0700 Subject: [PATCH 027/108] docs: note latent SVAR1 max_ends same-POS-tie under-extension bug Co-Authored-By: Claude Opus 4.8 --- .../svar1-max-ends-tie-underextension.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/known-issues/svar1-max-ends-tie-underextension.md diff --git a/docs/known-issues/svar1-max-ends-tie-underextension.md b/docs/known-issues/svar1-max-ends-tie-underextension.md new file mode 100644 index 00000000..cb43cff0 --- /dev/null +++ b/docs/known-issues/svar1-max-ends-tie-underextension.md @@ -0,0 +1,29 @@ +# SVAR1 `_write_from_svar` max_ends under-extends at same-POS ties + +**Status:** open (latent bug in the SVAR1 write path; not yet fixed) +**Found:** 2026-07-05, during SVAR2 read-bound dataset wiring. + +## Symptom +`_write_from_svar` (`python/genvarloader/_dataset/_write.py`) computes each region's +`max_ends` as the end of the **highest-store-order variant** (`max v_idx`) overlapping +the region, then `chromEnd = max(max_ends, chromEnd)`. When two variant records share a +POS but have different ends — e.g. a SNP (`end = POS`) and a deletion (`end = POS - ILEN`) +at the same position — and the SNP has the higher store-order index, SVAR1 selects the +SNP's (shorter) end and **under-extends** the region. The deletion's coverage past the +region boundary is then truncated. + +Example: region `[11,13)`, `chr1:12 G>A` (SNP) and `chr1:12 GTA>G` (DEL, ILEN -2, end 14), +DEL indexed before SNP. SVAR1 → `chromEnd=13` (under-extended); the correct value is `14`. + +The code comment at the `max_ends` computation already hedges: "this is fine if there +aren't any overlapping variants that could make a v_idx < -1 have a further end." + +## SVAR2 behavior (correct) +The SVAR2 read-bound write path (`_write_from_svar2`) computes `max_ends` as the true end +of the max-position variant (`pos - min(ilen,0)`), yielding the **correct** `chromEnd=14`. + +## Parity policy +SVAR1↔SVAR2 `chromEnd` parity is byte-identical **except** same-POS multi-record tie +regions, where SVAR1 is the buggy oracle. Such regions are excluded from strict parity +(see the SVAR2 wiring plan's Task 7). Fixing SVAR1's `max_ends` is deferred (it would +change SVAR1 output for tie cases, i.e. it is not an additive change). From 503d24545bf5e839177299bb3e763d2e7481ada2 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 10:05:25 -0700 Subject: [PATCH 028/108] feat(rust): link genoray_core (query-only) + Svar2Store pyclass Links genoray's query-only core crate (default-features=false, pkg genoray / lib genoray_core) and adds the Svar2Store pyclass: opens one ContigReader per contig at Dataset.open, held for the store's lifetime (SVAR2 analog of SVAR1's cached FFI-static). Requires bumping gvl pyo3 0.28.3->0.29 + numpy 0.28->0.29 to match genoray @ aaf44fd (pyo3-ffi links="python" allows only one pyo3 version in the graph). Bump is clean: no API migration, no new warnings; full Rust suite (120 tests) green under 0.29. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 282 +++++++++++++++++++++++-- Cargo.toml | 5 +- src/lib.rs | 6 +- src/svar2/mod.rs | 14 +- src/svar2/store.rs | 43 ++++ tests/unit/dataset/test_svar2_store.py | 53 +++++ 6 files changed, 384 insertions(+), 19 deletions(-) create mode 100644 src/svar2/store.rs create mode 100644 tests/unit/dataset/test_svar2_store.py diff --git a/Cargo.lock b/Cargo.lock index 957d464f..c741a2e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -67,6 +73,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "attohttpc" version = "0.25.0" @@ -116,7 +131,7 @@ dependencies = [ "serde", "smallvec", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "ufmt", ] @@ -136,6 +151,18 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -241,6 +268,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -275,6 +311,17 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -320,6 +367,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -435,6 +492,23 @@ dependencies = [ "slab", ] +[[package]] +name = "genoray" +version = "0.1.0" +dependencies = [ + "bytemuck", + "crossbeam-channel", + "memmap2", + "ndarray", + "ndarray-npy", + "numpy", + "pyo3", + "rayon", + "serde_json", + "svar2-codec", + "thiserror 2.0.18", +] + [[package]] name = "genvarloader" version = "0.2.1" @@ -442,6 +516,7 @@ dependencies = [ "anyhow", "bigtools", "coitrees", + "genoray", "itertools 0.14.0", "ndarray", "numpy", @@ -641,6 +716,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -735,6 +819,25 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "ndarray" version = "0.17.2" @@ -751,6 +854,30 @@ dependencies = [ "rayon", ] +[[package]] +name = "ndarray-npy" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e8a348bca0075000d999d750420d74434fd0d3e0993b456554f885e7657a11" +dependencies = [ + "byteorder", + "ndarray", + "num-complex", + "num-traits", + "py_literal", + "zip", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -780,9 +907,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778da78c64ddc928ebf5ad9df5edf0789410ff3bdbf3619aed51cd789a6af1e2" +checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" dependencies = [ "libc", "ndarray", @@ -818,6 +945,48 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +dependencies = [ + "pest", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -876,12 +1045,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "py_literal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102df7a3d46db9d3891f178dcc826dc270a6746277a9ae6436f8d29fd490a8e1" +dependencies = [ + "num-bigint", + "num-complex", + "num-traits", + "pest", + "pest_derive", +] + [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ + "inventory", "libc", "once_cell", "portable-atomic", @@ -892,18 +1075,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -911,9 +1094,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -923,13 +1106,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.117", ] @@ -1124,6 +1306,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.23" @@ -1237,6 +1425,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "slab" version = "0.4.12" @@ -1323,7 +1517,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1337,6 +1540,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1386,6 +1600,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "ufmt" version = "0.2.0" @@ -1797,8 +2017,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index d2c5196c..4a010fbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,16 +16,17 @@ bigtools = "0.5.6" coitrees = "0.4" itertools = "0.14.0" ndarray = "0.17.2" -numpy = "0.28.0" +numpy = "0.29.0" rayon = "1.12.0" seqpro-core = "0.1" svar2-codec = { path = "/carter/users/dlaub/projects/genoray/svar2-codec" } +genoray_core = { path = "/carter/users/dlaub/projects/genoray", package = "genoray", default-features = false } [features] extension-module = ["pyo3/extension-module"] [dependencies.pyo3] -version = "0.28.3" +version = "0.29" features = ["abi3-py310"] [dev-dependencies] diff --git a/src/lib.rs b/src/lib.rs index b8a9db15..c5c8d654 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(bigwig_intervals, m)?)?; m.add_function(wrap_pyfunction!(bigwig_write_track, m)?)?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(ragged::ragged_to_padded, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_to_tracks, m)?)?; m.add_function(wrap_pyfunction!(ffi::get_diffs_sparse, m)?)?; @@ -58,7 +59,10 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { m )?)?; m.add_function(wrap_pyfunction!(ffi::shift_and_realign_tracks_sparse, m)?)?; - m.add_function(wrap_pyfunction!(ffi::shift_and_realign_tracks_from_svar2, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::shift_and_realign_tracks_from_svar2, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index cf79705f..8f4910db 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -7,6 +7,8 @@ use std::borrow::Cow; use ndarray::{Array2, ArrayView2}; use svar2_codec::{decode_key, DecodedKey}; +pub mod store; + /// Decode one uniform key into `(v_diff, allele)`, resolving long-INS via the LUT /// arrays. Mirrors genoray's `decode_keyref`. pub fn decode_alt<'a>(key: u32, lut_bytes: &'a [u8], lut_off: &[i64]) -> (i64, Cow<'a, [u8]>) { @@ -83,7 +85,17 @@ pub fn hap_diffs_svar2( let bit = base_bit + j; (dense_present[bit / 8] >> (bit % 8)) & 1 == 1 }; - let merged = merge_hap(vk_pos, vk_key, vk_lo, vk_hi, dense_pos, dense_key, ds, de, present_bit); + let merged = merge_hap( + vk_pos, + vk_key, + vk_lo, + vk_hi, + dense_pos, + dense_key, + ds, + de, + present_bit, + ); if merged.is_empty() { continue; } diff --git a/src/svar2/store.rs b/src/svar2/store.rs new file mode 100644 index 00000000..d284dc8e --- /dev/null +++ b/src/svar2/store.rs @@ -0,0 +1,43 @@ +use std::collections::HashMap; + +use genoray_core::query::ContigReader; +use pyo3::exceptions::PyIOError; +use pyo3::prelude::*; + +/// Opened once at Dataset.open; holds one query-only ContigReader per contig for +/// the store's lifetime (SVAR2 analog of SVAR1's cached _HapsFfiStatic). +#[pyclass] +pub struct Svar2Store { + readers: HashMap, +} + +impl Svar2Store { + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } +} + +#[pymethods] +impl Svar2Store { + #[new] + fn new( + store_path: &str, + contigs: Vec, + n_samples: usize, + ploidy: usize, + ) -> PyResult { + let mut readers = HashMap::with_capacity(contigs.len()); + for c in contigs { + let r = ContigReader::open(store_path, &c, n_samples, ploidy) + .map_err(|e| PyIOError::new_err(format!("open contig {c}: {e}")))?; + readers.insert(c, r); + } + Ok(Self { readers }) + } + + fn contigs(&self) -> Vec { + let mut v: Vec = self.readers.keys().cloned().collect(); + v.sort(); + v + } +} diff --git a/tests/unit/dataset/test_svar2_store.py b/tests/unit/dataset/test_svar2_store.py new file mode 100644 index 00000000..d820adb5 --- /dev/null +++ b/tests/unit/dataset/test_svar2_store.py @@ -0,0 +1,53 @@ +"""Svar2Store pyclass: opens one query-only genoray_core ContigReader per contig +at construction (the SVAR2 analog of SVAR1's cached FFI-static), held for the +store's lifetime. Built from a real .svar2 store via genoray's conversion pipeline. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from genvarloader.genvarloader import Svar2Store # compiled extension + +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_store") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "cohort.svar2" + _core.run_conversion_pipeline( + str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], + 25_000, 2, 1, 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_store_opens_contigs(svar2_store: Path): + store = Svar2Store(str(svar2_store), ["chr1"], n_samples=2, ploidy=2) + assert store.contigs() == ["chr1"] From c7998a65429b43bada347990a776a819c411b332 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 10:45:27 -0700 Subject: [PATCH 029/108] feat(svar2): read-bound haplotype kernel (one all-Rust FFI call) Reuse the already byte-validated union-path kernels (svar2::hap_diffs_svar2, reconstruct::reconstruct_haplotypes_from_svar2) instead of duplicating a three-source merge/decode/sizing path: marshal genoray's read-bound BatchResultSplit into the same flat single-dense-channel layout via a new svar2::split_to_flat, then call the existing kernels unchanged through a new reconstruct_haplotypes_from_svar2_readbound FFI entry point. Parity test (tests/dataset/test_svar2_readbound_haps.py) confirms byte-identical output vs. the union-path oracle (SparseVar2Source.reconstruct) across region-shape variations and non-trivial per-hap jitter shifts. --- .../genvarloader/_dataset/_svar2_store_py.py | 108 +++++++++++ src/ffi/mod.rs | 164 +++++++++++++++- src/lib.rs | 4 + src/svar2/mod.rs | 149 +++++++++++++++ tests/dataset/test_svar2_readbound_haps.py | 178 ++++++++++++++++++ 5 files changed, 602 insertions(+), 1 deletion(-) create mode 100644 python/genvarloader/_dataset/_svar2_store_py.py create mode 100644 tests/dataset/test_svar2_readbound_haps.py diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py new file mode 100644 index 00000000..962be3fc --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -0,0 +1,108 @@ +"""Read-bound SVAR2 haplotype reconstruction: one all-Rust FFI call gathering off a +query-only genoray ``Svar2Store`` (``genoray_core::query::gather_haps_readbound``), with +NO interval-search-tree rebuild and NO dense-union rebuild. + +Byte-identical to the existing union-path oracle (``SparseVar2Source.reconstruct``, +``_svar2_source.py``), which calls ``reconstruct_haplotypes_from_svar2`` over +``SparseVar2.overlap_batch``'s eagerly-unioned dense channel. This module instead +marshals ``SparseVar2.find_ranges``'s per-class-split ranges through +``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` (Rust side) +and reuses that same validated kernel — see ``reconstruct_haplotypes_from_svar2_readbound`` +in ``src/ffi/mod.rs``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import numpy as np + +from .._flat import _Flat +from ..genvarloader import Svar2Store, reconstruct_haplotypes_from_svar2_readbound + +if TYPE_CHECKING: + from genoray import SparseVar2 + from numpy.typing import NDArray + from seqpro.rag import Ragged + + +def build_readbound_haps( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R + ref_: "NDArray[np.uint8]", # the contig reference bytes + ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) + pad_char: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + output_length: int = -1, + parallel: bool = False, +) -> "Ragged[np.bytes_]": + """Reconstruct the full-cohort haplotypes over ``regions`` via the read-bound kernel. + + Mirrors ``SparseVar2Source.reconstruct``'s signature/return shape exactly (query + order region-major, sample-minor: ``q = r*S + s``), but drives + ``SparseVar2.find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``overlap_batch``'s eager per-region dense union. + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2.find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + out_data, out_offsets = reconstruct_haplotypes_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + np.ascontiguousarray(ref_, np.uint8), + np.ascontiguousarray(ref_offsets, np.int64), + np.uint8(pad_char), + np.int64(output_length), + parallel, + ) + + shape = (R, S, P, None) + return cast( + "Ragged[np.bytes_]", _Flat.from_offsets(out_data, shape, out_offsets).view("S1") + ) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 47f03a8e..09665390 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1,5 +1,5 @@ //! PyO3 boundary for migrated core kernels. The ONLY place new kernels touch Python. -use ndarray::Array1; +use ndarray::{Array1, Array2}; use numpy::{ IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2, PyReadwriteArray1, }; @@ -888,6 +888,168 @@ pub fn reconstruct_haplotypes_from_svar2<'py>( (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) } +/// Read-bound SVAR2 haplotype reconstruction: gather off a query-only genoray +/// `Svar2Store` reader with NO interval-search-tree rebuild and NO dense-union +/// rebuild (`genoray_core::query::gather_haps_readbound`), marshal the split +/// result into the flat layout via [`crate::svar2::split_to_flat`], then reuse +/// the byte-validated [`reconstruct_haplotypes_from_svar2`] kernel unchanged — +/// one FFI crossing, byte-identical to the union-path oracle. +/// +/// `region_starts`/`orig_samples`/`vk_snp_range`/`vk_indel_range`/ +/// `dense_snp_range`/`dense_indel_range` are the per-query outputs of +/// `SparseVar2.find_ranges` (flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_haps`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + shifts: PyReadonlyArray2, + ref_: PyReadonlyArray1, + ref_offsets: PyReadonlyArray1, + pad_char: u8, + output_length: i64, + parallel: bool, +) -> PyResult<(Bound<'py, PyArray1>, Bound<'py, PyArray1>)> { + use crate::reconstruct; + use crate::svar2; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let shifts_a = shifts.as_array(); + let ploidy = shifts_a.ncols(); + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end) — `ref_` is the + // single contig slice the caller passed in (ref_offsets = [0, len]). + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize, r[1] as usize)) + .collect() + }; + let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); + let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); + let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); + let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + + let ref_a = ref_.as_array(); + let ref_offsets_a = ref_offsets.as_array(); + + let (out_data, out_offsets_vec) = py.detach(move || { + let br = genoray_core::query::gather_haps_readbound( + reader, + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + // Step 1: size via the same two-source diff core the union path uses. + let diffs = svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ); + + // Step 2: per-haplotype output lengths and prefix-sum offsets. + let n_work = n_q * ploidy; + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let len: i64 = if output_length >= 0 { + output_length + } else { + let ref_len = (regions[[query, 2]] - regions[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + (ref_len + diff).max(0) + }; + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = uninit_output(total); + + // Step 4: reconstruct — reuse the byte-validated union-path kernel + // unchanged, now fed the read-bound gather's flat channels. + reconstruct::reconstruct_haplotypes_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions.view(), + shifts_a, + numpy::ndarray::ArrayView1::from(flat.vk_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_key.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_off.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_key.as_slice()), + dense_range_a, + numpy::ndarray::ArrayView1::from(flat.dense_present.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_present_off.as_slice()), + numpy::ndarray::ArrayView1::from(lut_bytes.as_slice()), + numpy::ndarray::ArrayView1::from(lut_off.as_slice()), + ref_a, + ref_offsets_a, + pad_char, + None, + None, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) +} + /// Fused SVAR2 two-source track shift+realign: merge each hap's `var_key` ⋈ `dense` /// channels and decode via `svar2-codec` inline, sizing and allocating the output /// buffer in Rust — one FFI crossing, mirrors `reconstruct_haplotypes_from_svar2` diff --git a/src/lib.rs b/src/lib.rs index c5c8d654..ef02fedf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,6 +46,10 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_fused, m)?)?; m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_from_svar2, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_from_svar2_readbound, + m + )?)?; m.add_function(wrap_pyfunction!( ffi::reconstruct_annotated_haplotypes_fused, m diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 8f4910db..21f79da5 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; +use genoray_core::query::BatchResultSplit; use ndarray::{Array2, ArrayView2}; use svar2_codec::{decode_key, DecodedKey}; @@ -128,6 +129,106 @@ pub fn hap_diffs_svar2( diffs } +/// The flat, single-dense-channel layout consumed by [`hap_diffs_svar2`] / +/// `reconstruct::reconstruct_haplotypes_from_svar2` — see [`split_to_flat`]. +pub struct FlatChannels { + pub vk_pos: Vec, + pub vk_key: Vec, + pub vk_off: Vec, + pub dense_pos: Vec, + pub dense_key: Vec, + /// Flat, length `n_q*2`; view as `(n_q, 2)` at the call site. + pub dense_range: Vec, + pub dense_present: Vec, + pub dense_present_off: Vec, +} + +/// Marshal a read-bound [`BatchResultSplit`] (genoray's per-class-split dense +/// channels) into the flat single-dense-channel layout that the already-validated +/// [`hap_diffs_svar2`] / `reconstruct::reconstruct_haplotypes_from_svar2` kernels +/// consume — so read-bound gather can reuse those kernels unchanged instead of +/// duplicating the merge/decode logic for a three-source layout. +/// +/// Per query `q` the combined dense window is `dense_snp[q] ++ dense_indel[q]` +/// (SNP entries first — matches genoray's `merge_keys(vec![vk, dense_snp, +/// dense_indel])` tie order, where the dense side is `dense_snp` before +/// `dense_indel`). Per hap the combined presence bits are `snp_bits ++ +/// indel_bits` over that combined window, LSB-first packed into one bitstream, +/// so `dense_present_off`/`dense_range` line up with `dense_pos`/`dense_key` +/// exactly like the union path's single dense channel. +pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { + let ploidy = br.ploidy; + let n_q = br.n_regions; // n_samples == 1 for read-bound gather + let h_count = n_q * ploidy; + + let vk_pos: Vec = br.vk.iter().map(|k| k.position as i32).collect(); + let vk_key: Vec = br.vk.iter().map(|k| k.key as i32).collect(); + let vk_off: Vec = br.vk_off.iter().map(|&o| o as i64).collect(); + + let mut dense_pos: Vec = Vec::new(); + let mut dense_key: Vec = Vec::new(); + let mut dense_range: Vec = Vec::with_capacity(n_q * 2); + for q in 0..n_q { + let base = dense_pos.len() as i32; + let (ss, se) = br.dense_snp_range[q]; + for j in ss..se { + dense_pos.push(br.dense_snp[j].position as i32); + dense_key.push(br.dense_snp[j].key as i32); + } + let (is_, ie) = br.dense_indel_range[q]; + for j in is_..ie { + dense_pos.push(br.dense_indel[j].position as i32); + dense_key.push(br.dense_indel[j].key as i32); + } + dense_range.push(base); + dense_range.push(dense_pos.len() as i32); + } + + let mut dense_present: Vec = Vec::new(); + let mut dense_present_off: Vec = Vec::with_capacity(h_count + 1); + let mut bit_acc: usize = 0; + dense_present_off.push(0); + for h in 0..h_count { + let q = h / ploidy; + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + let snp_base = br.dense_snp_present_off[h]; + for k in 0..(se - ss) { + if genoray_core::bits_get_bit(&br.dense_snp_present, snp_base + k) { + let byte = bit_acc / 8; + if dense_present.len() <= byte { + dense_present.resize(byte + 1, 0); + } + dense_present[byte] |= 1 << (bit_acc % 8); + } + bit_acc += 1; + } + let indel_base = br.dense_indel_present_off[h]; + for k in 0..(ie - is_) { + if genoray_core::bits_get_bit(&br.dense_indel_present, indel_base + k) { + let byte = bit_acc / 8; + if dense_present.len() <= byte { + dense_present.resize(byte + 1, 0); + } + dense_present[byte] |= 1 << (bit_acc % 8); + } + bit_acc += 1; + } + dense_present_off.push(bit_acc as i64); + } + + FlatChannels { + vk_pos, + vk_key, + vk_off, + dense_pos, + dense_key, + dense_range, + dense_present, + dense_present_off, + } +} + #[cfg(test)] mod tests { use super::*; @@ -226,4 +327,52 @@ mod tests { assert_eq!(diffs[[0, 0]], -1); } + + #[test] + fn test_split_to_flat_marshals_readbound_split() { + use genoray_core::query::KeyRef; + + // ploidy=1, n_regions=1: one vk key, one dense_snp entry (present for + // this hap), one dense_indel entry (absent for this hap). + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 5, + key: 100, + }], + vk_off: vec![0, 1], + dense_snp: vec![KeyRef { + position: 10, + key: 200, + }], + dense_snp_range: vec![(0, 1)], + dense_snp_present: vec![0b1], // present + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 15, + key: 300, + }], + dense_indel_range: vec![(0, 1)], + dense_indel_present: vec![0b0], // absent + dense_indel_present_off: vec![0, 1], + }; + + let flat = split_to_flat(&br); + + assert_eq!(flat.vk_pos, vec![5]); + assert_eq!(flat.vk_key, vec![100]); + assert_eq!(flat.vk_off, vec![0, 1]); + + // Combined dense window: snp entries first, then indel entries. + assert_eq!(flat.dense_pos, vec![10, 15]); + assert_eq!(flat.dense_key, vec![200, 300]); + assert_eq!(flat.dense_range, vec![0, 2]); + + // Combined presence bits: snp bit (present) then indel bit (absent), + // LSB-first -> byte 0 = 0b01. + assert_eq!(flat.dense_present, vec![0b01]); + assert_eq!(flat.dense_present_off, vec![0, 2]); + } } diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py new file mode 100644 index 00000000..2ea69838 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -0,0 +1,178 @@ +"""Parity test for the read-bound SVAR2 haplotype kernel (Task 4). + +Oracle: ``SparseVar2Source.reconstruct`` (genoray ``overlap_batch``, eager dense-union +path). Under test: ``build_readbound_haps`` (genoray ``find_ranges`` + one Rust FFI call +via ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> +the SAME validated ``reconstruct_haplotypes_from_svar2`` kernel the oracle uses). + +Both paths must be byte-identical: same offsets, same data. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_matches_union_oracle(svar2_store, regions): + import genoray + + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._svar2_store_py import build_readbound_haps + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + + oracle_offsets = np.asarray(oracle.offsets) + rb_offsets = np.asarray(rb.offsets) + assert np.array_equal(oracle_offsets, rb_offsets), ( + f"offsets mismatch: oracle={oracle_offsets.tolist()} rb={rb_offsets.tolist()}" + ) + + oracle_data = np.asarray(oracle.data).view("u1") + rb_data = np.asarray(rb.data).view("u1") + if not np.array_equal(oracle_data, rb_data): + # Locate the first mismatching (query, hap, byte) for debuggability. + R = len(regions) + H = P + n_q = R * S + for h in range(n_q * H): + s0, e0 = int(oracle_offsets[h]), int(oracle_offsets[h + 1]) + s1, e1 = int(rb_offsets[h]), int(rb_offsets[h + 1]) + a = oracle_data[s0:e0] + b = rb_data[s1:e1] + if not np.array_equal(a, b): + pytest.fail( + f"data mismatch at hap {h}: oracle={a.tobytes()!r} rb={b.tobytes()!r}" + ) + pytest.fail("data mismatch but no single hap slice differed (offset bug?)") + + +def test_readbound_matches_union_oracle_with_shifts(svar2_store): + """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" + import genoray + + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._svar2_store_py import build_readbound_haps + + contig = "chr1" + regions = [(0, 40), (5, 20)] + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + n_q = len(regions) * S + rng = np.random.default_rng(0) + shifts = rng.integers(-2, 3, size=(n_q, P), dtype=np.int32) + + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=shifts, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=shifts, + output_length=-1, + parallel=False, + ) + + assert np.array_equal(np.asarray(oracle.offsets), np.asarray(rb.offsets)) + assert np.array_equal( + np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") + ) From 1dbfac0d1c841f935688eed4a8c7ebd534b8b0ac Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 11:07:07 -0700 Subject: [PATCH 030/108] fix(svar2): byte-size split_to_flat presence buffer; cover dense_snp + byte boundary split_to_flat only grew dense_present on set bits, leaving a trailing all-zero byte unallocated -> OOB panic when the reused kernels read every window bit for a final hap landing in that byte. Resize to bit_acc.div_ceil(8) unconditionally (matches genoray's byte-sizing). Add a >8-bit unit test with an unset trailing byte (regression for the OOB) and a parity case routing a >=2-carrier SNP into dense_snp (mixed with dense indels) to exercise the snp --- src/svar2/mod.rs | 68 ++++++++++++ tests/dataset/test_svar2_readbound_haps.py | 114 +++++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 21f79da5..444f1c87 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -216,6 +216,13 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { } dense_present_off.push(bit_acc as i64); } + // The reused kernels read `dense_present[bit/8]` for EVERY window entry of + // every hap, so the buffer must always be ceil(total_bits/8) bytes — even + // when the last hap's window bits are all zero (the in-loop grow-on-set + // above only extends up to the highest SET bit, leaving a trailing all-zero + // byte unallocated → OOB panic downstream). genoray byte-sizes its presence + // arrays via div_ceil unconditionally; match that here. + dense_present.resize(bit_acc.div_ceil(8), 0); FlatChannels { vk_pos, @@ -375,4 +382,65 @@ mod tests { assert_eq!(flat.dense_present, vec![0b01]); assert_eq!(flat.dense_present_off, vec![0, 2]); } + + #[test] + fn test_split_to_flat_trailing_zero_byte_is_allocated() { + use genoray_core::query::KeyRef; + + // Regression for the OOB defect: `dense_present` must always be + // ceil(total_bits/8) bytes, even when the trailing byte is entirely + // zero. 12 haps (ploidy=1, n_regions=12), each with a 1-entry dense/snp + // window and no dense/indel -> 12 combined presence bits spanning 2 + // bytes. Only haps 0 and 3 carry the entry (both in byte 0); haps 8..12 + // (byte 1) are all UNSET. The grow-on-set path alone would leave byte 1 + // unallocated, so the reused kernels' `dense_present[bit/8]` read for + // hap 8..12 would panic. + let n = 12usize; + + // genoray's own bitstream: window size 1 per hap, bits set at 0 and 3. + let mut dense_snp_present = vec![0u8; n.div_ceil(8)]; // 2 bytes + dense_snp_present[0] = 0b0000_1001; // bits 0 and 3 + let dense_snp_present_off: Vec = (0..=n).collect(); + + let br = BatchResultSplit { + n_regions: n, + n_samples: 1, + ploidy: 1, + vk: vec![], + vk_off: vec![0; n + 1], + dense_snp: vec![KeyRef { + position: 42, + key: 7, + }], + dense_snp_range: vec![(0, 1); n], // every query points at the lone entry + dense_snp_present, + dense_snp_present_off, + dense_indel: vec![], + dense_indel_range: vec![(0, 0); n], // no indel window + dense_indel_present: vec![], + dense_indel_present_off: vec![0; n + 1], + }; + + // Must not panic. + let flat = split_to_flat(&br); + + // Buffer sized to ceil(total_bits/8) = ceil(12/8) = 2, NOT just up to + // the highest set bit (byte 0). + assert_eq!(flat.dense_present.len(), 2); + assert_eq!(flat.dense_present, vec![0b0000_1001, 0b0000_0000]); + // 12 haps, each contributing exactly 1 combined presence bit. + assert_eq!(*flat.dense_present_off.last().unwrap(), n as i64); + assert_eq!( + flat.dense_present.len(), + (*flat.dense_present_off.last().unwrap() as usize).div_ceil(8) + ); + + // The set bits land exactly where expected (haps 0 and 3), and the + // trailing-byte haps are unset — proving no shift/corruption. + for h in 0..n { + let want = h == 0 || h == 3; + let got = genoray_core::bits_get_bit(&flat.dense_present, h); + assert_eq!(got, want, "hap {h} presence bit"); + } + } } diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py index 2ea69838..7ffbfc2a 100644 --- a/tests/dataset/test_svar2_readbound_haps.py +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -176,3 +176,117 @@ def test_readbound_matches_union_oracle_with_shifts(svar2_store): assert np.array_equal( np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") ) + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key), +# so split_to_flat's snp-block concatenation + snp-before-indel window ordering +# are exercised with real data. genoray dense-encodes a SNP when +# dense_bits (POS_BITS + 2 + n_samples*ploidy) < var_key_bits (34*x_calls); with +# 2 samples x ploidy 2 (np=4) that's 38 < 34*x, i.e. any SNP carried by >=2 +# haplotypes goes dense. The SNP@10 below is carried by 3 haps -> dense/snp. +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): + """A SNP routed into dense/snp must reconstruct byte-identically. + + Also sanity-checks (before asserting parity) that the SNP actually landed in + dense/snp — i.e. ``find_ranges``' ``dense_snp_range`` is a non-empty window + for a region covering it — so this test genuinely exercises split_to_flat's + snp-block path rather than silently falling back to the var_key channel. + """ + import genoray + + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._svar2_store_py import build_readbound_haps + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a + # region spanning it has a non-empty dense_snp window. + d = sv.find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + indel_win = int(dense_indel_range[0, 1] - dense_indel_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + # Non-triviality: dense/indel is also populated (INS@7 + DEL@12), so the + # combined window mixes snp and indel entries (concatenation under test). + assert indel_win >= 1, dense_indel_range.tolist() + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + + assert np.array_equal(np.asarray(oracle.offsets), np.asarray(rb.offsets)) + assert np.array_equal( + np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") + ) From 161d2e7b259be6fff37901e7d6aecfb94a61f4c6 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 11:45:21 -0700 Subject: [PATCH 031/108] feat(rust): read-bound svar2 track re-alignment kernel Adds shift_and_realign_tracks_from_svar2_readbound, the track analog of Task 4's reconstruct_haplotypes_from_svar2_readbound: gathers off a query-only genoray Svar2Store reader (no interval-search-tree rebuild, no dense-union rebuild) via gather_haps_readbound, marshals through svar2::split_to_flat, then reuses the byte-validated shift_and_realign_tracks_from_svar2 kernel unchanged. Adds build_readbound_tracks (reproduces the union oracle's per-region -> per-query track expansion) and a parity test exercising var_key, dense/snp, and dense/indel channels; all 11 svar2 track/readbound tests pass. Co-Authored-By: Claude Opus 4.8 --- .../genvarloader/_dataset/_svar2_store_py.py | 102 ++++++++- src/ffi/mod.rs | 161 ++++++++++++++ src/lib.rs | 4 + tests/dataset/test_svar2_readbound_tracks.py | 201 ++++++++++++++++++ 4 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 tests/dataset/test_svar2_readbound_tracks.py diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py index 962be3fc..ab445251 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -18,7 +18,11 @@ import numpy as np from .._flat import _Flat -from ..genvarloader import Svar2Store, reconstruct_haplotypes_from_svar2_readbound +from ..genvarloader import ( + Svar2Store, + reconstruct_haplotypes_from_svar2_readbound, + shift_and_realign_tracks_from_svar2_readbound, +) if TYPE_CHECKING: from genoray import SparseVar2 @@ -106,3 +110,99 @@ def build_readbound_haps( return cast( "Ragged[np.bytes_]", _Flat.from_offsets(out_data, shape, out_offsets).view("S1") ) + + +def build_readbound_tracks( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R + tracks: "NDArray[np.float32]", # flat per-REGION track buffer + track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks + params: "NDArray[np.float64]", + strategy_id: int, + base_seed: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + parallel: bool = False, +) -> "Ragged[np.float32]": + """Realign the full-cohort tracks over ``regions`` via the read-bound kernel. + + Mirrors ``SparseVar2Source.realign_tracks``'s signature/return shape exactly + (query order region-major, sample-minor: ``q = r*S + s``), but drives + ``SparseVar2.find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``overlap_batch``'s eager per-region dense union. + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2.find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + + # `tracks`/`track_offsets` are per-REGION (R of them), but the kernel reads + # `track_offsets` by `query` (= r*S+s) — expand the R track windows to R*S + # by repeating each region's window S times (mirrors + # `SparseVar2Source.realign_tracks`, `_svar2_source.py:108-114`). + t = np.asarray(tracks, np.float32) + toff = np.asarray(track_offsets, np.int64) + tracks_rs = ( + np.concatenate([t[toff[r] : toff[r + 1]] for r in range(R) for _ in range(S)]) + if R + else t + ) + lengths = np.repeat(np.diff(toff), S) + track_offsets_rs = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + out_data, out_offsets = shift_and_realign_tracks_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + np.ascontiguousarray(tracks_rs, np.float32), + track_offsets_rs, + np.ascontiguousarray(params, np.float64), + np.int64(strategy_id), + np.uint64(base_seed), + parallel, + ) + + shape = (R, S, P, None) + return cast("Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets)) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 09665390..23b48f95 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1050,6 +1050,167 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) } +/// Read-bound SVAR2 track re-alignment: gather off a query-only genoray +/// `Svar2Store` reader with NO interval-search-tree rebuild and NO dense-union +/// rebuild (`genoray_core::query::gather_haps_readbound`), marshal the split +/// result into the flat layout via [`crate::svar2::split_to_flat`], then reuse +/// the byte-validated [`shift_and_realign_tracks_from_svar2`] kernel unchanged — +/// one FFI crossing, byte-identical to the union-path oracle. +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range`/`region_bounds` +/// argument semantics (the per-query outputs of `SparseVar2.find_ranges`, +/// flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_tracks`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + shifts: PyReadonlyArray2, + tracks: PyReadonlyArray1, + track_offsets: PyReadonlyArray1, + params: PyReadonlyArray1, + strategy_id: i64, + base_seed: u64, + parallel: bool, +) -> PyResult<(Bound<'py, PyArray1>, Bound<'py, PyArray1>)> { + use crate::svar2; + use crate::tracks; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let shifts_a = shifts.as_array(); + let ploidy = shifts_a.ncols(); + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end). + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize, r[1] as usize)) + .collect() + }; + let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); + let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); + let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); + let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + + let tracks_a = tracks.as_array(); + let track_offsets_a = track_offsets.as_array(); + let params_a = params.as_array(); + + let (out_data, out_offsets_vec) = py.detach(move || { + let br = genoray_core::query::gather_haps_readbound( + reader, + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + // Step 1: size via the same two-source diff core the union path uses. + let diffs = svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ); + + // Step 2: per-haplotype output lengths and prefix-sum offsets — tracks + // always size to ref_len + diff (no `output_length` override). + let n_work = n_q * ploidy; + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let ref_len = (regions[[query, 2]] - regions[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + let len: i64 = (ref_len + diff).max(0); + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = Array1::::zeros(total); + + // Step 4: realign — reuse the byte-validated union-path kernel unchanged, + // now fed the read-bound gather's flat channels. + tracks::shift_and_realign_tracks_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions.view(), + shifts_a, + numpy::ndarray::ArrayView1::from(flat.vk_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_key.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_off.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_key.as_slice()), + dense_range_a, + numpy::ndarray::ArrayView1::from(flat.dense_present.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_present_off.as_slice()), + numpy::ndarray::ArrayView1::from(lut_bytes.as_slice()), + numpy::ndarray::ArrayView1::from(lut_off.as_slice()), + tracks_a, + track_offsets_a, + params_a, + strategy_id, + base_seed, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) +} + /// Fused SVAR2 two-source track shift+realign: merge each hap's `var_key` ⋈ `dense` /// channels and decode via `svar2-codec` inline, sizing and allocating the output /// buffer in Rust — one FFI crossing, mirrors `reconstruct_haplotypes_from_svar2` diff --git a/src/lib.rs b/src/lib.rs index ef02fedf..d303c895 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,6 +67,10 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { ffi::shift_and_realign_tracks_from_svar2, m )?)?; + m.add_function(wrap_pyfunction!( + ffi::shift_and_realign_tracks_from_svar2_readbound, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review diff --git a/tests/dataset/test_svar2_readbound_tracks.py b/tests/dataset/test_svar2_readbound_tracks.py new file mode 100644 index 00000000..71a015ea --- /dev/null +++ b/tests/dataset/test_svar2_readbound_tracks.py @@ -0,0 +1,201 @@ +"""Parity test for the read-bound SVAR2 track re-alignment kernel (Task 5). + +Oracle: ``SparseVar2Source.realign_tracks`` (genoray ``overlap_batch``, eager +dense-union path). Under test: ``build_readbound_tracks`` (genoray +``find_ranges`` + one Rust FFI call via +``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> +the SAME validated ``shift_and_realign_tracks_from_svar2`` kernel the oracle +uses). + +Both paths must be byte-identical: same offsets, same (NaN-equal) data. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G, low-carrier, +# routes to var_key), INS@6 (C>CAT), SNP@9 (G>C, carried by 3 haps -> dense/snp +# per the cost model used in test_svar2_readbound_haps.py), DEL@11 (GTA>G, +# ilen -2). Exercises both var_key and dense/snp + dense/indel channels. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_tracks") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _synthetic_track_inputs(regions, seed=0): + """Per-region flat f32 tracks + (R+1) offsets, plus a known-valid + (params, strategy_id) combo mirroring tests/test_svar2_realign_tracks.py.""" + rng = np.random.default_rng(seed) + lengths = [e - s for s, e in regions] + toff = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + tracks = rng.random(int(toff[-1])).astype(np.float32) + strategy_id = 0 # irrelevant for insertion-fill in this test + params = np.zeros(1, np.float64) + base_seed = 0 + return tracks, toff, params, strategy_id, base_seed + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + dense-SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the variant boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_tracks_match_union_oracle(svar2_store, regions): + import genoray + + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._svar2_store_py import build_readbound_tracks + + contig = "chr1" + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # Self-verify the fixture routes the >=2-carrier SNP@9 to dense/snp (mixed + # with the dense indels) so this parity test genuinely exercises the dense + # path for tracks. Without this, a future cost-model change could silently + # demote the SNP to var_key and the test would still pass while covering less. + d = sv.find_ranges(contig, [0], [40], samples=None) + snp_win = int( + np.asarray(d["dense_snp_range"])[0, 1] - np.asarray(d["dense_snp_range"])[0, 0] + ) + indel_win = int( + np.asarray(d["dense_indel_range"])[0, 1] + - np.asarray(d["dense_indel_range"])[0, 0] + ) + assert snp_win >= 1 and indel_win >= 1, ( + f"fixture must populate both dense channels; got dense_snp_range=" + f"{np.asarray(d['dense_snp_range']).tolist()}, dense_indel_range=" + f"{np.asarray(d['dense_indel_range']).tolist()}" + ) + + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + + union = SparseVar2Source(sv).realign_tracks( + contig, regions, tracks, toff, params, strat, seed, shifts=None, parallel=False + ) + rb = build_readbound_tracks( + sv, + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=None, + parallel=False, + ) + + union_offsets = np.asarray(union.offsets) + rb_offsets = np.asarray(rb.offsets) + assert np.array_equal(union_offsets, rb_offsets), ( + f"offsets mismatch: union={union_offsets.tolist()} rb={rb_offsets.tolist()}" + ) + + union_data = np.asarray(union.data) + rb_data = np.asarray(rb.data) + if not np.allclose(union_data, rb_data, equal_nan=True): + R = len(regions) + n_q = R * S + for h in range(n_q * P): + s0, e0 = int(union_offsets[h]), int(union_offsets[h + 1]) + s1, e1 = int(rb_offsets[h]), int(rb_offsets[h + 1]) + a = union_data[s0:e0] + b = rb_data[s1:e1] + if not np.allclose(a, b, equal_nan=True): + pytest.fail(f"data mismatch at hap {h}: union={a!r} rb={b!r}") + pytest.fail("data mismatch but no single hap slice differed (offset bug?)") + + +def test_readbound_tracks_match_union_oracle_with_shifts(svar2_store): + """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" + import genoray + + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._svar2_store_py import build_readbound_tracks + + contig = "chr1" + regions = [(0, 40), (5, 20)] + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + n_q = len(regions) * S + + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + + rng = np.random.default_rng(1) + shifts = rng.integers(-2, 3, size=(n_q, P), dtype=np.int32) + + union = SparseVar2Source(sv).realign_tracks( + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=shifts, + parallel=False, + ) + rb = build_readbound_tracks( + sv, + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=shifts, + parallel=False, + ) + + assert np.array_equal(np.asarray(union.offsets), np.asarray(rb.offsets)) + assert np.allclose(np.asarray(union.data), np.asarray(rb.data), equal_nan=True) From d17c7690fc17f03a9afec44bffe6ebc8d57f8478 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 13:04:41 -0700 Subject: [PATCH 032/108] feat(rust): read-bound svar2 variants/variant-windows decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decode_variants_from_split merges each hap's vk + present-dense (reusing split_to_flat + merge_hap) and decodes each key via decode_alt into a RaggedVariants SoA (pos, ilen, alt bytes + str offsets, per-hap var offsets), mirroring genoray decode_hap (no overlap/clip filter — the gather already restricts to overlapping variants). New FFI decode_variants_from_svar2_readbound + build_readbound_variants helper. Parity vs genoray decode oracle (SNP/INS/DEL + dense_snp) byte-identical. Co-Authored-By: Claude Opus 4.8 --- .../genvarloader/_dataset/_svar2_store_py.py | 75 +++++++ src/ffi/mod.rs | 84 +++++++ src/lib.rs | 4 + src/svar2/mod.rs | 131 +++++++++++ .../dataset/test_svar2_readbound_variants.py | 206 ++++++++++++++++++ 5 files changed, 500 insertions(+) create mode 100644 tests/dataset/test_svar2_readbound_variants.py diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py index ab445251..49e916ca 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -20,6 +20,7 @@ from .._flat import _Flat from ..genvarloader import ( Svar2Store, + decode_variants_from_svar2_readbound, reconstruct_haplotypes_from_svar2_readbound, shift_and_realign_tracks_from_svar2_readbound, ) @@ -29,6 +30,8 @@ from numpy.typing import NDArray from seqpro.rag import Ragged + from ._rag_variants import RaggedVariants + def build_readbound_haps( svar2: "SparseVar2", @@ -206,3 +209,75 @@ def build_readbound_tracks( shape = (R, S, P, None) return cast("Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets)) + + +def build_readbound_variants( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R +) -> "RaggedVariants": + """Decode the full-cohort overlapping variants over ``regions`` via the + read-bound kernel. + + Mirrors ``SparseVar2.decode``'s return shape exactly (region-major, + sample-minor: ``q = r*S + s``), but drives ``SparseVar2.find_ranges`` + (search-only, no dense union) + one Rust FFI call instead of + ``decode_batch``'s eager per-region dense union. Unlike + ``build_readbound_haps``/``build_readbound_tracks`` there is no reconstruct + sizing pass and no ``shifts``/``ref``/``pad_char`` — decoding a hap's merged + variant set has no output-length dependency on the query region bounds (no + overlap/clip filter; the gather already restricts to overlapping variants). + """ + from ._rag_variants import RaggedVariants + + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2.find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + pos, ilen, alt_bytes, str_off, var_off = decode_variants_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + P, + ) + + from seqpro.rag import Ragged + + shape = (R, S, P, None) + pos_r = Ragged.from_offsets(pos, shape, var_off) + ilen_r = Ragged.from_offsets(ilen, shape, var_off) + alt_r = Ragged.from_offsets( + alt_bytes.view("S1"), shape, var_off, str_offsets=str_off + ) + + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 23b48f95..8cdd54c7 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1211,6 +1211,90 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) } +/// Read-bound SVAR2 variants decode: gather off a query-only genoray `Svar2Store` +/// reader with NO interval-search-tree rebuild and NO dense-union rebuild +/// (`genoray_core::query::gather_haps_readbound`), then decode each hap's merged +/// `var_key ⋈ dense` keys via [`crate::svar2::decode_variants_from_split`] — one +/// FFI crossing, mirroring genoray's `decode_hap` (no overlap/clip filter; the +/// gather already restricts to overlapping variants). +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range` argument semantics +/// (the per-query outputs of `SparseVar2.find_ranges`, flattened region-major, +/// sample-minor). `ploidy` is passed explicitly (there is no `shifts` array to +/// infer it from here). Returns the `RaggedVariants` SoA: `(pos, ilen, alt_bytes, +/// str_off, var_off)`; see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_variants`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn decode_variants_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + ploidy: usize, +) -> PyResult<( + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, +)> { + use crate::svar2; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize, r[1] as usize)) + .collect() + }; + let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); + let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); + let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); + let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + + let soa = py.detach(move || { + let br = genoray_core::query::gather_haps_readbound( + reader, + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + svar2::decode_variants_from_split(&br, &lut_bytes, &lut_off) + }); + + Ok(( + Array1::from_vec(soa.pos).into_pyarray(py), + Array1::from_vec(soa.ilen).into_pyarray(py), + Array1::from_vec(soa.alt_bytes).into_pyarray(py), + Array1::from_vec(soa.str_off).into_pyarray(py), + Array1::from_vec(soa.var_off).into_pyarray(py), + )) +} + /// Fused SVAR2 two-source track shift+realign: merge each hap's `var_key` ⋈ `dense` /// channels and decode via `svar2-codec` inline, sizing and allocating the output /// buffer in Rust — one FFI crossing, mirrors `reconstruct_haplotypes_from_svar2` diff --git a/src/lib.rs b/src/lib.rs index d303c895..ddfcc6b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,6 +71,10 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { ffi::shift_and_realign_tracks_from_svar2_readbound, m )?)?; + m.add_function(wrap_pyfunction!( + ffi::decode_variants_from_svar2_readbound, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 444f1c87..97da8d72 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -236,6 +236,89 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { } } +/// Per-hap decoded variant SoA, matching genoray's `decode_hap` output layout — +/// see [`decode_variants_from_split`]. +pub struct VariantsSoa { + /// Per-variant (flat, hap-major) 0-based start position. + pub pos: Vec, + /// Per-variant (flat) `ilen` (ALT len - 1 for inline/lookup keys, negative + /// deletion length for pure-deletion keys). + pub ilen: Vec, + /// Concatenated ALT bytes for all variants (pure-deletion ALT is empty). + pub alt_bytes: Vec, + /// Per-variant byte offsets into `alt_bytes`, len = `total_variants + 1`. + pub str_off: Vec, + /// Per-hap offsets into `pos`/`ilen`/`str_off`'s variant axis, len = `H + 1`. + pub var_off: Vec, +} + +/// Per-hap decode of a read-bound split into the [`VariantsSoa`], mirroring +/// genoray's `decode_hap`: merge each hap's `vk` with its present-dense entries +/// (position-sorted, `vk` before dense, snp before indel — see [`merge_hap`] via +/// [`split_to_flat`]), then decode each merged key via [`decode_alt`]. There is +/// NO overlap/clip filter here — the gather already restricts to overlapping +/// variants, unlike [`hap_diffs_svar2`]/reconstruct's ref_idx-consumed clipping, +/// which only matters for sizing a fixed-length output buffer. +pub fn decode_variants_from_split( + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], +) -> VariantsSoa { + let flat = split_to_flat(br); + let ploidy = br.ploidy; + let n_q = br.n_regions; + let h_count = n_q * ploidy; + + let mut pos: Vec = Vec::new(); + let mut ilen: Vec = Vec::new(); + let mut alt_bytes: Vec = Vec::new(); + let mut str_off: Vec = vec![0]; + let mut var_off: Vec = Vec::with_capacity(h_count + 1); + var_off.push(0); + + for h in 0..h_count { + let q = h / ploidy; + let vk_lo = flat.vk_off[h] as usize; + let vk_hi = flat.vk_off[h + 1] as usize; + let ds = flat.dense_range[q * 2] as usize; + let de = flat.dense_range[q * 2 + 1] as usize; + let base_bit = flat.dense_present_off[h] as usize; + let present_bit = |k: usize| -> bool { + let bit = base_bit + k; + (flat.dense_present[bit / 8] >> (bit % 8)) & 1 == 1 + }; + + let merged = merge_hap( + &flat.vk_pos, + &flat.vk_key, + vk_lo, + vk_hi, + &flat.dense_pos, + &flat.dense_key, + ds, + de, + present_bit, + ); + + for &(p, key) in &merged { + let (il, alt) = decode_alt(key, lut_bytes, lut_off); + pos.push(p as i32); + ilen.push(il as i32); + alt_bytes.extend_from_slice(&alt); + str_off.push(alt_bytes.len() as i64); + } + var_off.push(pos.len() as i64); + } + + VariantsSoa { + pos, + ilen, + alt_bytes, + str_off, + var_off, + } +} + #[cfg(test)] mod tests { use super::*; @@ -443,4 +526,52 @@ mod tests { assert_eq!(got, want, "hap {h} presence bit"); } } + + #[test] + fn test_decode_variants_from_split_merges_and_decodes() { + use genoray_core::query::KeyRef; + + // 1 region, 1 sample (read-bound), ploidy 1 -> 1 hap. var_key: SNP at + // pos 5 (inline ALT "T"). dense_snp: one entry at pos 8, PRESENT for + // this hap (inline ALT "G"). dense_indel: one entry at pos 12, PRESENT + // for this hap (pure DEL, ilen -2, empty ALT). + let vk_key = svar2_codec::encode_alt_inline(b"T", 0); + let dense_snp_key = svar2_codec::encode_alt_inline(b"G", 0); + let dense_indel_key = svar2_codec::encode_pure_del(-2); + + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 5, + key: vk_key, + }], + vk_off: vec![0, 1], + dense_snp: vec![KeyRef { + position: 8, + key: dense_snp_key, + }], + dense_snp_range: vec![(0, 1)], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 12, + key: dense_indel_key, + }], + dense_indel_range: vec![(0, 1)], + dense_indel_present: vec![0b1], + dense_indel_present_off: vec![0, 1], + }; + + let soa = decode_variants_from_split(&br, &[], &[0]); + + // Position-sorted: var_key SNP@5, dense/snp@8, dense/indel@12. + assert_eq!(soa.var_off, vec![0, 3]); + assert_eq!(soa.pos, vec![5, 8, 12]); + assert_eq!(soa.ilen, vec![0, 0, -2]); + assert_eq!(soa.alt_bytes, b"TG".to_vec()); + // Pure-del ALT is empty -> the 3rd variant's [start, end) is [2, 2). + assert_eq!(soa.str_off, vec![0, 1, 2, 2]); + } } diff --git a/tests/dataset/test_svar2_readbound_variants.py b/tests/dataset/test_svar2_readbound_variants.py new file mode 100644 index 00000000..759413b5 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_variants.py @@ -0,0 +1,206 @@ +"""Parity test for the read-bound SVAR2 variants decode kernel (Task 6). + +Oracle: ``SparseVar2.decode`` (genoray's own record-``Ragged`` decode, no +overlap/clip filter — the gather already restricts to overlapping variants). +Under test: ``build_readbound_variants`` (genoray ``find_ranges`` + one Rust FFI +call via ``genoray_core::query::gather_haps_readbound`` -> per-hap ``merge_hap`` + +``decode_alt``, mirroring genoray's ``decode_hap``). + +Both paths decode the SAME full cohort (``samples=None``), so the flat per-hap +(pos, ilen, alt) arrays and the shared variant-axis offsets must be identical. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_variants") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _assert_variants_match(oracle, rb) -> None: + """Compare a genoray ``decode()`` record Ragged (pos/ilen/allele) against a + ``RaggedVariants`` (alt/start/ilen) built by the read-bound kernel.""" + oracle_off = np.asarray(oracle.offsets) + rb_off = np.asarray(rb.offsets) + assert np.array_equal(oracle_off, rb_off), ( + f"variant-axis offsets mismatch: oracle={oracle_off.tolist()} " + f"rb={rb_off.tolist()}" + ) + + pos_match = oracle["pos"].to_ak().to_list() == rb["start"].to_ak().to_list() + ilen_match = oracle["ilen"].to_ak().to_list() == rb["ilen"].to_ak().to_list() + allele_match = oracle["allele"].to_ak().to_list() == rb["alt"].to_ak().to_list() + + if pos_match and ilen_match and allele_match: + return + + # Locate the first mismatching (hap, variant) for debuggability. + n_hap = len(oracle_off) - 1 + o_pos, r_pos = oracle["pos"].to_ak().to_list(), rb["start"].to_ak().to_list() + o_ilen, r_ilen = oracle["ilen"].to_ak().to_list(), rb["ilen"].to_ak().to_list() + o_alt, r_alt = oracle["allele"].to_ak().to_list(), rb["alt"].to_ak().to_list() + for h in range(n_hap): + if (o_pos[h], o_ilen[h], o_alt[h]) != (r_pos[h], r_ilen[h], r_alt[h]): + pytest.fail( + f"mismatch at hap {h}: " + f"oracle=(pos={o_pos[h]}, ilen={o_ilen[h]}, alt={o_alt[h]}) " + f"rb=(pos={r_pos[h]}, ilen={r_ilen[h]}, alt={r_alt[h]})" + ) + pytest.fail("mismatch but no single hap differed (offset/field bug?)") + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_variants_match_decode_oracle(svar2_store, regions): + import genoray + + from genvarloader._dataset._svar2_store_py import build_readbound_variants + + contig = "chr1" + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + oracle = sv.decode(contig, regions) + rb = build_readbound_variants(sv, contig, regions) + + _assert_variants_match(oracle, rb) + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key), +# so split_to_flat's snp-block concatenation + snp-before-indel window ordering +# are exercised with real data (see test_svar2_readbound_haps.py's identical +# fixture recipe for the routing rationale). +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_variants_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_readbound_variants_dense_snp_match_decode_oracle(svar2_store_dense_snp): + """A SNP routed into dense/snp must decode identically to the oracle. + + Also sanity-checks (before asserting parity) that the SNP actually landed in + dense/snp — i.e. ``find_ranges``' ``dense_snp_range`` is a non-empty window + for a region covering it — so this test genuinely exercises split_to_flat's + snp-block path rather than silently falling back to the var_key channel. + """ + import genoray + + from genvarloader._dataset._svar2_store_py import build_readbound_variants + + contig = "chr1" + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a + # region spanning it has a non-empty dense_snp window. + d = sv.find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + indel_win = int(dense_indel_range[0, 1] - dense_indel_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + # Non-triviality: dense/indel is also populated (INS@7 + DEL@12), so the + # combined window mixes snp and indel entries (concatenation under test). + assert indel_win >= 1, dense_indel_range.tolist() + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + oracle = sv.decode(contig, regions) + rb = build_readbound_variants(sv, contig, regions) + + _assert_variants_match(oracle, rb) From b186d2375a6eb5de52c73c509ad2416892466db4 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 19:54:17 -0700 Subject: [PATCH 033/108] feat(rust): read-bound svar2 per-hap diffs FFI (for jitter shift computation) Adds ffi::hap_diffs_from_svar2_readbound, which reuses the first half of reconstruct_haplotypes_from_svar2_readbound's gather (gather_haps_readbound -> split_to_flat -> hap_diffs_svar2) but stops before sizing/allocating/ reconstructing, returning just the (n_q, ploidy) diffs. The SVAR2 dataset read path needs these diffs before reconstruction to compute random jitter shifts, mirroring how the SVAR1 path derives shifts from diffs in _prepare_request. Also adds the build_readbound_diffs Python helper mirroring build_readbound_haps' gather-input construction, and a parity test asserting the exposed diffs match the diffs implied by the read-bound haplotype reconstruction (len(hap) - ref_len per hap). --- .../genvarloader/_dataset/_svar2_store_py.py | 68 ++++++ src/ffi/mod.rs | 100 +++++++++ src/lib.rs | 1 + tests/dataset/test_svar2_readbound_diffs.py | 207 ++++++++++++++++++ 4 files changed, 376 insertions(+) create mode 100644 tests/dataset/test_svar2_readbound_diffs.py diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py index 49e916ca..aece4d55 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -21,6 +21,7 @@ from ..genvarloader import ( Svar2Store, decode_variants_from_svar2_readbound, + hap_diffs_from_svar2_readbound, reconstruct_haplotypes_from_svar2_readbound, shift_and_realign_tracks_from_svar2_readbound, ) @@ -115,6 +116,73 @@ def build_readbound_haps( ) +def build_readbound_diffs( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R +) -> "NDArray[np.int32]": + """Per-hap ilen diffs over ``regions`` via the read-bound kernel, WITHOUT + reconstructing haplotypes. + + Mirrors ``build_readbound_haps``'s query order (region-major, sample-minor: + ``q = r*S + s``) and gather-input construction, but stops after + ``svar2::hap_diffs_svar2`` — no ``ref``/``pad_char``/``shifts``/reconstruct + pass. Used by the dataset read path to compute random jitter shifts from + diffs BEFORE reconstructing (mirrors how the SVAR1 path derives shifts from + diffs in ``_prepare_request``). + + Returns the ``(R*S, P)`` diffs array (query order region-major, sample-minor). + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2.find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + diffs = hap_diffs_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + P, + ) + + return cast("NDArray[np.int32]", diffs) + + def build_readbound_tracks( svar2: "SparseVar2", contig: str, diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 8cdd54c7..48b4f9a5 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1050,6 +1050,106 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) } +/// Read-bound SVAR2 per-hap ilen diffs: the same gather +/// (`genoray_core::query::gather_haps_readbound` + [`crate::svar2::split_to_flat`]) as +/// [`reconstruct_haplotypes_from_svar2_readbound`], but stops after +/// [`crate::svar2::hap_diffs_svar2`] and returns just the `(n_q, ploidy)` diffs — +/// no reconstruct sizing/allocation/kernel pass. Used by the dataset read path to +/// compute random jitter shifts from diffs BEFORE reconstructing (mirrors how the +/// SVAR1 path derives shifts from diffs in `_prepare_request`). +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range`/`region_bounds` +/// argument semantics (the per-query outputs of `SparseVar2.find_ranges`, +/// flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_diffs`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn hap_diffs_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + ploidy: usize, +) -> PyResult>> { + use crate::svar2; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end) — matches the + // reconstruct-readbound FFI's convention. + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize, r[1] as usize)) + .collect() + }; + let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); + let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); + let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); + let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + + let diffs = py.detach(move || { + let br = genoray_core::query::gather_haps_readbound( + reader, + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ) + }); + + Ok(diffs.into_pyarray(py)) +} + /// Read-bound SVAR2 track re-alignment: gather off a query-only genoray /// `Svar2Store` reader with NO interval-search-tree rebuild and NO dense-union /// rebuild (`genoray_core::query::gather_haps_readbound`), marshal the split diff --git a/src/lib.rs b/src/lib.rs index ddfcc6b2..80338888 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,7 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { ffi::reconstruct_haplotypes_from_svar2_readbound, m )?)?; + m.add_function(wrap_pyfunction!(ffi::hap_diffs_from_svar2_readbound, m)?)?; m.add_function(wrap_pyfunction!( ffi::reconstruct_annotated_haplotypes_fused, m diff --git a/tests/dataset/test_svar2_readbound_diffs.py b/tests/dataset/test_svar2_readbound_diffs.py new file mode 100644 index 00000000..e36d42c7 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_diffs.py @@ -0,0 +1,207 @@ +"""Parity test for the read-bound SVAR2 per-hap diffs kernel (Task 7a). + +Oracle: the diffs implied by the read-bound HAPLOTYPE reconstruction +(``build_readbound_haps``) — per (region, hap), ``len(haplotype) - (region_end - +region_start)`` is exactly the ilen diff the reconstruct kernel computed internally +via ``svar2::hap_diffs_svar2`` before sizing/writing the output. Under test: +``build_readbound_diffs`` (same gather, but stops after ``hap_diffs_svar2`` and +returns just the diffs — no reconstruct pass). + +This proves the newly-exposed diffs FFI matches what the reconstruct kernel uses +internally, without needing a second independent oracle. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# Same fixtures as tests/dataset/test_svar2_readbound_haps.py: 40 bp reference +# (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), DEL@11 +# (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_diffs") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key) — +# see test_svar2_readbound_haps.py for the routing rationale. +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_diffs_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) -> np.ndarray: + """Diffs implied by the read-bound haplotype reconstruction: per (region, + hap), ``len(haplotype) - (region_end - region_start)``. + + Query order matches ``build_readbound_haps``/``build_readbound_diffs``: + region-major, sample-minor (``q = r*S + s``), hap-minor within a query. + """ + from genvarloader._dataset._svar2_store_py import build_readbound_haps + + S, P = sv.n_samples, sv.ploidy + R = len(regions) + + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + offsets = np.asarray(rb.offsets) # (R*S*P + 1,) + + lengths = np.diff(offsets) # (R*S*P,) + ref_lens = np.repeat( + np.asarray([e - s for s, e in regions], np.int64), S * P + ) # (R*S*P,) region-major, sample-minor, hap-minor + diffs = (lengths - ref_lens).astype(np.int32) + return diffs.reshape(R * S, P) + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_diffs_matches_implied_haps(svar2_store, regions): + import genoray + + from genvarloader._dataset._svar2_store_py import build_readbound_diffs + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + implied = _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) + diffs = np.asarray(build_readbound_diffs(sv, contig, regions)) + + assert diffs.shape == implied.shape + assert np.array_equal(diffs, implied), ( + f"diffs mismatch: implied={implied.tolist()} diffs={diffs.tolist()}" + ) + + +def test_readbound_diffs_dense_snp_matches_implied_haps(svar2_store_dense_snp): + """A SNP routed into dense/snp must diff-clip identically to what the + reconstruct kernel implies.""" + import genoray + + from genvarloader._dataset._svar2_store_py import build_readbound_diffs + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table. + d = sv.find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + implied = _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) + diffs = np.asarray(build_readbound_diffs(sv, contig, regions)) + + assert diffs.shape == implied.shape + assert np.array_equal(diffs, implied), ( + f"diffs mismatch: implied={implied.tolist()} diffs={diffs.tolist()}" + ) From 2db8f49c9f9927be79501dacfecb960bbf1e2273 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 20:38:08 -0700 Subject: [PATCH 034/108] feat(dataset): Svar2Haps reconstructor + svar2 read dispatch (haplotypes, variants) Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_impl.py | 8 + python/genvarloader/_dataset/_open.py | 42 +- python/genvarloader/_dataset/_svar2_haps.py | 606 ++++++++++++++++++++ tests/dataset/test_svar2_dataset.py | 186 ++++++ 4 files changed, 830 insertions(+), 12 deletions(-) create mode 100644 python/genvarloader/_dataset/_svar2_haps.py create mode 100644 tests/dataset/test_svar2_dataset.py diff --git a/python/genvarloader/_dataset/_impl.py b/python/genvarloader/_dataset/_impl.py index ee2e7927..862e51a9 100644 --- a/python/genvarloader/_dataset/_impl.py +++ b/python/genvarloader/_dataset/_impl.py @@ -102,6 +102,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[MaybeRSEQ, MaybeRTRK]: ... @staticmethod @overload @@ -120,6 +121,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[RaggedSeqs, MaybeRTRK]: ... @staticmethod def open( @@ -137,6 +139,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[MaybeRSEQ, MaybeRTRK]: """Open a dataset from a path. If no reference genome is provided, the dataset cannot yield sequences. Will initialize the dataset such that it will return tracks and haplotypes (reference sequences if no genotypes) if possible. @@ -179,6 +182,10 @@ def open( Override the recorded SVAR location. Use when the original SVAR has moved and the dataset cannot find it via the stored relative/absolute path or by sibling discovery. + svar2 + Override the recorded ``.svar2`` location. Use when the original + ``.svar2`` store has moved and the dataset cannot find it via the + stored relative/absolute path or by sibling discovery. """ from ._open import OpenRequest @@ -196,6 +203,7 @@ def open( splice_info=splice_info, var_filter=var_filter, svar=svar, + svar2=svar2, ).resolve() def with_settings( diff --git a/python/genvarloader/_dataset/_open.py b/python/genvarloader/_dataset/_open.py index c720a266..f6f33747 100644 --- a/python/genvarloader/_dataset/_open.py +++ b/python/genvarloader/_dataset/_open.py @@ -56,6 +56,7 @@ class OpenRequest: splice_info: str | tuple[str, str] | None = None var_filter: Literal["exonic"] | None = None svar: str | Path | None = None + svar2: str | Path | None = None var_fields: list[str] | None = None def resolve(self) -> RaggedDataset: @@ -149,19 +150,36 @@ def _build_seqs( if self._has_genotypes(): if metadata.ploidy is None: raise ValueError("Malformed dataset: found genotypes but not ploidy.") - seqs = Haps.from_path( - path=self.path, - reference=reference, - regions=regions, - samples=metadata.samples, - ploidy=metadata.ploidy, - version=metadata.version, - svar_link=metadata.svar_link, - svar_override=self.svar, - min_af=self.min_af, - max_af=self.max_af, - var_fields=self.var_fields, + svar2_meta = ( + self.path / "genotypes" / "svar2_ranges" / "svar2_meta.json" ) + seqs: Haps | Ref | None + if svar2_meta.exists(): + from ._svar2_haps import Svar2Haps + + seqs = Svar2Haps.from_path( + path=self.path, + reference=reference, + samples=metadata.samples, + ploidy=metadata.ploidy, + svar2_link=metadata.svar2_link, + svar2_override=self.svar2, + contigs=metadata.contigs, + ) + else: + seqs = Haps.from_path( + path=self.path, + reference=reference, + regions=regions, + samples=metadata.samples, + ploidy=metadata.ploidy, + version=metadata.version, + svar_link=metadata.svar_link, + svar_override=self.svar, + min_af=self.min_af, + max_af=self.max_af, + var_fields=self.var_fields, + ) if reference is None: logger.warning( "No reference: dataset only has genotypes but no reference was given." diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py new file mode 100644 index 00000000..73cba72b --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -0,0 +1,606 @@ +"""SVAR2-backed haplotype/variant reconstructor (dataset read dispatch). + +``Svar2Haps`` is a *separate* reconstructor from the SVAR1 :class:`Haps` — the +SVAR1 path is left byte-unchanged. It subclasses :class:`Haps` only so the many +``isinstance(_, Haps)`` / ``case Haps()`` checks throughout the dataset +machinery keep working; every read method is overridden. + +For a query block of ``n_q`` rows, each row ``q = (region r_q, sample slot si_q)`` +with post-jitter bounds ``[start_q, end_q)``, the cache (written by +``_write._write_from_svar2`` under ``genotypes/svar2_ranges/``) is sliced by +fancy-indexing — NO per-read interval search, NO dense-union rebuild — and fed to +the read-bound Rust kernels (``reconstruct_haplotypes_from_svar2_readbound`` / +``decode_variants_from_svar2_readbound`` / ``hap_diffs_from_svar2_readbound``). + +The FFI-input shaping + output wrapping mirror +``_svar2_store_py.build_readbound_*`` exactly; the only difference is the source +of the per-query ranges (this module slices the on-disk cache for the specific +``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2.find_ranges`` over +the full cohort). + +Out of scope for this plan (guarded with ``NotImplementedError``): spliced +output, ``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated +haps, in-kernel reverse-complement, and ``unphased_union``. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Literal, cast + +import numpy as np +from genoray._types import POS_TYPE, V_IDX_TYPE +from numpy.typing import NDArray +from seqpro.rag import Ragged + +from .._flat import _Flat +from .._ragged import RaggedAnnotatedHaps +from .._threads import should_parallelize +from .._utils import lengths_to_offsets +from .._variants._records import RaggedAlleles +from ..genvarloader import ( + Svar2Store, + decode_variants_from_svar2_readbound, + hap_diffs_from_svar2_readbound, + reconstruct_haplotypes_from_svar2_readbound, +) +from ._flat_variants import _FlatVariantWindows +from ._haps import _H, Haps, _Variants +from ._rag_variants import RaggedVariants +from ._reference import Reference +from ._svar2_link import Svar2Link, _resolve_svar2, _verify_svar2_fingerprint + +if TYPE_CHECKING: + from ._splice import SplicePlan + + +@dataclass(slots=True) +class _Svar2Cache: + """The six memmapped ``svar2_ranges/`` arrays (all int64), sliced per query. + + ``vk_*_range`` are ``(R, S, P, 2)`` (per region/sample/ploid byte windows into + the store's var_key tables); ``dense_*_range`` are ``(R, 2)`` (per-region, + sample-independent); ``region_starts`` is ``(R,)`` (write-time starts; kept for + parity/debug, NOT fed to the FFI — the read path uses post-jitter starts); + ``sample_cols`` is ``(S,)`` (selected slot -> original store sample index). + """ + + vk_snp_range: NDArray[np.int64] + vk_indel_range: NDArray[np.int64] + dense_snp_range: NDArray[np.int64] + dense_indel_range: NDArray[np.int64] + region_starts: NDArray[np.int64] + sample_cols: NDArray[np.int64] + + +def _ragged_arange_gather( + data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray, NDArray[np.int64]]: + """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``. + + ``offsets`` has length ``n_rows + 1``; ``perm`` is the new row order + (final row ``i`` == old row ``perm[i]``). Fully vectorized (no Python loop + over rows). Returns ``(new_data, new_offsets)``. + """ + offsets = np.asarray(offsets, np.int64) + lens = np.diff(offsets) + new_lens = lens[perm] + new_off = lengths_to_offsets(new_lens, np.int64) + n = int(new_off[-1]) + if n == 0: + return data[:0].copy(), new_off + within = np.arange(n, dtype=np.int64) - np.repeat(new_off[:-1], new_lens) + src = np.repeat(offsets[perm], new_lens) + within + return data[src], new_off + + +def _ragged_arange_gather_2level( + data: NDArray, + var_off: NDArray[np.integer], + str_off: NDArray[np.integer], + perm: NDArray[np.integer], +) -> tuple[NDArray, NDArray[np.int64], NDArray[np.int64]]: + """Reorder the rows of a 2-level ragged (opaque-string) array by ``perm``. + + ``var_off`` (len ``n_rows + 1``) bounds variants per row; ``str_off`` (len + ``n_variants + 1``) bounds bytes per variant. Returns + ``(new_data, new_var_off, new_str_off)``. Fully vectorized. + """ + var_off = np.asarray(var_off, np.int64) + str_off = np.asarray(str_off, np.int64) + var_lens = np.diff(var_off) + new_var_lens = var_lens[perm] + new_var_off = lengths_to_offsets(new_var_lens, np.int64) + total_vars = int(new_var_off[-1]) + if total_vars == 0: + return data[:0].copy(), new_var_off, np.zeros(1, np.int64) + within_var = np.arange(total_vars, dtype=np.int64) - np.repeat( + new_var_off[:-1], new_var_lens + ) + old_var_idx = np.repeat(var_off[perm], new_var_lens) + within_var + var_byte_len = np.diff(str_off) + new_byte_len = var_byte_len[old_var_idx] + new_str_off = lengths_to_offsets(new_byte_len, np.int64) + nbytes = int(new_str_off[-1]) + if nbytes == 0: + return data[:0].copy(), new_var_off, new_str_off + within_b = np.arange(nbytes, dtype=np.int64) - np.repeat( + new_str_off[:-1], new_byte_len + ) + src = np.repeat(str_off[old_var_idx], new_byte_len) + within_b + return data[src], new_var_off, new_str_off + + +@dataclass(slots=True) +class Svar2Haps(Haps[_H]): + """Read-bound SVAR2 reconstructor. See module docstring.""" + + # New fields must default (they follow base Haps' defaulted fields). + store: "Svar2Store | None" = None + cache: "_Svar2Cache | None" = None + store_contigs: list[str] = field(default_factory=list) + """The .svar2 store's contig names (used to open the store's ContigReaders).""" + ds_contigs: list[str] = field(default_factory=list) + """The dataset's contig names (``regions[:, 0]`` indexes into this).""" + + def __post_init__(self): + # Deliberately does NOT call Haps.__post_init__ (that reads an SVAR1 + # variants table / AF cache which svar2 has no analogue for). Set only + # the init=False fields the base machinery reads. + self.n_variants = self.genotypes.lengths + self.available_var_fields = ["alt", "ilen", "start"] + + # ---- construction ---- + + @classmethod + def from_path( # type: ignore[override] # separate svar2 signature; base returns Haps[RaggedVariants] + cls, + path: Path, + reference: Reference | None, + samples: list[str], + ploidy: int, + svar2_link: Svar2Link | None, + svar2_override: Path | str | None, + contigs: list[str], + kind: type = RaggedVariants, + ) -> "Svar2Haps": + ranges_dir = path / "genotypes" / "svar2_ranges" + with open(ranges_dir / "svar2_meta.json") as f: + meta = json.load(f) + + def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: + return np.memmap( + ranges_dir / name, dtype=np.int64, mode="r", shape=tuple(shape) + ) + + R = int(meta["region_starts"]["shape"][0]) + S = int(meta["vk_snp_range"]["shape"][1]) + P = int(meta["ploidy"]) + if P != ploidy: + raise ValueError(f"svar2 cache ploidy ({P}) != dataset ploidy ({ploidy}).") + + cache = _Svar2Cache( + vk_snp_range=_mm("vk_snp_range.npy", meta["vk_snp_range"]["shape"]), + vk_indel_range=_mm("vk_indel_range.npy", meta["vk_indel_range"]["shape"]), + dense_snp_range=_mm( + "dense_snp_range.npy", meta["dense_snp_range"]["shape"] + ), + dense_indel_range=_mm( + "dense_indel_range.npy", meta["dense_indel_range"]["shape"] + ), + region_starts=_mm("region_starts.npy", meta["region_starts"]["shape"]), + sample_cols=np.load(ranges_dir / "sample_cols.npy"), + ) + + svar2_path = _resolve_svar2(path, svar2_link, svar2_override) + _verify_svar2_fingerprint(svar2_path, svar2_link) + + # Open the query-only store. n_samples must be the store's FULL sample + # count (orig_samples / sample_cols index into it), not len(samples). + from genoray import SparseVar2 + + sv = SparseVar2(str(svar2_path)) + store = Svar2Store(str(svar2_path), sv.contigs, sv.n_samples, sv.ploidy) + + # Minimal base-Haps fields. genotypes carries only the (R, S, P, None) + # shape (so ploidy = shape[-2] and n_variants.shape are available); its + # data is empty (svar2 has no per-region sparse genotype store). + empty_geno = Ragged.from_offsets( + np.empty(0, V_IDX_TYPE), + (R, S, P, None), + np.zeros(R * S * P + 1, np.int64), + ) + empty_alt = RaggedAlleles.from_offsets( + np.empty(0, np.uint8).view("S1"), (0, None), np.zeros(1, np.int64) + ) + dummy_variants = _Variants( + path=svar2_path, + start=np.empty(0, POS_TYPE), + ilen=np.empty(0, np.int32), + ref=None, + alt=empty_alt, + info={}, + ) + + return cls( + path=path, + reference=reference, + variants=dummy_variants, + genotypes=empty_geno, + dosages=None, + kind=cast("type[_H]", kind), + filter=None, + min_af=None, + max_af=None, + store=store, + cache=cache, + store_contigs=list(sv.contigs), + ds_contigs=list(contigs), + ) + + # ---- reconstructor entry ---- + + def __call__( + self, + idx: NDArray[np.integer], + r_idx: NDArray[np.integer], + regions: NDArray[np.int32], + output_length: Literal["ragged", "variable"] | int, + jitter: int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: "SplicePlan | None" = None, + flat: bool = False, + to_rc: "NDArray[np.bool_] | None" = None, + ) -> _H: + self._guard_unsupported(splice_plan) + + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + if issubclass(self.kind, _FlatVariantWindows): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('variant-windows') yet." + ) + # RaggedVariants: RC is applied by the caller (_getitem_unspliced), + # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). + return cast(_H, self._reconstruct_variants(idx, regions)) + + if issubclass(self.kind, RaggedAnnotatedHaps): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('annotated') yet." + ) + + # Haplotypes: RC would need to be folded in-kernel; the read-bound haps + # kernel has no to_rc param, so any real RC is unsupported here. + if to_rc is not None and bool(np.asarray(to_rc).any()): + raise NotImplementedError( + "In-kernel reverse-complement is not supported for svar2 haplotypes." + ) + + haps, *_ = self.get_haps_and_shifts( + idx=idx, + regions=regions, + output_length=output_length, + rng=rng, + deterministic=deterministic, + splice_plan=splice_plan, + to_rc=to_rc, + ) + return cast(_H, haps) + + def get_haps_and_shifts( + self, + idx: NDArray[np.integer], + regions: NDArray[np.integer], + output_length: Literal["ragged", "variable"] | int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: "SplicePlan | None" = None, + to_rc: "NDArray[np.bool_] | None" = None, + ) -> tuple[ + Ragged[np.bytes_], + NDArray[np.intp], + NDArray[np.int32], + NDArray[np.int32], + NDArray[np.int32], + NDArray[np.bool_] | None, + NDArray[np.int64] | None, + ]: + """Reconstruct haplotypes + return the SVAR1-shaped 7-tuple. + + The tracks follow-up (7c) reuses this for the shared shifts/diffs/ + hap_lengths; ``geno_offset_idx`` is a placeholder for svar2 (the cache is + re-sliced from ``idx`` there), and ``keep``/``keep_offsets`` are None. + """ + self._guard_unsupported(splice_plan) + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + lengths = (regions[:, 2] - regions[:, 1]).astype(np.int64) + + groups = self._contig_groups(contig_ids) + + # --- diffs (per contig group, stitched back to (b, P) query order) --- + diffs = np.empty((b, P), np.int32) + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + d = hap_diffs_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + P, + ) + diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) + + hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + + # --- shifts (single rng draw; mirrors Haps._prepare_request) --- + if deterministic or isinstance(output_length, str): + shifts = np.zeros((b, P), np.int32) + else: + max_shift = diffs.clip(min=0) + max_shift = max_shift + (lengths - output_length).clip(min=0)[:, None] + shifts = rng.integers(0, max_shift + 1, dtype=np.int32) + + ffi_out_len = ( + np.int64(-1) if isinstance(output_length, str) else np.int64(output_length) + ) + + # --- reconstruct (per contig group), collect in grouped query order --- + cat_data: list[NDArray[np.uint8]] = [] + cat_hap_lens: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + ref_, ref_offsets = self._ref_for_contig(ci) + g_shifts = np.ascontiguousarray(shifts[qsel], np.int32) + if isinstance(output_length, int): + g_total = int(output_length) * len(qsel) * P + else: + g_total = int(hap_lengths[qsel].sum()) + g_data, g_off = reconstruct_haplotypes_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + g_shifts, + ref_, + ref_offsets, + np.uint8(self.reference.pad_char), # type: ignore[union-attr] # reference guaranteed for haplotypes + ffi_out_len, + should_parallelize(g_total), + ) + cat_data.append(np.asarray(g_data, np.uint8)) + cat_hap_lens.append(np.diff(np.asarray(g_off, np.int64))) + cat_query_order.append(qsel) + + out = self._assemble_haps(cat_data, cat_hap_lens, cat_query_order, b, P) + + geno_offset_idx = np.repeat( + np.asarray(idx, np.intp)[:, None], P, axis=1 + ) # svar2 placeholder; 7c re-slices the cache from idx. + return out, geno_offset_idx, shifts, diffs, hap_lengths, None, None + + # ---- variants ---- + + def _reconstruct_variants( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] + ) -> RaggedVariants: + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + cat_var_lens: list[NDArray[np.int64]] = [] + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_alt: list[NDArray[np.uint8]] = [] + cat_var_bytelen: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + ) + ) + var_off = np.asarray(var_off, np.int64) + str_off = np.asarray(str_off, np.int64) + cat_var_lens.append(np.diff(var_off)) + cat_pos.append(np.asarray(pos, np.int32)) + cat_ilen.append(np.asarray(ilen, np.int32)) + cat_alt.append(np.asarray(alt_bytes, np.uint8)) + cat_var_bytelen.append(np.diff(str_off)) + cat_query_order.append(qsel) + + # Concatenate grouped outputs, then permute hap-rows back to global order. + var_lens = ( + np.concatenate(cat_var_lens) if cat_var_lens else np.zeros(0, np.int64) + ) + grouped_var_off = lengths_to_offsets(var_lens, np.int64) + pos_c = np.concatenate(cat_pos) if cat_pos else np.zeros(0, np.int32) + ilen_c = np.concatenate(cat_ilen) if cat_ilen else np.zeros(0, np.int32) + alt_c = np.concatenate(cat_alt) if cat_alt else np.zeros(0, np.uint8) + var_bytelen = ( + np.concatenate(cat_var_bytelen) + if cat_var_bytelen + else np.zeros(0, np.int64) + ) + grouped_str_off = lengths_to_offsets(var_bytelen, np.int64) + + perm = self._inverse_row_perm(cat_query_order, b, P) + + pos_g, var_off_g = _ragged_arange_gather(pos_c, grouped_var_off, perm) + ilen_g, _ = _ragged_arange_gather(ilen_c, grouped_var_off, perm) + alt_g, alt_var_off_g, alt_str_off_g = _ragged_arange_gather_2level( + alt_c, grouped_var_off, grouped_str_off, perm + ) + + shape = (b, P, None) + pos_r = Ragged.from_offsets(pos_g, shape, var_off_g) + ilen_r = Ragged.from_offsets(ilen_g, shape, var_off_g) + alt_r = Ragged.from_offsets( + alt_g.view("S1"), shape, alt_var_off_g, str_offsets=alt_str_off_g + ) + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r) + + # ---- helpers ---- + + def _guard_unsupported(self, splice_plan: "SplicePlan | None") -> None: + if splice_plan is not None: + raise NotImplementedError( + "Spliced output is not supported for svar2 datasets yet." + ) + if self.filter == "exonic": + raise NotImplementedError( + "var_filter='exonic' (keep mask) is not supported for svar2 yet." + ) + if self.min_af is not None or self.max_af is not None: + raise NotImplementedError( + "min_af/max_af filtering is not supported for svar2 datasets yet." + ) + if self.unphased_union: + raise NotImplementedError( + "unphased_union is not supported for svar2 datasets yet." + ) + + def _contig_groups( + self, contig_ids: NDArray[np.int64] + ) -> list[tuple[int, NDArray[np.intp]]]: + """Group query positions by contig id (store readers are per-contig). + + Preserves original order within each contig group. + """ + groups: list[tuple[int, NDArray[np.intp]]] = [] + for ci in np.unique(contig_ids): + qsel = np.nonzero(contig_ids == ci)[0].astype(np.intp) + groups.append((int(ci), qsel)) + return groups + + def _gather_inputs( + self, + r_q: NDArray[np.integer], + si_q: NDArray[np.integer], + regions_grp: NDArray[np.int32], + P: int, + ) -> tuple[ + NDArray[np.uint32], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int32], + ]: + """Cache-slice a per-contig query block into the read-bound FFI inputs. + + Fancy-indexes the memmapped cache (sub-linear; no per-read search). The + vk_* rows come out ``(n, P, 2)`` -> reshaped ``(n*P, 2)`` in row = q*P+p + order, which is exactly what the kernel expects. + """ + assert self.cache is not None + c = self.cache + region_starts = np.ascontiguousarray(regions_grp[:, 1], np.uint32) + orig_samples = np.ascontiguousarray(c.sample_cols[si_q], np.int64) + vk_snp = np.ascontiguousarray( + np.asarray(c.vk_snp_range[r_q, si_q]).reshape(-1, 2), np.int64 + ) + vk_indel = np.ascontiguousarray( + np.asarray(c.vk_indel_range[r_q, si_q]).reshape(-1, 2), np.int64 + ) + dense_snp = np.ascontiguousarray(np.asarray(c.dense_snp_range[r_q]), np.int64) + dense_indel = np.ascontiguousarray( + np.asarray(c.dense_indel_range[r_q]), np.int64 + ) + region_bounds = np.ascontiguousarray(regions_grp[:, 1:3], np.int32) + return ( + region_starts, + orig_samples, + vk_snp, + vk_indel, + dense_snp, + dense_indel, + region_bounds, + ) + + def _ref_for_contig( + self, contig_idx: int + ) -> tuple[NDArray[np.uint8], NDArray[np.int64]]: + """The single-contig reference slice + ``[0, len]`` offsets the kernel wants. + + ``reference.offsets`` is built in ``ds_contigs`` order (Reference.from_path + was called with the dataset's contigs), so ``contig_idx`` indexes it + directly. + """ + ref = self.reference + assert ref is not None + o_s = int(ref.offsets[contig_idx]) + o_e = int(ref.offsets[contig_idx + 1]) + ref_ = np.ascontiguousarray(ref.reference[o_s:o_e], np.uint8) + ref_offsets = np.array([0, o_e - o_s], np.int64) + return ref_, ref_offsets + + @staticmethod + def _inverse_row_perm( + cat_query_order: list[NDArray[np.intp]], b: int, P: int + ) -> NDArray[np.intp]: + """Permutation mapping grouped hap-row order back to global (b, P) order. + + ``final_row[k] == grouped_row[perm[k]]``. + """ + if cat_query_order: + grouped_queries = np.concatenate(cat_query_order) + else: + grouped_queries = np.zeros(0, np.intp) + grouped_rows = ( + grouped_queries[:, None] * P + np.arange(P, dtype=np.intp) + ).ravel() + perm = np.empty(b * P, np.intp) + perm[grouped_rows] = np.arange(b * P, dtype=np.intp) + return perm + + def _assemble_haps( + self, + cat_data: list[NDArray[np.uint8]], + cat_hap_lens: list[NDArray[np.int64]], + cat_query_order: list[NDArray[np.intp]], + b: int, + P: int, + ) -> Ragged[np.bytes_]: + data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.uint8) + hap_lens = ( + np.concatenate(cat_hap_lens) if cat_hap_lens else np.zeros(0, np.int64) + ) + grouped_off = lengths_to_offsets(hap_lens, np.int64) + perm = self._inverse_row_perm(cat_query_order, b, P) + out_data, out_off = _ragged_arange_gather(data, grouped_off, perm) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, (b, P, None), out_off).view("S1"), + ) diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py new file mode 100644 index 00000000..9813f397 --- /dev/null +++ b/tests/dataset/test_svar2_dataset.py @@ -0,0 +1,186 @@ +"""End-to-end SVAR2 dataset read dispatch parity (Task 7b). + +Builds two gvl datasets over the same bed/samples/reference from matched stores +built from the SAME VCF -- one ``.svar`` (SVAR1) and one ``.svar2`` -- and asserts +the SVAR2 read path (``Svar2Haps``) is byte-identical to the SVAR1 path for +``with_seqs('haplotypes')`` and ``with_seqs('variants')``. + +Parity is exact because both sides open with ``deterministic=True`` (shifts=0) +and the datasets are written with ``max_jitter=0``, so no RNG is involved. The +fixture VCF is tie-free (no same-POS SNP+DEL) so the SVAR1 max_ends tie bug +(docs/known-issues/svar1-max-ends-tie-underextension.md) is not exercised. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import polars as pl +import pytest + +import genvarloader as gvl + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 +# (C>CAT), dense SNP@9 (G>C, carried by 3 haps -> dense/snp channel), DEL@11 +# (GTA>G, ilen -2). No same-POS ties. Mirrors the readbound-haps dense-SNP +# fixture so both var_key and dense channels are exercised. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def _src(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_ds_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar_fixture(_src, tmp_path_factory) -> Path: + bcf, _ref = _src + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1") / "store.svar" + SparseVar.from_vcf( + out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return out + + +@pytest.fixture(scope="module") +def svar2_fixture(_src, tmp_path_factory) -> Path: + bcf, ref = _src + from genoray import _core + + out = tmp_path_factory.mktemp("svar2") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +@pytest.fixture(scope="module") +def bed() -> pl.DataFrame: + # Tie-free windows spanning the SNP/INS/dense-SNP/DEL and a variant-free tail. + return pl.DataFrame( + { + "chrom": ["chr1"] * 4, + "chromStart": [0, 0, 5, 20], + "chromEnd": [40, 15, 20, 40], + } + ) + + +def _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref): + from genoray import SparseVar, SparseVar2 + + d1 = tmp_path / "d1.gvl" + d2 = tmp_path / "d2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + return ( + gvl.Dataset.open(d1, reference=ref), + gvl.Dataset.open(d2, reference=ref), + ) + + +def test_svar2_haplotypes_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, _src): + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)), ( + f"offsets differ: svar1={np.asarray(a.offsets).tolist()} " + f"svar2={np.asarray(b.offsets).tolist()}" + ) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +def _assert_ragged_equal(a, b, name: str) -> None: + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"{name} offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad = np.asarray(a.data).view("u1") + bd = np.asarray(b.data).view("u1") + assert np.array_equal(ad, bd), f"{name} data differ" + + +def test_svar2_variants_positions_match_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """The decoded variant SET (positions + ilens) is byte-identical to SVAR1. + + NOTE: the ALT allele *bytes* are intentionally NOT compared to SVAR1 here. + The two genoray formats encode a deletion's ALT differently -- SVAR1 keeps + the VCF anchor base (e.g. ``G`` for ``GTA>G``) while SVAR2 decodes the + atomized empty ALT (``""``). Haplotype reconstruction is unaffected (see + ``test_svar2_haplotypes_match_svar1``), and the ALT bytes are validated + against the SVAR2 decode oracle in ``test_svar2_variants_match_svar2_oracle``. + """ + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants")[:, :] + b = ds2.with_seqs("variants")[:, :] + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") + + +def test_svar2_variants_match_svar2_oracle( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Full RaggedVariants (start/ilen/alt) match the validated SVAR2 decode oracle. + + ``build_readbound_variants`` (parity-tested against genoray's ``SparseVar2.decode`` + in ``test_svar2_readbound_variants.py``) is driven over the dataset's own regions + in the same (region, sample, ploid) order, so this pins the ``Svar2Haps`` dispatch + (cache slicing + FFI wrapping + contig-group stitching) end-to-end. + """ + from genoray import SparseVar2 + + from genvarloader._dataset._svar2_store_py import build_readbound_variants + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + b = ds2.with_seqs("variants")[:, :] + + # Dataset regions in on-disk (sorted) order == the getitem's reconstruction + # regions (jitter=0). Build the oracle over the same (start, end) windows. + regions = ds2._full_regions + reg_list = [(int(s), int(e)) for s, e in regions[:, 1:3]] + sv = SparseVar2(svar2_fixture) + oracle = build_readbound_variants(sv, "chr1", reg_list) + + _assert_ragged_equal(b.start.to_packed(), oracle.start.to_packed(), "start") + _assert_ragged_equal(b.ilen.to_packed(), oracle.ilen.to_packed(), "ilen") + _assert_ragged_equal( + b.alt.to_chars().to_packed(), oracle.alt.to_chars().to_packed(), "alt" + ) From f7423d8a92a20d4219444d64889c788d0e1faf07 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 20:57:19 -0700 Subject: [PATCH 035/108] fix(dataset): svar2 min_af/max_af guard + variants jitter guard + coverage F1: forward min_af/max_af into Svar2Haps.from_path so the NotImplementedError guard fires for both Dataset.open(min_af=) and with_settings(min_af=) (was silently dropping the filter -> unfiltered output). F2: guard variants mode on max_jitter>0 OR jitter>0 (the read-bound variants decode has no right-clip, so a jittered/padded gather over-includes variants). Add guard-raises tests + a committed multi-contig haplotype parity test. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_open.py | 7 +- python/genvarloader/_dataset/_svar2_haps.py | 29 ++- tests/dataset/test_svar2_dataset.py | 196 ++++++++++++++++++++ 3 files changed, 227 insertions(+), 5 deletions(-) diff --git a/python/genvarloader/_dataset/_open.py b/python/genvarloader/_dataset/_open.py index f6f33747..5a20649e 100644 --- a/python/genvarloader/_dataset/_open.py +++ b/python/genvarloader/_dataset/_open.py @@ -150,9 +150,7 @@ def _build_seqs( if self._has_genotypes(): if metadata.ploidy is None: raise ValueError("Malformed dataset: found genotypes but not ploidy.") - svar2_meta = ( - self.path / "genotypes" / "svar2_ranges" / "svar2_meta.json" - ) + svar2_meta = self.path / "genotypes" / "svar2_ranges" / "svar2_meta.json" seqs: Haps | Ref | None if svar2_meta.exists(): from ._svar2_haps import Svar2Haps @@ -165,6 +163,9 @@ def _build_seqs( svar2_link=metadata.svar2_link, svar2_override=self.svar2, contigs=metadata.contigs, + min_af=self.min_af, + max_af=self.max_af, + max_jitter=metadata.max_jitter, ) else: seqs = Haps.from_path( diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 73cba72b..025e86c6 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -144,6 +144,11 @@ class Svar2Haps(Haps[_H]): """The .svar2 store's contig names (used to open the store's ContigReaders).""" ds_contigs: list[str] = field(default_factory=list) """The dataset's contig names (``regions[:, 0]`` indexes into this).""" + max_jitter: int = 0 + """The dataset's write-time max_jitter. When > 0 the cache's per-query ranges + were computed over a max_jitter-padded window, which over-includes variants past + the (unpadded) read window in variants mode (the decode kernel has no right-clip); + guarded below.""" def __post_init__(self): # Deliberately does NOT call Haps.__post_init__ (that reads an SVAR1 @@ -165,6 +170,9 @@ def from_path( # type: ignore[override] # separate svar2 signature; base retur svar2_override: Path | str | None, contigs: list[str], kind: type = RaggedVariants, + min_af: float | None = None, + max_af: float | None = None, + max_jitter: int = 0, ) -> "Svar2Haps": ranges_dir = path / "genotypes" / "svar2_ranges" with open(ranges_dir / "svar2_meta.json") as f: @@ -232,12 +240,13 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: dosages=None, kind=cast("type[_H]", kind), filter=None, - min_af=None, - max_af=None, + min_af=min_af, + max_af=max_af, store=store, cache=cache, store_contigs=list(sv.contigs), ds_contigs=list(contigs), + max_jitter=max_jitter, ) # ---- reconstructor entry ---- @@ -262,6 +271,22 @@ def __call__( raise NotImplementedError( "svar2 datasets do not support with_seqs('variant-windows') yet." ) + # ``decode_variants_from_svar2_readbound`` has NO right-clip: it emits + # every gathered variant that passes the left-clip. The cache's per-query + # ranges cover the read window ONLY when it equals the write window -- + # i.e. no jitter anywhere. max_jitter>0 pads the cache at WRITE (so even + # a jitter=0 read over-includes variants in (end, end+max_jitter]); a + # jitter>0 read narrows/slides the window at READ. Guard on BOTH. + # (Haplotypes/tracks are unaffected: their kernel right-clips to q_end.) + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants output for svar2 datasets written with max_jitter>0" + f" (here max_jitter={self.max_jitter}) or read with jitter>0" + f" (here jitter={jitter}) is not yet supported: the read-bound" + " variants decode does not right-clip to the post-jitter window." + " Use max_jitter=0 at write and jitter=0 at read, or use" + " haplotypes/tracks modes." + ) # RaggedVariants: RC is applied by the caller (_getitem_unspliced), # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). return cast(_H, self._reconstruct_variants(idx, regions)) diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index 9813f397..8edc4483 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -184,3 +184,199 @@ def test_svar2_variants_match_svar2_oracle( _assert_ragged_equal( b.alt.to_chars().to_packed(), oracle.alt.to_chars().to_packed(), "alt" ) + + +# -------------------------------------------------------------------------- +# Guard-contract tests: the unsupported combos must RAISE, not silently return +# wrong output. (These lock the guards; the min_af one would have caught the +# open()-drops-min_af bug.) +# -------------------------------------------------------------------------- + + +def test_svar2_min_af_guard_raises_open(tmp_path, bed, svar2_fixture, _src): + """Dataset.open(min_af=...) must reach the NotImplementedError guard. + + Regression for the bug where _build_seqs dropped min_af/max_af for svar2, + leaving Svar2Haps.min_af=None so the guard never fired. + """ + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref, min_af=0.05) + with pytest.raises(NotImplementedError, match="min_af"): + ds.with_seqs("haplotypes")[:, :] + + +def test_svar2_min_af_guard_raises_with_settings(tmp_path, bed, svar2_fixture, _src): + """with_settings(min_af=...) must also reach the guard.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_settings(min_af=0.05) + with pytest.raises(NotImplementedError, match="min_af"): + ds.with_seqs("haplotypes")[:, :] + + +def test_svar2_splice_and_rc_guards_raise(tmp_path, bed, svar2_fixture, _src): + """splice_plan and a real (all-True) in-kernel to_rc must raise for haplotypes.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_seqs("haplotypes") + recon = ds._recon # the Svar2Haps reconstructor (RaggedSeqs kind) + + idx = np.array([0], np.intp) + r_idx = np.array([0], np.intp) + regions = ds._full_regions[[0]].copy() + rng = np.random.default_rng(0) + + with pytest.raises(NotImplementedError, match="[Ss]plice"): + recon(idx, r_idx, regions, "ragged", 0, rng, True, splice_plan=object()) + + with pytest.raises(NotImplementedError, match="reverse-complement"): + recon( + idx, + r_idx, + regions, + "ragged", + 0, + rng, + True, + to_rc=np.ones(len(idx), np.bool_), + ) + + +def test_svar2_variants_jitter_guard_raises(tmp_path, svar2_fixture, _src): + """variants mode must raise when the dataset was written with max_jitter>0. + + The read-bound variants decode does not right-clip, so a padded cache would + silently over-include variants; the guard prevents that. + """ + from genoray import SparseVar2 + + _bcf, ref = _src + # chromStart >= max_jitter so the padded window stays non-negative. + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, + jbed, + variants=SparseVar2(svar2_fixture), + samples=None, + max_jitter=2, + overwrite=True, + ) + ds = gvl.Dataset.open(d, reference=ref) + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variants")[:, :] + + +# -------------------------------------------------------------------------- +# Multi-contig haplotype parity: locks the contig-group split + inverse +# row-permutation stitching in Svar2Haps. +# -------------------------------------------------------------------------- + +# chr2 reference; VCF REF alleles match _REF2 exactly (idx4='C', idx8='T'). +_REF2 = "TTGGCCAATTGGCCAATTACGTACGTTTGGCCAATTGGCC" +_VCF2 = """\ +##fileformat=VCFv4.2 +##contig= +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +chr2\t5\t.\tC\tT\t.\t.\t.\tGT\t1|0\t1|1 +chr2\t9\t.\tT\tTGG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def _src2(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_mc_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n>chr2\n{_REF2}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + vcf = d / "in.vcf" + vcf.write_text(_VCF2) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar_fixture2(_src2, tmp_path_factory) -> Path: + bcf, _ref = _src2 + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1_mc") / "store.svar" + SparseVar.from_vcf( + out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return out + + +@pytest.fixture(scope="module") +def svar2_fixture2(_src2, tmp_path_factory) -> Path: + bcf, ref = _src2 + from genoray import _core + + out = tmp_path_factory.mktemp("svar2_mc") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1", "chr2"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +def test_svar2_haplotypes_match_svar1_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """Haplotypes byte-identical to SVAR1 across a TWO-contig, out-of-order bed. + + The interleaved chr2/chr1 bed forces Svar2Haps' contig-group split + inverse + row-permutation stitching (single-contig fast path is bypassed). + """ + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + # Interleaved contigs + a variant-free tail region, out of sorted order. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "mc1.gvl" + d2 = tmp_path / "mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)), ( + f"offsets differ: svar1={np.asarray(a.offsets).tolist()} " + f"svar2={np.asarray(b.offsets).tolist()}" + ) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) From 0f5b791759f6f11681c9da45a39dd679f6a784e7 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 21:19:54 -0700 Subject: [PATCH 036/108] feat(dataset): svar2 haplotype-realigned tracks via HapsTracks Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_reconstruct.py | 149 +++++++++++++++++++ python/genvarloader/_dataset/_svar2_haps.py | 101 +++++++++++++ tests/dataset/test_svar2_dataset.py | 142 ++++++++++++++++++ 3 files changed, 392 insertions(+) diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index 0d6b80e5..85d3f5b5 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -140,6 +140,24 @@ def __call__( flat: bool = False, to_rc: "NDArray[np.bool_] | None" = None, ) -> tuple[_H, _T]: + # SVAR2 read path (Task 7c): route to the split materialize→realign + # kernel. The isinstance guard keeps the SVAR1 body below byte-unchanged. + from ._svar2_haps import Svar2Haps + + if isinstance(self.haps, Svar2Haps): + return self._call_svar2( + idx, + r_idx, + regions, + output_length, + jitter, + rng, + deterministic, + splice_plan, + flat, + to_rc, + ) + if splice_plan is not None: raise NotImplementedError( "Splicing of haplotypes + tracks (shape (b, t, p, ~l)) is not " @@ -286,6 +304,137 @@ def __call__( return haps, tracks + def _call_svar2( + self, + idx: NDArray[np.integer], # (b) + r_idx: NDArray[np.integer], # (b) + regions: NDArray[np.int32], # (b 3) + output_length: Literal["ragged", "variable"] | int, + jitter: int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: SplicePlan | None = None, + flat: bool = False, + to_rc: "NDArray[np.bool_] | None" = None, + ) -> tuple[_H, _T]: + """SVAR2 haplotype-realigned tracks (see :class:`Svar2Haps`). + + Produces the SAME ``(b, t, p, ~l)`` ``_Flat`` layout the SVAR1 + :meth:`__call__` above produces — the per-track ``_out`` slices are filled + byte-identically, only the interval→realign step is SPLIT into the two + standalone SVAR2 kernels (``intervals_to_tracks`` + + ``shift_and_realign_tracks_from_svar2_readbound``) instead of the fused + SVAR1 kernel. Haps come from the shared ``get_haps_and_shifts`` 7-tuple. + """ + from ._svar2_haps import Svar2Haps + + haps_recon = cast(Svar2Haps, self.haps) + + if splice_plan is not None: + raise NotImplementedError( + "Splicing of haplotypes + tracks is not supported for svar2 " + "datasets yet." + ) + # The realign kernel has no in-kernel reverse-complement. + if to_rc is not None and bool(np.asarray(to_rc).any()): + raise NotImplementedError( + "In-kernel reverse-complement is not supported for svar2 " + "haplotype-realigned tracks." + ) + + lengths = regions[:, 2] - regions[:, 1] + + # ragged (b p l), (b p), (b p), (b p), (b p), None, None + haps, _geno_idx, shifts, diffs, hap_lengths, _keep, _keep_offsets = ( + haps_recon.get_haps_and_shifts( + idx=idx, + regions=regions, + output_length=output_length, + rng=rng, + deterministic=deterministic, + to_rc=to_rc, + ) + ) + + if issubclass(self.tracks.kind, RaggedTracks): + # The readbound track kernel always sizes each hap to ref_len + diff + # (no output_length override), so a fixed-length request cannot be + # honored byte-identically. Guard rather than silently mis-size. + if isinstance(output_length, int): + raise NotImplementedError( + "Fixed-length (int output_length) haplotype-realigned tracks " + "are not supported for svar2 datasets yet; use ragged/variable " + "output." + ) + # (b p) — ragged output: hap output length == hap_lengths. + out_lengths = hap_lengths + # (b) = lengths (b) + max deletion length across ploidy (b p) -> (b) + track_lengths = lengths - diffs.clip(max=0).min(1) + + # (b*p+1) + out_ofsts_per_t = lengths_to_offsets(out_lengths) + n_per_track: int = out_ofsts_per_t[-1] + # ragged (b t p l) + out = np.empty(len(self.tracks.active_tracks) * n_per_track, np.float32) + out_lens = repeat( + out_lengths, "b p -> b t p", t=len(self.tracks.active_tracks) + ) + out_offsets = lengths_to_offsets(out_lens) + + # Lower per-track strategies into numba-friendly arrays. + strat_list = [ + self.tracks.insertion_fill.get(name, Repeat5p()) + for name in self.tracks.active_tracks + ] + strat_ids, strat_params = _lower_insertion_fills(strat_list) + # Base seed identical to the SVAR1 path (idx-xor when deterministic). + if deterministic: + base_seed = np.uint64( + np.bitwise_xor.reduce(idx.astype(np.uint64, copy=False)) + ) + else: + base_seed = np.uint64( + rng.integers(0, np.iinfo(np.uint64).max, dtype=np.uint64) + ) + + for track_ofst, (name, tracktype) in enumerate( + self.tracks.active_tracks.items() + ): + intervals = self.tracks.intervals[name] + o_idx = idx if tracktype is TrackType.SAMPLE else r_idx + + _out = out[track_ofst * n_per_track : (track_ofst + 1) * n_per_track] + block_data, _block_off = haps_recon.realign_track_block( + idx=idx, + o_idx=o_idx, + regions=regions, + shifts=shifts, + track_lengths=track_lengths, + intervals=intervals, + params=np.ascontiguousarray(strat_params[track_ofst], np.float64), + strategy_id=int(strat_ids[track_ofst]), + base_seed=int(base_seed), + ) + # block_data is (b, P) C-ordered with per-hap lengths == hap_lengths, + # so its offsets equal out_ofsts_per_t; copy into the track slice. + _out[:] = block_data + + out_shape = ( + len(idx), + len(self.tracks.active_tracks), + self.haps.genotypes.shape[-2], + None, + ) + + # flat (b t p l) + tracks = _Flat.from_offsets(out, out_shape, out_offsets) + + else: + tracks = self.tracks._call_intervals(idx) + + tracks = cast(_T, tracks) + return cast(_H, haps), tracks + def _build_reconstructor( seqs: Haps | Ref | None, diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 025e86c6..679492f9 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -45,8 +45,10 @@ decode_variants_from_svar2_readbound, hap_diffs_from_svar2_readbound, reconstruct_haplotypes_from_svar2_readbound, + shift_and_realign_tracks_from_svar2_readbound, ) from ._flat_variants import _FlatVariantWindows +from ._intervals import intervals_to_tracks from ._haps import _H, Haps, _Variants from ._rag_variants import RaggedVariants from ._reference import Reference @@ -421,6 +423,105 @@ def get_haps_and_shifts( ) # svar2 placeholder; 7c re-slices the cache from idx. return out, geno_offset_idx, shifts, diffs, hap_lengths, None, None + # ---- tracks (7c) ---- + + def realign_track_block( + self, + idx: NDArray[np.integer], + o_idx: NDArray[np.integer], + regions: NDArray[np.integer], + shifts: NDArray[np.int32], + track_lengths: NDArray[np.integer], + intervals, + params: NDArray[np.float64], + strategy_id: int, + base_seed: int, + ) -> tuple[NDArray[np.float32], NDArray[np.int64]]: + """Haplotype-realign ONE track for a query block, returning a flat f32 + buffer + offsets in global ``(b, P)`` C-order (row = ``q * P + p``). + + The two-step SVAR2 track path (there is no fused interval→realign kernel): + + 1. **Materialize** the per-query reference-space track window from + ``intervals`` via the standalone :func:`intervals_to_tracks` FFI. Each + window starts at ``regions[q, 1]`` and spans ``track_lengths[q]`` ref + bases (= region length + max deletion across ploidy), exactly the + ``tracks``/``track_offsets`` input the realign kernel expects (one + window per query; all P haps of a query share it). + 2. **Realign** to haplotype coordinates via + :func:`shift_and_realign_tracks_from_svar2_readbound`, cache-sliced per + contig group EXACTLY like :meth:`get_haps_and_shifts` slices for haps. + + ``o_idx`` selects the interval row per query (``idx`` for SAMPLE tracks, + ``r_idx`` otherwise). Per-hap output length is ``ref_len + diff`` (the + kernel's native sizing), so the stitched offsets equal the haps' + ``hap_lengths`` in the same order. + """ + assert self.store is not None + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + params_c = np.ascontiguousarray(params, np.float64) + o_idx = np.asarray(o_idx) + track_lengths = np.asarray(track_lengths, np.int64) + + cat_data: list[NDArray[np.float32]] = [] + cat_lens: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + + # (1) materialize ref-space track windows for this group's queries. + tl_g = track_lengths[qsel] + track_ofsts_g = lengths_to_offsets(tl_g, np.int64) + tracks_buf = np.empty(int(track_ofsts_g[-1]), np.float32) + intervals_to_tracks( + offset_idxs=o_idx[qsel], + starts=regions[qsel, 1], + itv_starts=intervals.starts.data, + itv_ends=intervals.ends.data, + itv_values=intervals.values.data, + itv_offsets=intervals.starts.offsets, + out=tracks_buf, + out_offsets=track_ofsts_g, + ) + + # (2) realign to haplotype coordinates (cache-sliced, per contig). + g_shifts = np.ascontiguousarray(shifts[qsel], np.int32) + g_total = int(track_ofsts_g[-1]) + out_data, out_off = shift_and_realign_tracks_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + g_shifts, + np.ascontiguousarray(tracks_buf, np.float32), + track_ofsts_g, + params_c, + np.int64(strategy_id), + np.uint64(base_seed), + should_parallelize(g_total * 4), + ) + cat_data.append(np.asarray(out_data, np.float32)) + cat_lens.append(np.diff(np.asarray(out_off, np.int64))) + cat_query_order.append(qsel) + + data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.float32) + lens = np.concatenate(cat_lens) if cat_lens else np.zeros(0, np.int64) + grouped_off = lengths_to_offsets(lens, np.int64) + perm = self._inverse_row_perm(cat_query_order, b, P) + return _ragged_arange_gather(data, grouped_off, perm) + # ---- variants ---- def _reconstruct_variants( diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index 8edc4483..cd697e4c 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -124,6 +124,148 @@ def test_svar2_haplotypes_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture assert np.array_equal(a.data.view("u1"), b.data.view("u1")) +def _make_bigwig(path: Path, contig: str, length: int, seed: int) -> None: + """Write a dense per-bp BigWig over ``contig`` (deterministic given ``seed``).""" + import pyBigWig + + rng = np.random.default_rng(seed) + starts = list(range(length)) + ends = list(range(1, length + 1)) + values = [float(v) for v in rng.standard_normal(length).astype(np.float32)] + with pyBigWig.open(str(path), "w") as bw: + bw.addHeader([(contig, length)], maxZooms=0) + bw.addEntries([contig] * length, starts, ends=ends, values=values) + + +@pytest.fixture(scope="module") +def bigwig_fixture(tmp_path_factory): + """Per-sample BigWigs over chr1 (len 40) -> a SAMPLE-indexed gvl.BigWigs track.""" + bw_dir = tmp_path_factory.mktemp("svar2_bw") + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + _make_bigwig(p, "chr1", 40, seed=100 + i) + paths[s] = str(p) + return gvl.BigWigs("signal", paths) + + +def test_svar2_tracks_match_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, bigwig_fixture, _src +): + """Haplotype-realigned tracks byte-identical (f32, NaN-equal) to SVAR1. + + Both datasets are written from the SAME VCF + SAME BigWig track; at read the + SVAR1 backend realigns via the fused kernel and the SVAR2 backend + (``Svar2Haps`` + ``HapsTracks._call_svar2``) realigns via the split + ``intervals_to_tracks`` + ``shift_and_realign_tracks_from_svar2_readbound`` + path. deterministic=True + max_jitter=0 => shifts=0, so parity is exact. + """ + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src + d1 = tmp_path / "t1.gvl" + d2 = tmp_path / "t2.gvl" + gvl.write( + d1, + bed, + variants=SparseVar(svar_fixture), + tracks=bigwig_fixture, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, + bed, + variants=SparseVar2(svar2_fixture), + tracks=bigwig_fixture, + samples=None, + max_jitter=0, + overwrite=True, + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + # HapsTracks (seqs active by default) -> (haps, tracks) tuple. + _h1, a = ds1.with_tracks("signal")[:, :] + _h2, b = ds2.with_tracks("signal")[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "track data differ" + + +def test_svar2_tracks_match_svar1_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """Realigned tracks byte-identical to SVAR1 across a TWO-contig, out-of-order + bed -- exercises ``_call_svar2``'s contig-group split + inverse row-perm + stitching for the track path (single-contig fast path bypassed).""" + import pyBigWig + + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bw_dir = tmp_path / "bw_mc" + bw_dir.mkdir() + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + rng = np.random.default_rng(200 + i) + with pyBigWig.open(str(p), "w") as bw: + bw.addHeader([("chr1", 40), ("chr2", 40)], maxZooms=0) + for contig in ("chr1", "chr2"): + vals = [float(v) for v in rng.standard_normal(40).astype(np.float32)] + bw.addEntries( + [contig] * 40, list(range(40)), ends=list(range(1, 41)), values=vals + ) + paths[s] = str(p) + track = gvl.BigWigs("signal", paths) + + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "tmc1.gvl" + d2 = tmp_path / "tmc2.gvl" + gvl.write( + d1, + bed, + variants=SparseVar(svar_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, + bed, + variants=SparseVar2(svar2_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + _h1, a = ds1.with_tracks("signal")[:, :] + _h2, b = ds2.with_tracks("signal")[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "track data differ" + + def _assert_ragged_equal(a, b, name: str) -> None: ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) assert np.array_equal(ao, bo), ( From 9bacdac2b7148722e10f1b84c9a5578bd5bc93bc Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 21:32:23 -0700 Subject: [PATCH 037/108] fix(dataset): guard svar2 FlankSample multi-contig + annotated tracks FlankSample (the only seed-dependent insertion fill) diverges from SVAR1 on multi-contig svar2 datasets: _call_svar2 realigns per-contig-group, so the fill hash gets a contig-local query index vs SVAR1's global row. Guard FlankSample + >1 contig group with NotImplementedError (single-contig and non-seeded fills unaffected). Also guard annotated+tracks (HapsTracks reaches _call_svar2 before Svar2Haps's annotated guard). Proper follow-up: pass a global-query offset into the track FFI. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_reconstruct.py | 33 ++++++++++- tests/dataset/test_svar2_dataset.py | 60 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index 85d3f5b5..6c8058dd 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -25,7 +25,7 @@ from .._utils import lengths_to_offsets from ._genotypes import _as_starts_stops from ._haps import _H, Haps, ReconstructionRequest, _NewH, _Variants -from ._insertion_fill import Repeat5p +from ._insertion_fill import FLANK_SAMPLE, Repeat5p from ._insertion_fill import lower as _lower_insertion_fills from ._flat_variants import _FlatVariantWindows from ._protocol import Reconstructor @@ -335,6 +335,14 @@ def _call_svar2( "Splicing of haplotypes + tracks is not supported for svar2 " "datasets yet." ) + # Annotated haps are out of scope for svar2 (matches Svar2Haps.__call__). + # HapsTracks routes here BEFORE Svar2Haps's own annotated guard, and + # get_haps_and_shifts returns plain Ragged[S1] regardless of kind, so + # without this an annotated view would silently yield non-annotated haps. + if issubclass(self.haps.kind, RaggedAnnotatedHaps): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('annotated') with tracks yet." + ) # The realign kernel has no in-kernel reverse-complement. if to_rc is not None and bool(np.asarray(to_rc).any()): raise NotImplementedError( @@ -387,6 +395,29 @@ def _call_svar2( for name in self.tracks.active_tracks ] strat_ids, strat_params = _lower_insertion_fills(strat_list) + + # FIX 1 guard: FlankSample (the only seed-dependent fill) diverges + # from SVAR1 across MULTIPLE contigs. SVAR1 realigns the whole batch + # in ONE fused call, so the fill hash `hash4(base_seed, query, hap, + # out_idx+i)` uses the GLOBAL row `query`. `_call_svar2` calls the + # readbound kernel once PER CONTIG GROUP, where `query = k/ploidy` is + # contig-LOCAL, so a global row landing at a different local position + # gets different fill offsets in inserted regions. base_seed matches + # (both derive from the full idx); only the per-query index diverges. + # Single-contig is exact (local == global); non-seeded fills (Repeat5p + # etc.) are exact regardless. Proper fix (follow-up): pass a per-group + # global-query-offset into the FFI so the kernel seeds with the global + # row index; for now, guard. + n_contig_groups = int(np.unique(regions[:, 0]).size) + if n_contig_groups > 1 and bool((strat_ids == FLANK_SAMPLE).any()): + raise NotImplementedError( + "svar2 haplotype-realigned tracks with a seed-dependent " + "insertion fill (FlankSample) across multiple contigs are not " + "yet supported: the per-contig-group kernel calls seed the fill " + "with a contig-local query index that diverges from the " + "single-batch SVAR1 path. Use a single-contig query, or a " + "non-seeded insertion fill (e.g. the default Repeat5p)." + ) # Base seed identical to the SVAR1 path (idx-xor when deterministic). if deterministic: base_seed = np.uint64( diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index cd697e4c..e8a359c9 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -266,6 +266,66 @@ def test_svar2_tracks_match_svar1_multicontig( assert np.allclose(ad, bd, equal_nan=True), "track data differ" +def test_svar2_flanksample_multicontig_guard(tmp_path, svar2_fixture2, _src2): + """FlankSample (seed-dependent fill) + MULTI-contig must raise, not silently + diverge from SVAR1. + + ``_call_svar2`` realigns per contig group, seeding the fill with a + contig-LOCAL query index; SVAR1 seeds with the GLOBAL row. For a seed-only + fill (FlankSample) this diverges across >1 contig, so it is guarded. (The + single-contig and default-Repeat5p paths remain exact -- see the parity tests + above, which never set a fill.) + """ + import pyBigWig + + from genoray import SparseVar2 + + from genvarloader._dataset._insertion_fill import FlankSample + + _bcf, ref = _src2 + bw_dir = tmp_path / "bw_fs" + bw_dir.mkdir() + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + rng = np.random.default_rng(300 + i) + with pyBigWig.open(str(p), "w") as bw: + bw.addHeader([("chr1", 40), ("chr2", 40)], maxZooms=0) + for contig in ("chr1", "chr2"): + vals = [float(v) for v in rng.standard_normal(40).astype(np.float32)] + bw.addEntries( + [contig] * 40, list(range(40)), ends=list(range(1, 41)), values=vals + ) + paths[s] = str(p) + track = gvl.BigWigs("signal", paths) + + # Interleaved chr2/chr1 bed -> >1 contig group. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1"], + "chromStart": [0, 0], + "chromEnd": [40, 40], + } + ) + d = tmp_path / "fs.gvl" + gvl.write( + d, + bed, + variants=SparseVar2(svar2_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + ds = ( + gvl.Dataset.open(d, reference=ref) + .with_tracks("signal") + .with_insertion_fill({"signal": FlankSample(flank_width=3)}) + ) + with pytest.raises(NotImplementedError, match="FlankSample"): + ds[:, :] + + def _assert_ragged_equal(a, b, name: str) -> None: ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) assert np.array_equal(ao, bo), ( From eb4a783fe2aa526e25c80f26e263e7675dd54103 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 21:37:26 -0700 Subject: [PATCH 038/108] refactor(dataset): retire svar2 live overlap_batch dispatch (oracle-only) Live .svar2 dataset reconstruction is handled by the read-bound Svar2Haps path (_svar2_haps.py); SparseVar2Source (the genoray overlap_batch union path) is retained only as the byte-identical parity oracle for the read-bound kernel tests. Removes the deferred TODO(svar2-dataset-dispatch) marker. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_source.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_source.py b/python/genvarloader/_dataset/_svar2_source.py index 38b49a15..8d61f520 100644 --- a/python/genvarloader/_dataset/_svar2_source.py +++ b/python/genvarloader/_dataset/_svar2_source.py @@ -1,12 +1,16 @@ -"""SVAR2 two-source reconstruction adapter. +"""SVAR2 two-source reconstruction adapter — parity oracle only (not a live read path). Bridges genoray ``SparseVar2.overlap_batch``'s raw two-channel dict to gvl's SVAR2 kernels (``reconstruct_haplotypes_from_svar2`` / ``shift_and_realign_tracks_from_svar2``), decoding -``var_key ⋈ dense`` inline with no intermediate variant table. Additive to the SVAR 1.0 path. +``var_key ⋈ dense`` inline with no intermediate variant table. This is the *union* path +(genoray ``overlap_batch``, whole-cohort). -TODO(svar2-dataset-dispatch): wiring this into ``Haps``/``Dataset`` (an svar2-source flag on the -dataset that routes reconstruction here) is deferred — it touches the central SVAR 1.0 reconstructor. -This adapter is self-contained and is validated end-to-end in tests against genoray's decode oracle. +Live dataset dispatch is NOT wired through here. ``Dataset`` reconstruction for ``.svar2``-backed +datasets is handled by the read-bound path in ``Svar2Haps`` (``_svar2_haps.py``), which gathers off +the write-time ranges cache and calls the ``*_from_svar2_readbound`` kernels — no interval-search-tree +rebuild and no dense-union rebuild per read. This ``SparseVar2Source`` adapter is retained solely as +the byte-identical *parity oracle* the read-bound kernels are tested against (see +``tests/dataset/test_svar2_readbound_*.py``); it is not imported on any live read path. """ from __future__ import annotations From b5bceb74cd833b4e9b425422a1ad109e8f93aa0b Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 21:48:36 -0700 Subject: [PATCH 039/108] docs: .svar2 as a write variant source + read-bound wiring (skill, format, faq, roadmap) Document the shipped .svar2 write source and read-bound Dataset wiring: - SKILL.md: .svar2 alongside .svar as a write source, Dataset.open(svar2=), the read-bound no-tree/no-union read path, the Phase-1 unsupported matrix, and the pure-deletion ALT bytes difference vs .svar. - write.md: .svar2 accepted as a variants= source, produces the ranges cache. - format.md: genotypes/svar2_ranges/ layout (six arrays + svar2_meta.json), metadata.json svar2_link field, .svar2 resolution-at-open order, and the ALT convention. - faq.md: .svar2's on-disk size advantage, no interval tree / no dense union at read, and the variants ALT backend difference. - README.md: .svar2 as a supported variant source. - rust-migration.md: new Phase 6a section + decisions-log entry recording the read-bound wiring, parity results (31/31 svar2 tests byte-identical), and links to the four svar2 design/implementation plans. api.md <-> __all__ gate confirmed clean (MISSING: none) -- svar2= is a parameter, not a new public symbol. Co-Authored-By: Claude Opus 4.8 --- README.md | 2 +- docs/roadmaps/rust-migration.md | 97 ++++++++++++++++++++++++++++++++- docs/source/faq.md | 11 ++++ docs/source/format.md | 76 ++++++++++++++++++++++++-- docs/source/write.md | 18 +++++- skills/genvarloader/SKILL.md | 34 +++++++++++- 6 files changed, 228 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7c4513e7..709b6390 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ GenVarLoader provides a fast, memory efficient data structure for training seque - Generate haplotypes up to 1,000 times faster than reading a FASTA file - Generate tracks up to 450 times faster than reading a BigWig - **Supports indels** and re-aligns tracks to haplotypes that have them -- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, and BigWig +- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, BigWig, and [genoray](https://github.com/mcvickerlab/genoray)'s sparse `.svar`/`.svar2` variant stores Documentation is available [here](https://genvarloader.readthedocs.io/). See our [preprint](https://www.biorxiv.org/content/10.1101/2025.01.15.633240) for benchmarking and implementation details. diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index 8ed11a58..fdfdba26 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -777,12 +777,72 @@ _PR: —_ > on every mode (tracks-only 1.07×, haplotypes/tracks-seqs 1.66×, annotated 1.43×, variants > 1.38×, variant-windows 4.58×). +### Phase 6a — SVAR2 read-bound dataset wiring (genoray query-only) ✅ +_PR: TBD (branch `svar2-m6b-kernel`)_ +_Specs: `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` (genoray side), +`docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md` (this side); earlier design +specs `docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md` and +`docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md`._ + +A scoped slice of Phase 6, pulled forward: genoray's newer `.svar2` sparse variant format +becomes a `gvl.write` variant source and a live `Dataset` read backend, wired via a +**read-bound** path rather than the general genoray-VCF/PGEN absorption Phase 6 describes below. +gvl links `genoray_core` (query-only, `default-features = false` — no htslib conversion path) +as a Rust path-dep, the first place gvl's Rust crate depends on genoray's Rust core directly. + +- [x] `_svar2_link.py` (`Svar2Link`/`Svar2Fingerprint`/`_resolve_svar2`/`_verify_svar2_fingerprint`, + mirrors `_svar_link.py`) + `Metadata.svar2_link` field. +- [x] `_write_from_svar2`: write-time 6-array ranges cache (`vk_snp_range`, `vk_indel_range`, + `dense_snp_range`, `dense_indel_range`, `region_starts`, `sample_cols`, all int64) under + `genotypes/svar2_ranges/` + `svar2_meta.json`, sized to the dataset's **selected** samples; + `.svar2` write dispatch in `_write.py`. Same-POS-tie `max_ends` under-extension excluded + from parity — SVAR1-side bug, not introduced here (see + `docs/known-issues/svar1-max-ends-tie-underextension.md`). +- [x] `genoray_core` path-dep + `Svar2Store` pyclass (opens one `ContigReader` per contig once, + at `Dataset.open` — the SVAR2 analog of SVAR1's cached FFI statics). +- [x] Read-bound haplotype kernel: `reconstruct_haplotypes_from_svar2_readbound` — one FFI call, + `gather_haps_readbound` + `merge_hap3`, builds **zero** interval-search trees and **zero** + dense unions per read (structural guarantee, not just measured). +- [x] Read-bound track re-alignment kernel: `shift_and_realign_tracks_from_svar2_readbound`. +- [x] Read-bound variants/variant-windows decode kernel: `decode_variants_from_svar2_readbound` + (+ `hap_diffs_from_svar2_readbound` for jitter-shift computation). +- [x] Dataset read dispatch: `Svar2Haps` reconstructor, `source` discriminant, `Dataset.open(svar2=...)` + override mirroring `svar=`. Live `overlap_batch`/union dispatch in `_svar2_source.py` retired + (kept only as the parity oracle for tests). +- [x] Guard matrix (Phase-1 scope; raise `NotImplementedError`, not silent mis-compute): spliced + output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, + `unphased_union`, `"variant-windows"`, fixed-length (`int output_length`) haplotype-realigned + tracks, `variants` output with jitter (write `max_jitter>0` or read `jitter>0` — the + readbound variants decode has no right-clip), and multi-contig `FlankSample` track fills + (contig-local vs. global fill-seed query index divergence). +- [x] Docs/skill audit (this task): `skills/genvarloader/SKILL.md`, `docs/source/{write,format,faq}.md`, + `README.md`; `api.md` ↔ `__all__` gate confirmed clean (no new public symbol — `svar2=` is a + parameter, not an exported name). + +**Gate (parity — MET):** all four output modes (haplotypes, tracks, variants, variant-windows*) +byte-identical to the `.svar`/union-oracle (`SparseVar2Source.reconstruct`/`realign_tracks`, genoray +`decode`) across `tests/dataset/test_svar2_dataset.py`, `test_svar2_readbound_{haps,tracks,variants,diffs}.py`, +`test_write_svar2.py` — 31/31 passed. (*`"variant-windows"` is guarded `NotImplementedError` for +`.svar2` in Phase 1, so its parity claim covers `variants`, not the flat-window mode.) One documented, +intentional non-identity: for a pure deletion, `.svar2` decodes the atomized empty ALT (`b""`) where +`.svar` reports the VCF anchor base (`b"G"` for `GTA>G`) — a genoray format convention; reconstructed +haplotype bytes are unaffected (see `docs/source/format.md` "`.svar2` variants ALT convention"). +Full-tree regression: SVAR1 path byte-unchanged (additive-only change). + +**Checkpoint:** `.svar2` is a supported `gvl.write` source and `Dataset` read backend with a +structurally read-bound query path (no per-read tree build, no per-read dense union), on-disk +smaller than `.svar` especially for large cohorts. Remaining genoray absorption (VCF/PGEN ingest, +`.svar`→Rust, `.svar2` conversion/write path) stays in Phase 6 below. + ### Phase 6 — Absorb genoray (future) ⬜ _PR: —_ Sequenced last; a candidate to graduate into its own roadmap once Phases 0–5 land. seqpro-core remains the ragged substrate (decision 2026-06-23) — Phase 6 is -narrowed to genoray (variant IO) only. +narrowed to genoray (variant IO) only. Phase 6a above already pulled the `.svar2` +**read-only query** surface (`genoray_core`, `default-features = false`) into the Rust +stack as a scoped precursor; Phase 6 covers the remaining VCF/PGEN ingest and +conversion/write paths. - [ ] Bring variant IO (genoray VCF/PGEN + sparse genotypes) into the Rust stack. @@ -792,6 +852,41 @@ narrowed to genoray (variant IO) only. ## Notes & decisions log +- 2026-07-05 (Phase 6a — SVAR2 read-bound dataset wiring; branch `svar2-m6b-kernel`): + `.svar2` (genoray's newer sparse variant format) is now a `gvl.write` variant source and a + live `Dataset` read backend, wired end-to-end: write-time 6-array ranges cache + (`genotypes/svar2_ranges/`, sized to the dataset's selected samples, mirrors SVAR1's + `offsets.npy`/`svar_meta.json` pattern) + `Svar2Link` back-reference/fingerprint + (mirrors `SvarLink`, keyed on file-count + byte-size since `.svar2` exposes no cheap + `variant_idxs.npy` analogue); a new `genoray_core` Rust path-dep (query-only, + `default-features = false` — no htslib) backing a `Svar2Store` pyclass opened once at + `Dataset.open`; three read-bound all-Rust kernels + (`reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, + `decode_variants_from_svar2_readbound` + a `hap_diffs_from_svar2_readbound` helper for jitter + shifts) that gather directly off the write-time cache via `gather_haps_readbound` + a new + `merge_hap3` (var_key ⋈ dense_snp ⋈ dense_indel) — **no interval-search-tree build and no + dense-union rebuild per read**, unlike the existing union-based `SparseVar2Source` + (`overlap_batch`) path, which is retired from live dispatch and kept only as the parity + oracle. All four output modes parity-tested against that oracle / SVAR1: + `tests/dataset/test_svar2_dataset.py` + `test_svar2_readbound_{haps,tracks,variants,diffs}.py` + + `test_write_svar2.py`, 31/31 passed. Phase-1 scope excludes (guarded + `NotImplementedError`, not silent mis-compute): splicing, `var_filter="exonic"`, + `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, `unphased_union`, + `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with any + jitter (write or read), and multi-contig `FlankSample` track fills. One intentional, + documented non-identity vs. SVAR1: pure-deletion ALT bytes (`.svar2` emits the atomized + empty ALT `b""`; `.svar` emits the VCF anchor base) — a genoray format convention that does + not affect reconstructed haplotype bytes. The pre-existing SVAR1 same-POS-tie `max_ends` + under-extension bug (`docs/known-issues/svar1-max-ends-tie-underextension.md`) is excluded + from the `.svar2` write-path max_ends parity check, same as the rest of the SVAR1 parity + suite. Specs: `docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md`, + `docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md`, + `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` (genoray side), + `docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md` (this side, Phase 6a + above). Docs/skill updated (`skills/genvarloader/SKILL.md`, + `docs/source/{write,format,faq}.md`, `README.md`); `api.md` ↔ `__all__` gate confirmed + clean (no new public symbol — `svar2=` is a keyword argument, not an exported name). + - 2026-06-27 (Phase 5 W6 — wrap-up: thin-shim audit + cargo-standalone + seqpro-core + perf re-baseline; branch `phase-5-w6-wrapup`): Four parallel threads closed Phase 5: **(A) Thin-shim audit (Task 1, commit `0932374`):** Classified every Python step over the diff --git a/docs/source/faq.md b/docs/source/faq.md index bb3b8a32..1d2c0413 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -71,6 +71,17 @@ GVL's read path (haplotype reconstruction and track re-alignment) is parallelize - **`GVL_FORCE_PARALLEL`** — set to a truthy value (`1`, `true`, `yes`, `on`) to force the multithreaded paths even on small inputs. By default GVL runs small inputs serially because thread overhead would dominate; this bypasses that size gate. Mainly useful for benchmarking. - **`RAYON_NUM_THREADS`** — GVL **overwrites** this with its own resolved count so an inherited value (e.g. baked into a base image) can't defeat the cgroup-aware cap. To size the pool yourself, use `GVL_NUM_THREADS` instead. +## Should I use `.svar` or `.svar2` as my variant source? + +Both are sparse columnar variant archives from [`genoray`](https://github.com/mcvickerlab/genoray) that `gvl.write(variants=...)` accepts alongside BCF/PGEN; see [write.md](write.md) for how to build one. The two differ in their read-time behavior: + +- **`.svar`** reconstructs by building an interval search tree over the queried window and a per-read dense union of the overlapping variants. +- **`.svar2`** reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges at write time, and `Dataset.__getitem__` gathers directly off that cache and calls all-Rust kernels — it builds **no interval search tree and no dense union per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. + +`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, in-kernel `to_rc`, `unphased_union`, `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants at any supported jitter/output-length combination — is byte-identical between the two backends. + +One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs, and only for pure-deletion records. + ## How can I get personalized protein/spliced RNA sequences? This is not yet supported but on GVL's roadmap for the near future. Keep an eye out in future releases! diff --git a/docs/source/format.md b/docs/source/format.md index 165dd249..f98b5684 100644 --- a/docs/source/format.md +++ b/docs/source/format.md @@ -11,17 +11,21 @@ dataset_dir/ ├── metadata.json # the Metadata schema (below) ├── input_regions.arrow # original BED regions + region-index map ├── genotypes/ # present iff variants were provided to gvl.write -│ ├── offsets.npy # per (region, sample, ploidy) offsets into variant_idxs.npy +│ ├── offsets.npy # per (region, sample, ploidy) offsets into variant_idxs.npy; absent when sourced from .svar2 │ ├── svar_meta.json # shape + dtype of offsets.npy — present iff source was .svar -│ ├── variant_idxs.npy # variant indices; absent when sourced from .svar -│ ├── dosages.npy # optional, absent when sourced from .svar -│ └── variants.arrow # variant table; absent when sourced from .svar +│ ├── variant_idxs.npy # variant indices; absent when sourced from .svar or .svar2 +│ ├── dosages.npy # optional, absent when sourced from .svar or .svar2 +│ ├── variants.arrow # variant table; absent when sourced from .svar or .svar2 +│ └── svar2_ranges/ # present iff source was .svar2 — see "svar2_ranges layout" below └── intervals/ # or annot_intervals/ when annotated; present iff tracks given ``` When the dataset was built from an `.svar`, the heavy per-variant arrays (`variant_idxs.npy`, `dosages.npy`, `index.arrow`) are **not duplicated** into the dataset. Instead the dataset records a back-reference to the source `.svar` in `metadata.json` (see `svar_link` below). +Likewise, a dataset built from an `.svar2` records a back-reference (`svar2_link`, below) +and caches only small per-`(region, sample, ploidy)` range arrays under `genotypes/svar2_ranges/` +— the bulk variant data stays in the `.svar2` store. ## `metadata.json` schema @@ -36,6 +40,7 @@ records a back-reference to the source `.svar` in `metadata.json` (see `svar_lin | `max_jitter` | `int` | Maximum coordinate jitter (defaults to 0). | | `version` | `SemanticVersion \| None` | Package version that wrote this dataset. Drives format dispatch. | | `svar_link` | `SvarLink \| None` | Back-reference to a source `.svar`, when present. | +| `svar2_link` | `Svar2Link \| None` | Back-reference to a source `.svar2`, when present. | `SvarLink`: @@ -52,6 +57,44 @@ records a back-reference to the source `.svar` in `metadata.json` (see `svar_lin | `n_variants` | `int` | Row count of the svar's `index.arrow`. | | `variant_idxs_bytes` | `int` | Byte size of the svar's `variant_idxs.npy`. | +`Svar2Link` (mirrors `SvarLink` for a `.svar2` source): + +| Field | Type | Notes | +|-------|------|-------| +| `relative_path` | `str` | POSIX path from `dataset_dir` to the `.svar2`. | +| `absolute_path` | `str` | Original absolute path; used as a fallback. | +| `fingerprint` | `Svar2Fingerprint` | Integrity check (see below). | + +`Svar2Fingerprint`: + +| Field | Type | Notes | +|-------|------|-------| +| `n_files` | `int` | Count of the `.svar2` store's `.bin`/`.npy` data files. | +| `store_bytes` | `int` | Summed byte size of those data files. | + +`.svar2` has no `variant_idxs.npy`/`index.arrow` analogue exposed cheaply, so its fingerprint +keys on file count + total byte size of the store's data files rather than a variant count. + +## `genotypes/svar2_ranges/` layout + +Written only when the dataset's variant source is a `.svar2` store. `R` = number of regions, +`S` = number of the dataset's **selected** samples (not necessarily the full `.svar2` cohort), +`P` = ploidy. All arrays are `int64`: + +| File | Shape | Notes | +|------|-------|-------| +| `vk_snp_range.npy` | `(R, S, P, 2)` | Per-`(region, sample, ploid)` half-open range into the `.svar2` store's SNP variant-key column. | +| `vk_indel_range.npy` | `(R, S, P, 2)` | Same, for the indel variant-key column. | +| `dense_snp_range.npy` | `(R, 2)` | Per-region (sample-independent) range into the dense SNP store. | +| `dense_indel_range.npy` | `(R, 2)` | Per-region (sample-independent) range into the dense indel store. | +| `region_starts.npy` | `(R,)` | Per-region write-time start coordinate. Retained for parity/debugging; the read path derives per-query starts from the (post-jitter) query regions and does **not** read this array's values. | +| `sample_cols.npy` | `(S,)` | Maps the dataset's selected-sample slot to the `.svar2` store's original sample index. | +| `svar2_meta.json` | — | Records each array's `shape`/`dtype` plus `ploidy`. | + +At read time, `Dataset.__getitem__` slices these memmaps (numpy fancy-indexing; no interval +search) to build the flat per-query inputs for the read-bound Rust kernels — no interval-search +tree and no dense-union rebuild happen per read, unlike the `.svar` path. + ## SVAR resolution at open time When opening a dataset whose `metadata.svar_link` is non-null, @@ -66,6 +109,30 @@ If none match, a `FileNotFoundError` is raised naming the expected `.svar` basen resolution, the fingerprint is verified; a mismatch raises `ValueError` and lists both expected and observed values. +## `.svar2` resolution at open time + +When opening a dataset whose `metadata.svar2_link` is non-null, +[`Dataset.open`](api.md#genvarloader.Dataset.open) resolves the `.svar2` store in the same order +as `.svar`: + +1. Caller-provided `svar2=...` argument. +2. `svar2_link.relative_path` resolved against the dataset directory. +3. `svar2_link.absolute_path`. +4. A unique `*.svar2` directory next to the dataset. + +If none match, a `FileNotFoundError` is raised naming the expected `.svar2` basename and +suggesting `svar2=`. After resolution, the fingerprint (`Svar2Fingerprint`, above) is verified; +a mismatch raises `ValueError` and lists both expected and observed values. + +## `.svar2` variants ALT convention + +For a pure deletion (e.g. VCF `GTA>G`), decoding `with_seqs("variants")` yields different raw +ALT bytes depending on the backing store: `.svar` reports the VCF anchor base (`b"G"`), while +`.svar2` reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a +bug. Both stores consume the ALT identically when reconstructing haplotype sequence, so +`with_seqs("haplotypes")` / `with_seqs("annotated")` output is byte-identical between the two +backends; only `RaggedVariants.alt` differs, and only for pure-deletion records. + ## Format changelog | Version | Change | @@ -73,6 +140,7 @@ expected and observed values. | `< 0.18.0` | Variant coordinates stored 0-based. | | `0.18.0` | Variant coordinates switched to 1-based. | | `0.25.0` | `metadata.json` gains `svar_link`; old `genotypes/link.svar` symlink layout deprecated. `Metadata.version` typed as `SemanticVersion` (on-disk JSON unchanged). | +| (unreleased) | `metadata.json` gains `svar2_link`; `.svar2` accepted as a `gvl.write` variant source, cached under `genotypes/svar2_ranges/` and read via a read-bound, all-Rust path. | > **Upgrading legacy datasets.** A dataset written before `0.25.0` that was built from an > `.svar` will still open (with a `DeprecationWarning`). Run diff --git a/docs/source/write.md b/docs/source/write.md index a612d7a0..9ad58f87 100644 --- a/docs/source/write.md +++ b/docs/source/write.md @@ -79,4 +79,20 @@ gvl.write( ) ``` -This dataset would have both haplotypes and two tracks (`pos` and `neg`) available for samples that exist in both `all_chroms.bcf` and the BigWig tables (i.e. `gvl.write()` performs an inner join on samples). \ No newline at end of file +This dataset would have both haplotypes and two tracks (`pos` and `neg`) available for samples that exist in both `all_chroms.bcf` and the BigWig tables (i.e. `gvl.write()` performs an inner join on samples). + +## Variants from a genoray sparse store (`.svar` / `.svar2`) + +Besides BCF/VCF and PGEN, `variants=` also accepts a genoray sparse columnar variant store — either the original `.svar` format or the newer `.svar2` format: + +```python +gvl.write( + path="1000_genomes_haplotypes.gvl", + bed="tiling_windows.bed", + variants="all_chroms.svar2", # or "all_chroms.svar", or a SparseVar/SparseVar2 instance +) +``` + +Both formats store a back-reference in the dataset's `metadata.json` instead of duplicating per-variant arrays, so the source store must remain accessible when the dataset is later opened with [`gvl.Dataset.open()`](api.md#genvarloader.Dataset.open) (override its location with `svar=`/`svar2=` if it has moved). + +`.svar2` additionally produces a small write-time cache under `/genotypes/svar2_ranges/` and reads back through an all-Rust, read-bound path with no interval-search-tree build and no dense-union rebuild per read — see [the FAQ](faq.md) for the read-path and on-disk-size tradeoffs, and [the format reference](format.md) for the on-disk layout. `.svar2` currently has a Phase-1 scope: a handful of output combinations (splicing, `annotated` haplotypes, `min_af`/`max_af`, etc.) aren't wired yet and raise `NotImplementedError` — see the `genvarloader` skill or `format.md` for the full list. \ No newline at end of file diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index ea0da1d5..ff2932ef 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -80,6 +80,30 @@ dense2sparse(VCF("normed.bcf"), "normed.svar") # writes a .svar/ directory SVARs are resolved at `Dataset.open` time via `metadata.json` → caller `svar=` arg → recorded relative path → recorded absolute path → sibling `*.svar`. See `docs/source/format.md` ("SVAR resolution at open time") and `_dataset/_svar_link.py`. Legacy symlink-based SVAR layouts: run `gvl.migrate_svar_link(path)` once to upgrade. +## `.svar2` — the read-bound sparse variant format + +`.svar2` is genoray's newer sparse columnar variant store. Pass it to `gvl.write` exactly like a `.svar`, BCF, or PGEN — `gvl.write(path, bed, variants="cohort.svar2")` or `variants=SparseVar2("cohort.svar2")`. Like `.svar`, the dataset stores a back-reference (`metadata.json` → `svar2_link`) instead of duplicating per-variant arrays, so the `.svar2` store must remain accessible at read time. + +Unlike `.svar` (whose read path builds an interval search tree + a per-read dense-union over the queried window), a `.svar2`-backed dataset reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges under `/genotypes/svar2_ranges/` (sized to the dataset's *selected* samples, not the full `.svar2` cohort), and at read time gvl gathers directly off that cache and calls all-Rust kernels — **no interval-search-tree build and no dense-union rebuild per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. See `docs/source/faq.md`. + +`.svar2` is resolved at `Dataset.open` time in the same order as `.svar`: caller `svar2=` arg → recorded relative path → recorded absolute path → sibling `*.svar2`. `Dataset.open(path, svar2=)` mirrors `svar=`. See `docs/source/format.md` ("`.svar2` resolution at open time"). + +**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, but the following are not yet wired for `.svar2` and raise a clear error instead of silently mis-computing: +- Spliced output. +- The `var_filter="exonic"` (keep-mask) variant filter. +- `min_af` / `max_af` filtering. +- `annotated` haplotypes (`with_seqs("annotated")`). +- In-kernel reverse-complement (`to_rc`). +- `unphased_union`. +- `with_seqs("variant-windows")` (the flat-window variant mode). +- Fixed-length (integer `output_length`) haplotype-realigned **track** output (plain haplotype output at a fixed length is fine — only the track kernel is guarded). +- `variants` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound variants decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). +- `FlankSample` insertion-fill for tracks spanning **multiple contigs** in one query (single-contig queries and non-seeded fills like the default `Repeat5p` are exact). + +**`variants` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw `RaggedVariants.alt` bytes differ for pure-deletion records. See `docs/source/faq.md`. + +Symbolic/breakend variants are rejected the same as `.svar`, but for `.svar2` the rejection happens **upstream, at `.svar2` conversion time** (the store format cannot represent them) — a `.svar2` must be built from an already-filtered source; gvl cannot re-filter a materialized `.svar2` any more than it can a materialized `.svar`. + ## `gvl.write` — key arguments ```python @@ -149,11 +173,11 @@ gvl.Dataset.open( splice_info=None, # see "Spliced haplotypes" var_filter=None, # None | "exonic" var_fields=None, # list[str] | None — see below - *, svar=None, + *, svar=None, svar2=None, ) ``` -Without `reference=`, a genotypes-only dataset opens with the **`"variants"`** view by default (yielding `RaggedVariants`) — `Dataset.open(path)` just works, no `with_seqs` needed. The `"haplotypes"`, `"annotated"`, and `"reference"` views all require a reference; requesting one via `with_seqs` on a reference-less dataset raises a clear `ValueError`. `svar=` overrides the recorded SVAR location. +Without `reference=`, a genotypes-only dataset opens with the **`"variants"`** view by default (yielding `RaggedVariants`) — `Dataset.open(path)` just works, no `with_seqs` needed. The `"haplotypes"`, `"annotated"`, and `"reference"` views all require a reference; requesting one via `with_seqs` on a reference-less dataset raises a clear `ValueError`. `svar=` overrides the recorded SVAR location; `svar2=` mirrors it for a `.svar2`-backed dataset. **`with_settings(dummy_variant=...)`** — inserts a `gvl.DummyVariant` into every **empty** `(region, sample, ploid)` variant group so that every group has at least one variant. Only fills groups that are empty; non-empty groups are unchanged. Valid for both `"variants"` and `"variant-windows"` output kinds; indexing raises `ValueError` if `dummy_variant` is set and the output kind is any other kind (`"haplotypes"`, `"annotated"`, `"reference"`, or no seqs) — the check is order-independent with `with_seqs`. `False` disables dummy padding; `None` (default) leaves the current setting unchanged. Setting `dummy_variant=False` when the output is an unsupported kind is a harmless no-op. @@ -367,6 +391,7 @@ ds.gvl/ ├── input_regions.arrow # BED + region index map ├── genotypes/ # variant_idxs.npy, dosages.npy, variants.arrow │ # (absent when sourced from .svar; see svar_link) +│ # svar2_ranges/ present iff sourced from .svar2 (see svar2_link) ├── intervals// # per-sample track data (BigWigs / Table) └── annot_intervals// # sample-independent annotation track data ``` @@ -391,6 +416,7 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai | Track re-alignment internals | `python/genvarloader/_dataset/_tracks.py`, `_reconstruct.py` | | Insertion fill internals | `python/genvarloader/_dataset/_insertion_fill.py` | | SVAR back-reference / migration | `python/genvarloader/_dataset/_svar_link.py` | +| `.svar2` back-reference / read-bound wiring | `python/genvarloader/_dataset/_svar2_link.py`, `_svar2_haps.py` | | Format 1.x → 2.0 migration internals | `python/genvarloader/_dataset/_migrate.py` | | Flat-buffer ragged containers | `python/genvarloader/_flat.py` | | Flat variants + alleles types | `python/genvarloader/_dataset/_flat_variants.py` | @@ -402,7 +428,9 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`gvl.update` does not hot-reload open datasets.** A `Dataset` instance that was opened before `gvl.update` ran will not see the new track; reopen the dataset to pick it up. The update itself is safe to run while readers are active — each track is published atomically so a reader never sees a half-written track. - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). -- **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. +- **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, `unphased_union`, `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` `variants` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way — only raw `RaggedVariants.alt` differs for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. - `min_af` / `max_af` raise unless the dataset is SVAR-backed. From bbc773944ccc9d6ecb37262e3bf11933bad59ad6 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 22:55:06 -0700 Subject: [PATCH 040/108] docs(roadmap): SVAR2 read-bound benchmark results (perf DSO split + store sizes) MVP chr21 benchmark post-wiring: perf confirms the warm SVAR2 read builds zero SearchTree/dense_union (only gather_haps_readbound + merge_keys from genoray_core), flipping the ~80%-SearchTree union path found by E1. .svar2 5.67x smaller germline / 1.46x somatic on disk. Small-workload variants read ~3.4x slower than SVAR1 (per-read numpy cache-slice overhead); structural tree-free win pays off at scale. Co-Authored-By: Claude Opus 4.8 --- docs/roadmaps/rust-migration.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index fdfdba26..619e7e69 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -834,6 +834,25 @@ structurally read-bound query path (no per-read tree build, no per-read dense un smaller than `.svar` especially for large cohorts. Remaining genoray absorption (VCF/PGEN ingest, `.svar`→Rust, `.svar2` conversion/write path) stays in Phase 6 below. +**Benchmark (MVP chr21, `/carter/users/dlaub/projects/svar2_mvp`, relocated 2026-07-05; drivers +`bench_readbound.py`/`prof_svar2_read.py`, live outside the repo):** matched `.svar`/`.svar2` +datasets, warm `Dataset[:, :]` variants read, same-session relative (absolute wall-clock is not +comparable across allocations on shared Carter nodes). +- **On-disk store size:** germline (3202-sample cohort) `.svar2` **5.67× smaller** (193 MB vs 1.1 GB); + somatic (16007-sample) **1.46× smaller** (36 MB vs 53 MB). The `.svar2` advantage grows with cohort. +- **perf DSO split (the success criterion — MET):** `perf record`/`report` of the warm SVAR2 read + loop shows **zero** `SearchTree::build`, `dense_union`, or `overlap_batch` samples. The only + `genoray_core` symbols are `gather_haps_readbound` + `merge_keys` (the read-bound gather); the rest + is `genvarloader::svar2::{split_to_flat,decode_variants_from_split}` + numpy cache-slicing + (`PyArray_Repeat`/`mapiter_get`). This is the flip the wiring was designed for: the earlier E1 + profiling found the union (`overlap_batch`) path spent ~80% in genoray `SearchTree::build` + (rebuilt per read); the read-bound path builds none. +- **Latency (small workloads):** on 30 regions × 100–200 samples the SVAR2 variants read is ~3.4× + *slower* than SVAR1 (germline 1.5 ms vs 0.45 ms; somatic 2.0 ms vs 0.60 ms) — per-read Python/numpy + cache-slicing overhead dominates at this scale. The structural win (no per-read tree/union rebuild, + smaller store) is what pays off at cohort scale and for the union path's contig-wide-stride concern; + a fair large-workload latency sweep is a follow-up. + ### Phase 6 — Absorb genoray (future) ⬜ _PR: —_ From 60bdd4bd3bf0e4279a690a872861d84d40fe963f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 05:59:56 +0000 Subject: [PATCH 041/108] chore(pre-commit): auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- python/genvarloader/_dataset/_svar2_source.py | 33 ++++++---- tests/test_svar2_realign_tracks.py | 30 ++++++---- tests/test_svar2_reconstruct.py | 17 ++++-- tests/unit/dataset/test_svar2_store.py | 11 +++- tmp/svar2_mvp/benchmark.py | 60 +++++++++++++------ tmp/svar2_mvp/build_stores.py | 16 ++++- tmp/svar2_mvp/e1_bucket_dso.py | 16 +++-- tmp/svar2_mvp/prof_driver.py | 23 +++++-- tmp/svar2_mvp/split_folded.py | 8 ++- tmp/svar2_mvp/validate.py | 35 +++++++---- 10 files changed, 178 insertions(+), 71 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_source.py b/python/genvarloader/_dataset/_svar2_source.py index 8d61f520..7f248437 100644 --- a/python/genvarloader/_dataset/_svar2_source.py +++ b/python/genvarloader/_dataset/_svar2_source.py @@ -44,22 +44,22 @@ def _query(self, contig, regions): P = int(d["ploidy"]) reg = np.asarray(regions, dtype=np.int32).reshape(R, 2) # (R*S, 3): contig_idx=0, start, end — repeat each query region S times. - reg_rs = np.repeat(reg, S, axis=0) # (R*S, 2) + reg_rs = np.repeat(reg, S, axis=0) # (R*S, 2) regions_gvl = np.zeros((R * S, 3), dtype=np.int32) regions_gvl[:, 1:] = reg_rs dense_range_gvl = np.ascontiguousarray( np.repeat(np.asarray(d["dense_range"], np.int32), S, axis=0), np.int32 - ) # (R*S, 2) + ) # (R*S, 2) return d, R, S, P, regions_gvl, dense_range_gvl def reconstruct( self, contig: str, - regions, # iterable of (start, end), length R - ref_: "NDArray[np.uint8]", # the contig reference bytes - ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) + regions, # iterable of (start, end), length R + ref_: "NDArray[np.uint8]", # the contig reference bytes + ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) pad_char: int, - shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros output_length: int = -1, parallel: bool = False, ) -> "Ragged[np.bytes_]": @@ -89,14 +89,17 @@ def reconstruct( parallel, ) shape = (R, S, P, None) - return cast("Ragged[np.bytes_]", _Flat.from_offsets(out_data, shape, out_offsets).view("S1")) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, shape, out_offsets).view("S1"), + ) def realign_tracks( self, contig: str, regions, - tracks: "NDArray[np.float32]", # flat per-query track buffer - track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks + tracks: "NDArray[np.float32]", # flat per-query track buffer + track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks params: "NDArray[np.float64]", strategy_id: int, base_seed: int, @@ -113,7 +116,13 @@ def realign_tracks( # (= r*S+s), so expand the R track windows to R*S by repeating each S times. t = np.asarray(tracks, np.float32) toff = np.asarray(track_offsets, np.int64) - tracks_rs = np.concatenate([t[toff[r]:toff[r + 1]] for r in range(R) for _ in range(S)]) if R else t + tracks_rs = ( + np.concatenate( + [t[toff[r] : toff[r + 1]] for r in range(R) for _ in range(S)] + ) + if R + else t + ) lengths = np.repeat(np.diff(toff), S) track_offsets_rs = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) out_data, out_offsets = shift_and_realign_tracks_from_svar2( @@ -137,4 +146,6 @@ def realign_tracks( parallel, ) shape = (R, S, P, None) - return cast("Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets)) + return cast( + "Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets) + ) diff --git a/tests/test_svar2_realign_tracks.py b/tests/test_svar2_realign_tracks.py index c3d04f2d..da7c1884 100644 --- a/tests/test_svar2_realign_tracks.py +++ b/tests/test_svar2_realign_tracks.py @@ -47,8 +47,15 @@ def svar2_del_store(tmp_path_factory) -> Path: out = d / "store" _core.run_conversion_pipeline( - str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], - 25_000, 2, 1, 8 * 1024 * 1024, + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, ) assert (out / "meta.json").exists(), "conversion did not finish" return out @@ -73,7 +80,7 @@ def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): rng = np.random.default_rng(0) track = rng.random(region_len).astype(np.float32) - strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) + strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) params = np.zeros(1, np.float64) base_seed = 0 @@ -82,12 +89,12 @@ def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): out_rag = src.realign_tracks( contig, regions, - track, # flat per-region track buffer - np.array([0, region_len], np.int64), # (R+1) offsets + track, # flat per-region track buffer + np.array([0, region_len], np.int64), # (R+1) offsets params, strategy_id, base_seed, - shifts=None, # no jitter + shifts=None, # no jitter parallel=False, ) @@ -95,7 +102,7 @@ def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): raw = sv._readers[contig].decode_batch([(q_start, q_end)]) R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) assert (R, So, Po) == (1, S, P) - off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets d_pos = np.asarray(raw["pos"]) d_ilen = np.asarray(raw["ilen"]) @@ -112,7 +119,7 @@ def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): for s in range(S): for p in range(P): - h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p + h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p gi0, gi1 = int(off[h]), int(off[h + 1]) pos_h = np.ascontiguousarray(d_pos[gi0:gi1], np.int32) ilen_h = np.ascontiguousarray(d_ilen[gi0:gi1], np.int32) @@ -148,7 +155,10 @@ def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): hap=h, ) np.testing.assert_allclose( - got, expected, rtol=0, atol=0, + got, + expected, + rtol=0, + atol=0, err_msg=f"(s={s},p={p}) SVAR2 track != SVAR1 oracle " - f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", ) diff --git a/tests/test_svar2_reconstruct.py b/tests/test_svar2_reconstruct.py index 83c9d662..887c93fd 100644 --- a/tests/test_svar2_reconstruct.py +++ b/tests/test_svar2_reconstruct.py @@ -47,8 +47,15 @@ def svar2_store(tmp_path_factory) -> Path: out = d / "store" _core.run_conversion_pipeline( - str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], - 25_000, 2, 1, 8 * 1024 * 1024, + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, ) assert (out / "meta.json").exists(), "conversion did not finish" return out @@ -104,8 +111,8 @@ def test_svar2_two_source_matches_decode_oracle(svar2_store): np.frombuffer(ref_bytes, np.uint8), np.array([0, len(ref_bytes)], np.int64), pad_char=ord("N"), - shifts=None, # no jitter - output_length=-1, # ragged + shifts=None, # no jitter + output_length=-1, # ragged parallel=False, ) ts_data = np.asarray(hap_rag.data).view("S1").tobytes() @@ -116,7 +123,7 @@ def test_svar2_two_source_matches_decode_oracle(svar2_store): R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) assert (R, So, Po) == (1, S, P) H = R * So * Po - off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets str_off = np.asarray(raw["str_off"]) # per-variant allele-byte offsets d_pos = np.asarray(raw["pos"]) d_ilen = np.asarray(raw["ilen"]) diff --git a/tests/unit/dataset/test_svar2_store.py b/tests/unit/dataset/test_svar2_store.py index d820adb5..8f18d5f0 100644 --- a/tests/unit/dataset/test_svar2_store.py +++ b/tests/unit/dataset/test_svar2_store.py @@ -41,8 +41,15 @@ def svar2_store(tmp_path_factory) -> Path: out = d / "cohort.svar2" _core.run_conversion_pipeline( - str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], - 25_000, 2, 1, 8 * 1024 * 1024, + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, ) assert (out / "meta.json").exists(), "conversion did not finish" return out diff --git a/tmp/svar2_mvp/benchmark.py b/tmp/svar2_mvp/benchmark.py index c06bee84..31f409b1 100644 --- a/tmp/svar2_mvp/benchmark.py +++ b/tmp/svar2_mvp/benchmark.py @@ -1,6 +1,7 @@ """Benchmark SVAR1 (gvl Dataset over .svar) vs SVAR2 (SparseVar2Source over .svar2): hap latency, variant latency, store size, for one source prefix. Fair workload: ALL samples for a fixed region set. Warm caches, median of N.""" + import sys import time import subprocess @@ -14,10 +15,13 @@ REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" N = 5 # repeats + def _contig_ref(fasta, chrom): import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + def _timed(fn, warmup=1): for _ in range(warmup): fn() @@ -28,9 +32,13 @@ def _timed(fn, warmup=1): ts.append(time.perf_counter() - t0) return median(ts) + def main(prefix, chrom): - regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), - (40_000_000, 40_001_000)] + regions = [ + (20_000_000, 20_001_000), + (30_000_000, 30_000_500), + (40_000_000, 40_001_000), + ] ref_bytes = _contig_ref(REF, chrom) ref_u8 = np.frombuffer(ref_bytes, np.uint8) ref_off = np.array([0, len(ref_bytes)], np.int64) @@ -38,16 +46,29 @@ def main(prefix, chrom): # SVAR2 backend sv2 = SparseVar2(f"{prefix}.svar2") src = SparseVar2Source(sv2) - svar2_hap = _timed(lambda: src.reconstruct( - chrom, regions, ref_u8, ref_off, pad_char=ord("N"), - shifts=None, output_length=-1)) + svar2_hap = _timed( + lambda: src.reconstruct( + chrom, + regions, + ref_u8, + ref_off, + pad_char=ord("N"), + shifts=None, + output_length=-1, + ) + ) svar2_var = _timed(lambda: sv2.decode(chrom, regions)) # SVAR1 backend (all samples, same regions) import polars as pl - bed = pl.DataFrame({"chrom": [chrom] * len(regions), - "chromStart": [s for s, _ in regions], - "chromEnd": [e for _, e in regions]}) + + bed = pl.DataFrame( + { + "chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions], + } + ) ds_path = f"{prefix}.gvl" # Write the Dataset over the SAME region set the SVAR2 path benchmarks, so both # backends measure an identical workload (fairness rule). validate.py may have left @@ -57,19 +78,24 @@ def main(prefix, chrom): ds_hap = ds.with_seqs("haplotypes") ds_var = ds.with_seqs("variants") n_s = sv2.n_samples - svar1_hap = _timed(lambda: ds_hap[:len(regions), :n_s]) - svar1_var = _timed(lambda: ds_var[:len(regions), :n_s]) + svar1_hap = _timed(lambda: ds_hap[: len(regions), :n_s]) + svar1_var = _timed(lambda: ds_var[: len(regions), :n_s]) def du(path): - return subprocess.run(["du", "-sb", path], capture_output=True, - text=True).stdout.split()[0] + return subprocess.run( + ["du", "-sb", path], capture_output=True, text=True + ).stdout.split()[0] - print(f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " - f"regions={len(regions)} N={N}") + print( + f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " + f"regions={len(regions)} N={N}" + ) print(f" hap_latency_s svar1={svar1_hap:.4f} svar2={svar2_hap:.4f}") print(f" var_latency_s svar1={svar1_var:.4f} svar2={svar2_var:.4f}") - print(f" store_bytes svar1={du(prefix + '.svar')} " - f"svar2={du(prefix + '.svar2')}") + print( + f" store_bytes svar1={du(prefix + '.svar')} svar2={du(prefix + '.svar2')}" + ) + if __name__ == "__main__": - main(sys.argv[1], sys.argv[2]) # argv: + main(sys.argv[1], sys.argv[2]) # argv: diff --git a/tmp/svar2_mvp/build_stores.py b/tmp/svar2_mvp/build_stores.py index 935a0cda..3e8efb8a 100644 --- a/tmp/svar2_mvp/build_stores.py +++ b/tmp/svar2_mvp/build_stores.py @@ -1,25 +1,35 @@ """Build .svar (SVAR1) and .svar2 (SVAR2) stores from a normalized biallelic BCF.""" + import sys from pathlib import Path from genoray import VCF, SparseVar, _core + def build(bcf: str, chrom: str, samples: list[str], out_prefix: str, ploidy: int): bcf = str(bcf) # SVAR 1.0 SparseVar.from_vcf(f"{out_prefix}.svar", VCF(bcf), "8g", overwrite=True) # SVAR 2.0 _core.run_conversion_pipeline( - bcf, "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", - [chrom], f"{out_prefix}.svar2", samples, - 25_000, ploidy, 8, 8 * 1024 * 1024, + bcf, + "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", + [chrom], + f"{out_prefix}.svar2", + samples, + 25_000, + ploidy, + 8, + 8 * 1024 * 1024, ) print(f"built {out_prefix}.svar and {out_prefix}.svar2") + if __name__ == "__main__": # argv: bcf, chrom, out_prefix = sys.argv[1], sys.argv[2], sys.argv[3] import subprocess + samples = subprocess.run( ["bcftools", "query", "-l", bcf], capture_output=True, text=True, check=True ).stdout.split() diff --git a/tmp/svar2_mvp/e1_bucket_dso.py b/tmp/svar2_mvp/e1_bucket_dso.py index 3d9ce852..ada5d1c6 100644 --- a/tmp/svar2_mvp/e1_bucket_dso.py +++ b/tmp/svar2_mvp/e1_bucket_dso.py @@ -3,11 +3,17 @@ report --stdio --sort=dso --no-children -i data.perf | python e1_bucket_dso.py """ + import re import sys -buckets = {"native-gvl": 0.0, "native-genoray": 0.0, "numpy-conv": 0.0, - "python-interp": 0.0, "other": 0.0} +buckets = { + "native-gvl": 0.0, + "native-genoray": 0.0, + "numpy-conv": 0.0, + "python-interp": 0.0, + "other": 0.0, +} for line in sys.stdin: if line.lstrip().startswith("#") or not line.strip(): continue @@ -32,5 +38,7 @@ for k in ("native-gvl", "native-genoray", "numpy-conv", "python-interp", "other"): print(f" {buckets[k]:6.1f}% {k}") nat = buckets["native-gvl"] + buckets["native-genoray"] -print(f" ---- native(gvl+genoray)={nat:.1f}% numpy-conv={buckets['numpy-conv']:.1f}% " - f"python-interp={buckets['python-interp']:.1f}% (sum {tot:.1f}%)") +print( + f" ---- native(gvl+genoray)={nat:.1f}% numpy-conv={buckets['numpy-conv']:.1f}% " + f"python-interp={buckets['python-interp']:.1f}% (sum {tot:.1f}%)" +) diff --git a/tmp/svar2_mvp/prof_driver.py b/tmp/svar2_mvp/prof_driver.py index 060526a0..af22ce77 100644 --- a/tmp/svar2_mvp/prof_driver.py +++ b/tmp/svar2_mvp/prof_driver.py @@ -6,6 +6,7 @@ Prints: per_call_s= For svar1, the 3-region .gvl is written ONCE before the loop (we profile the query, not gvl.write).""" + import sys import time @@ -19,6 +20,7 @@ def _ref(): import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() return np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) @@ -26,12 +28,15 @@ def _ref(): def make_svar2(cohort): from genoray import SparseVar2 from genvarloader._dataset._svar2_source import SparseVar2Source + src = SparseVar2Source(SparseVar2(f"{W}/{cohort}.svar2")) ru, ro = _ref() def call(): - src.reconstruct(CHROM, REGIONS, ru, ro, pad_char=ord("N"), - shifts=None, output_length=-1) + src.reconstruct( + CHROM, REGIONS, ru, ro, pad_char=ord("N"), shifts=None, output_length=-1 + ) + return call @@ -39,16 +44,22 @@ def make_svar1(cohort): import polars as pl import genvarloader as gvl from genoray import SparseVar2 + n_s = SparseVar2(f"{W}/{cohort}.svar2").n_samples - bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), - "chromStart": [s for s, _ in REGIONS], - "chromEnd": [e for _, e in REGIONS]}) + bed = pl.DataFrame( + { + "chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS], + } + ) ds_path = f"{W}/{cohort}.gvl" gvl.write(ds_path, bed, variants=f"{W}/{cohort}.svar", overwrite=True) # ONCE ds_hap = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") def call(): - ds_hap[:len(REGIONS), :n_s] + ds_hap[: len(REGIONS), :n_s] + return call diff --git a/tmp/svar2_mvp/split_folded.py b/tmp/svar2_mvp/split_folded.py index 3fd1f621..d85c9fe2 100644 --- a/tmp/svar2_mvp/split_folded.py +++ b/tmp/svar2_mvp/split_folded.py @@ -3,6 +3,7 @@ python split_folded.py """ + import sys from collections import Counter @@ -34,8 +35,11 @@ def main(path): nat += n tot = py + nat if tot == 0: - print("no samples parsed"); return - print(f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}") + print("no samples parsed") + return + print( + f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}" + ) print("top-15 leaf frames (self-time):") for leaf, n in leaves.most_common(15): print(f" {100 * n / tot:5.1f}% [{classed[leaf]:6s}] {leaf}") diff --git a/tmp/svar2_mvp/validate.py b/tmp/svar2_mvp/validate.py index 4331ece5..df32bc47 100644 --- a/tmp/svar2_mvp/validate.py +++ b/tmp/svar2_mvp/validate.py @@ -2,6 +2,7 @@ both the SVAR1 (gvl Dataset over .svar) and SVAR2 (SparseVar2Source over .svar2) backends, on a handful of regions x a few samples. Correctness is already proven by the test suite; this proves the REAL-DATA plumbing works.""" + import sys from pathlib import Path @@ -12,6 +13,7 @@ REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + def main(prefix: str, chrom: str): # A few small regions (0-based, half-open) in a variant-dense chr21 window. regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500)] @@ -22,33 +24,44 @@ def main(prefix: str, chrom: str): ref_bytes = _contig_ref(REF, chrom) src = SparseVar2Source(sv2) hap = src.reconstruct( - chrom, regions, + chrom, + regions, np.frombuffer(ref_bytes, np.uint8), np.array([0, len(ref_bytes)], np.int64), - pad_char=ord("N"), shifts=None, output_length=-1, + pad_char=ord("N"), + shifts=None, + output_length=-1, ) lens = np.asarray(hap.offsets) - print(f"[svar2] hap ragged rows={len(lens) - 1} " - f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}") + print( + f"[svar2] hap ragged rows={len(lens) - 1} " + f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}" + ) var = sv2.decode(chrom, regions) print(f"[svar2] decode variants: {var}") # --- SVAR1 backend (gvl Dataset over .svar) --- import polars as pl - bed = pl.DataFrame({ - "chrom": [chrom] * len(regions), - "chromStart": [s for s, _ in regions], - "chromEnd": [e for _, e in regions], - }) + + bed = pl.DataFrame( + { + "chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions], + } + ) ds_path = f"{prefix}.gvl" gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) ds = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") - seqs = ds[:len(regions), :3] # a few regions x first 3 samples + seqs = ds[: len(regions), :3] # a few regions x first 3 samples print(f"[svar1] gvl haplotypes sample shape/type: {type(seqs)}") + def _contig_ref(fasta: str, chrom: str) -> bytes: import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + if __name__ == "__main__": - main(sys.argv[1], sys.argv[2]) # argv: + main(sys.argv[1], sys.argv[2]) # argv: From 5989b23298a8f911b2ff5f13a9c110381dd48571 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 5 Jul 2026 23:53:43 -0700 Subject: [PATCH 042/108] docs(spec): profile & optimize svar2 read-bound Dataset.__getitem__ path Design for profiling the live read-bound path (Svar2Haps -> *_readbound kernels), not the retired union oracle. py-spy is unusable (ptrace_scope=2) and Python 3.10 blocks perf trampoline, so cProfile+pyinstrument (python fns) + perf (native %) substitute. Two-repo landing: gvl kernels on #266, genoray kernels on svar-2. All three modes; parity-gated, instruction-count deltas. Co-Authored-By: Claude Opus 4.8 --- ...-05-svar2-readbound-getitem-perf-design.md | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md diff --git a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md new file mode 100644 index 00000000..48cbf4a1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md @@ -0,0 +1,155 @@ +# Profile & optimize the SVAR2 read-bound `Dataset.__getitem__` path + +> **Status:** design approved · **Date:** 2026-07-05 · **Repos/branches:** +> GenVarLoader `svar2-m6b-kernel` (PR #266, draft) + genoray `svar-2` (@ `aaf44fd`) +> +> Big picture: genoray `docs/roadmap/svar-2.md` (milestones M6b/M6d/M6e — the +> read-bound gather this work profiles). + +## 1. Motivation + +`Dataset.__getitem__` for a `.svar2`-backed dataset now dispatches through +`Svar2Haps` (`python/genvarloader/_dataset/_svar2_haps.py`) → the +`*_from_svar2_readbound` FFI kernels → genoray_core `gather_haps_readbound`. This +path builds **zero** interval-search trees and **zero** dense-union per read (the +structural win M6d/M6e were built for). + +The only profiling artifacts on disk (`tmp/svar2_mvp/prof_out/e1`) measure the +**retired union oracle** — `SparseVar2Source.reconstruct` → genoray +`overlap_batch` → 68% `SearchTree::build`. That path is no longer on any live +read. **We are profiling the real read-bound path effectively from scratch.** + +The PR bench also noted the small-workload latency is dominated by "per-read +Python/numpy cache-slicing overhead," never attributed to specific functions. +This effort attributes it and removes what is removable. + +Goal: identify the hottest functions in the live read-bound `Dataset.__getitem__` +path across all three supported modes, then optimize — Python by static +analysis/inspection, hot Rust by inspecting `cargo asm` — with parity preserved. + +## 2. Scope + +**In scope** — the live read path for the three supported output modes: + +- **haplotypes** — `Svar2Haps.get_haps_and_shifts` → `hap_diffs_from_svar2_readbound` + + `reconstruct_haplotypes_from_svar2_readbound`. +- **variants** — `Svar2Haps._reconstruct_variants` → `decode_variants_from_svar2_readbound`. +- **tracks** — `Svar2Haps.realign_track_block` → `intervals_to_tracks` + + `shift_and_realign_tracks_from_svar2_readbound`. + +Shared Rust spine (candidates for `cargo asm`): gvl-side +`svar2::split_to_flat` / `hap_diffs_svar2` / `reconstruct_haplotypes_from_svar2` +(`src/svar2/mod.rs`, `src/reconstruct.rs`, `src/ffi/mod.rs`); genoray-side +`query::gather_haps_readbound` / `spine::merge_keys` +(`genoray/src/query.rs`, `genoray/src/spine.rs`). + +**Out of scope:** the guarded-`NotImplementedError` modes (annotated, spliced, +`min_af`/`max_af`, in-kernel RC, `unphased_union`, variant-windows, +`max_jitter>0` variants); any on-disk **format** or **public API** change; the +union oracle (`SparseVar2Source`, `overlap_batch`) except as the parity oracle; +`gvl.write` (the write-time ranges cache producer). + +## 3. Landing targets (two repos) + +| Change site | Repo / branch | Rebuild to profile | +| --- | --- | --- | +| gvl Rust kernels (`src/svar2`, `src/reconstruct`, `src/ffi`) + Python (`_svar2_haps.py`) | GenVarLoader `svar2-m6b-kernel` (PR #266) | `maturin develop --release` | +| genoray Rust kernels (`gather_haps_readbound`, `merge_keys`) | genoray `svar-2` | gvl `maturin develop --release` rebuilds the `genoray_core` **crate path-dep** automatically | + +**Wheel caveat:** the Python `genoray` package is a pre-built cp310 wheel +(`pixi.toml`). It only needs rebuilding if a change touches genoray's **Python +API** (e.g. a `find_ranges` dict key). Pure hot-path kernel edits consumed by gvl +through the `genoray_core` crate do **not** need a genoray wheel rebuild. + +## 4. Tooling — forced substitution for py-spy + +Empirically verified on the Carter node this runs on: + +- **py-spy is unusable** — `ptrace_scope=2` + no sudo → "Permission Denied" for + `dump`/`record`, `--native` or not. +- **Python is 3.10** in `-e dev` → no `perf` trampoline (`-Xperf` is 3.12+), so + `perf` **cannot resolve Python frames** (Python stays a single opaque DSO). + +We recover both halves py-spy would have given from two ptrace-free tools: + +| Layer | Tool | Output | +| --- | --- | --- | +| Python fns ("python fns") | **cProfile** (stdlib) + **pyinstrument** (added to pixi deps for a low-overhead statistical wall-clock call-tree cross-check) | per-function cumulative/total time → the Python functions to inspect | +| Native/Rust ("total native %") | **perf** `record -g --call-graph fp -F 199`, run on `.pixi/envs/dev/bin/python` **directly** (not via `pixi run` — the launcher eats ~60% of samples), built with `-C force-frame-pointers=yes` | DSO split (native % = "total native %"), Rust symbol self-time, and the fp call-graph → the Rust fns to `cargo asm` | + +`pyinstrument` is added to `pixi.toml` deps (gvl, and genoray if it profits). + +## 5. Benchmark harness (rebuilt, real path) + +The existing `tmp/svar2_mvp` drivers profile the **wrong** (union) path and +reference a **stale** store path (`repos/for_loukik/...`). Stores actually live at +`/carter/users/dlaub/projects/svar2_mvp/{germline,somatic}.{svar,svar2}`; +reference FASTA `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` is present. + +Add a driver under `tmp/svar2_mvp/` that exercises the **live** +`Dataset.open(path, reference=REF, svar2=).with_seqs(mode)[regions, samples]` +warm loop for `mode ∈ {haplotypes, variants, tracks}`, over both cohorts +(germline 3202-smp, somatic 16007-smp), same fair workload as the PR bench +(fixed region set × all samples, warm caches, median of N). One code path per +capture (mirrors `prof_driver.py`) so cProfile/perf attribute cleanly. A tracks +run needs a BigWig or table track attached to the dataset; haplotypes/variants +reuse the PR bench setup. + +## 6. Optimization method (measure → confirm → fix → re-measure) + +Profile first; fix only what ranks hot. Static inspection already surfaces +concrete candidates to **confirm against the profile before touching**: + +- **Redundant double gather (Python/FFI, high confidence):** + `get_haps_and_shifts` calls `hap_diffs_from_svar2_readbound` *and then* + `reconstruct_haplotypes_from_svar2_readbound` per contig group — **both** run + the full `gather_haps_readbound` + `split_to_flat` internally, so genoray's + gather + AoS→SoA marshalling executes **twice per haplotype/track read**. + Candidate fix: gather once (fuse diffs into the reconstruct gather, or return + diffs from a single kernel), so the read draws shifts and reconstructs off one + gather. +- **Python FFI-shaping (`_svar2_haps.py`):** per-contig-group Python loops, + repeated `ascontiguousarray` copies in `_gather_inputs`, the + `_ragged_arange_gather` permutation passes — vectorize / drop copies where the + profile justifies. +- **Rust hot fns via `cargo asm`:** likely `svar2::split_to_flat` (AoS→SoA copy), + `gather_haps_readbound`, `merge_keys`, `hap_diffs_svar2`, and the + `reconstruct_haplotypes_from_svar2` inner kernel. `cargo asm` targets: + bounds-check elision, autovectorization of copy/merge loops, alloc churn + (`SpecFromIter`/`_int_malloc` were hot even in the old run). + +## 7. Measurement discipline & parity gate + +Per hard-won project lessons (shared-node noise): + +- **No cross-session absolute wall-clock claims.** Gate each optimization on + **same-session before/after** plus a **deterministic `perf stat -e + instructions,cycles`** delta (instruction count is noise-free). Record the + instruction-count delta per change. +- **Parity is a hard gate.** The read-bound kernels are byte-identical to the + union oracle. After every change: `pixi run -e dev pytest tests -q` (full tree + — the svar2 suite is 31/31, plus the parity oracle), and for Rust changes + `maturin develop --release` **first** (pytest imports the stale `.so` + otherwise), plus `cargo test` on both repos. Any divergence blocks the change. +- The two documented intentional non-identities (pure-DEL ALT `b""`; SVAR1 + `max_ends` tie under-extension) are pre-existing and untouched. + +## 8. Deliverables + +1. A profiling report (cProfile + pyinstrument + perf DSO/symbol/callgraph + tables, per mode × cohort) under `tmp/svar2_mvp/prof_out/`. +2. Confirmed optimizations implemented: gvl-side on `svar2-m6b-kernel` (#266), + genoray-side on `svar-2` — each parity-gated, each with a recorded + same-session instruction-count delta. +3. `pyinstrument` added to `pixi.toml`. +4. No format/API/doc-surface changes (read-path internals only), so the + skill/api.md/docs gates do not apply; the genoray roadmap needs no milestone + flip (this is perf work under shipped M6b/M6d/M6e, noted in passing if a + kernel signature changes). + +## 9. Open questions + +- Whether the double-gather fix needs a **new fused kernel signature** (a genoray + `svar-2` API touch → genoray wheel rebuild) or can be done gvl-side by having + the reconstruct kernel also return diffs (no genoray API change). Resolve after + the profile confirms the gather is actually the dominant cost. From 074c7e80e650765e9ebe747ca32a0e19f498ac66 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 00:09:55 -0700 Subject: [PATCH 043/108] docs(plan): svar2 read-bound getitem profiling & optimization (haplotypes + variants) Phase A: live-path profiling harness (cProfile+pyinstrument for Python fns, perf DSO/symbol for native) + committed baseline. Phase B: haplotype double-gather elimination (B1), split_to_flat + decode_variants_from_split alloc presize (B2), variants twin-ragged-gather dedup (B3), cargo asm pass on top native symbol (B4), results (B5). Tracks out of scope but parity-protected; variant-windows deferred (guarded). Parity + instruction-count gated. Co-Authored-By: Claude Opus 4.8 --- ...2026-07-05-svar2-readbound-getitem-perf.md | 758 ++++++++++++++++++ 1 file changed, 758 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md diff --git a/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md new file mode 100644 index 00000000..2931870c --- /dev/null +++ b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md @@ -0,0 +1,758 @@ +# SVAR2 read-bound `Dataset.__getitem__` profiling & optimization — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Attribute and then remove the hottest per-read costs in the live SVAR2 read-bound `Dataset.__getitem__` path for the two supported in-scope modes — **haplotypes** and **variants** — Python by inspection and Rust by `cargo asm`, with byte-identical parity preserved. + +**Architecture:** Phase A stands up a deterministic profiling harness on the *real* `Dataset.open(svar2).with_seqs(mode)[...]` path (not the retired union oracle) and captures a committed baseline: cProfile + pyinstrument for Python functions, `perf` DSO-split + Rust symbol self-time for native, and `perf stat -e instructions` for a noise-free gate. Phase B applies optimizations one at a time — each confirmed against the Phase A profile before it is written, then gated on byte-identical parity plus a same-session instruction-count delta. gvl-side changes land on `svar2-m6b-kernel` (PR #266); genoray-side changes land on genoray `svar-2`. + +**Tech Stack:** Python 3.10 (`-e dev` pixi env), Rust (maturin/PyO3), numpy, genoray_core (Rust crate path-dep) + genoray wheel, cProfile/pyinstrument, `perf`, `cargo asm`. + +## Global Constraints + +- **Design spec:** `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` (read it first). +- **In-scope modes:** **haplotypes** and **variants** only. **Tracks is out of scope** (not profiled) but its parity tests MUST stay green — no change may break the tracks path. **variant-windows** is a consumer target but is currently guarded `NotImplementedError` in `Svar2Haps.__call__` (`_FlatVariantWindows`), so it cannot be profiled or optimized here — note it as deferred until implemented. +- **Two repos / branches:** gvl = this worktree, branch `svar2-m6b-kernel` (PR #266). genoray = `/carter/users/dlaub/projects/genoray`, branch `svar-2` (currently @ `aaf44fd`). +- **Rebuild before testing/profiling any Rust change:** `pixi run -e dev maturin develop --release` — pytest/perf import the stale `.so` otherwise. genoray crate edits are picked up by this same gvl rebuild (path-dep); a genoray **wheel** rebuild is only needed if a genoray **Python-API** surface changes (it will not here). +- **Profiling build must keep frame pointers:** build the profiled `.so` with `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` (perf fp call-graph needs them). +- **Parity is a hard gate.** After every change: `pixi run -e dev pytest tests -q` (full tree — the svar2 suite + parity oracle live across `tests/dataset` and `tests/unit`; this includes the tracks parity tests), plus `cargo test` on the repo(s) touched. The read-bound kernels are byte-identical to the union oracle; any divergence blocks the change. Two documented intentional non-identities (pure-DEL ALT `b""`; SVAR1 `max_ends` tie under-extension) are pre-existing — do not "fix" them. +- **Measurement discipline.** No cross-session absolute wall-clock claims (shared Carter node). Gate on **same-session before/after** + **`perf stat -e instructions,cycles`** deltas. If HW counters are unavailable (`perf stat` errors with "not supported"), fall back to cProfile total primitive-call counts + same-session median wall-clock, and say so in the recorded result. +- **Fixed inputs (verified present):** + - Stores: `/carter/users/dlaub/projects/svar2_mvp/{germline,somatic}.{svar,svar2}`. + - Reference: `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`, contig `chr21`. + - Env python (call directly for perf; never via `pixi run` — the launcher eats ~60% of samples): `.pixi/envs/dev/bin/python`. + - perf binary: `/carter/users/dlaub/.pixi/bin/perf`. +- **py-spy is unusable here** (`ptrace_scope=2`, no sudo) and Python 3.10 has no perf trampoline — do not attempt py-spy or `perf` Python-frame resolution. Python attribution comes only from cProfile/pyinstrument. +- **Commit hygiene:** docs-only commits may need `--no-verify` (the `pyrefly` pre-commit hook fails spuriously on zero-Python-file commits and collides with the unstaged `pixi.lock`). Code commits run hooks normally; ensure prek hooks are installed first (`prek install` if needed). +- **Live read path recipe** (how every driver builds the real path): + ```python + import genvarloader as gvl + from genoray import SparseVar2 + gvl.write(ds_path, bed, variants=SparseVar2(f"{prefix}.svar2"), samples=None, + max_jitter=0, overwrite=True) # ONCE, before the profiled loop + ds = gvl.Dataset.open(ds_path, reference=REF) + ds.with_seqs("haplotypes")[:, :] # or with_seqs("variants") + ``` + +--- + +## Phase A — Profiling harness & committed baseline + +### Task A1: Add pyinstrument + write the live-path profiling driver + +**Files:** +- Modify: `pixi.toml` (add `pyinstrument` to the dev feature deps) +- Create: `tmp/svar2_mvp/prof_getitem.py` + +**Interfaces:** +- Produces: a CLI `python tmp/svar2_mvp/prof_getitem.py ` where `mode ∈ {haplotypes, variants}`, `cohort ∈ {germline, somatic}`; runs `gvl.write` + `Dataset.open` ONCE, then calls the selected read `K` times in a warm loop; prints `per_call_s=`. Exposes a module function `make_call(mode, cohort) -> Callable[[], object]` reused by the cProfile/perf drivers in A2/A3. + +- [ ] **Step 1: Add pyinstrument to the dev env** + +Find the dev feature's `pypi-dependencies` (or `dependencies`) table in `pixi.toml` and add `pyinstrument`: + +```toml +# in the [feature.dev.pypi-dependencies] (or nearest dev deps) table +pyinstrument = "*" +``` + +- [ ] **Step 2: Install it** + +Run: `pixi install -e dev` +Expected: resolves and installs `pyinstrument`; no other deps change materially. + +- [ ] **Step 3: Verify import** + +Run: `.pixi/envs/dev/bin/python -c "import pyinstrument; print(pyinstrument.__version__)"` +Expected: prints a version (e.g. `4.x`). + +- [ ] **Step 4: Write the driver** + +Create `tmp/svar2_mvp/prof_getitem.py`: + +```python +"""Profile the LIVE SVAR2 read-bound Dataset.__getitem__ path (not the union +oracle) for the in-scope modes. One (mode, cohort) per process so cProfile/perf +attribute cleanly. + + python tmp/svar2_mvp/prof_getitem.py + +gvl.write + Dataset.open run ONCE (we profile the READ, not the write). Prints +per_call_s over K warm calls. Tracks mode is out of scope; variant-windows is +guarded NotImplementedError in Svar2Haps and cannot be profiled yet.""" +import sys +import time +from pathlib import Path + +import polars as pl + +STORE_DIR = Path("/carter/users/dlaub/projects/svar2_mvp") +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] +WORK = Path("tmp/svar2_mvp/prof_out/readbound") + + +def _bed(): + return pl.DataFrame({ + "chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS], + }) + + +def make_call(mode, cohort): + import genvarloader as gvl + from genoray import SparseVar2 + + prefix = STORE_DIR / cohort + sv2 = SparseVar2(f"{prefix}.svar2") + n_s = sv2.n_samples + ds_path = WORK / f"{cohort}_{mode}.gvl" + WORK.mkdir(parents=True, exist_ok=True) + + gvl.write(ds_path, _bed(), variants=SparseVar2(f"{prefix}.svar2"), + samples=None, max_jitter=0, overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF) + view = ds.with_seqs(mode) # "haplotypes" or "variants" + + R = len(REGIONS) + + def call(): + view[:R, :n_s] + + return call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + print(f"per_call_s={(time.perf_counter() - t0) / K:.5f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 5: Smoke-test both modes (germline, small K)** + +Run: +```bash +cd "$(git rev-parse --show-toplevel)" +for m in haplotypes variants; do + echo "== $m =="; .pixi/envs/dev/bin/python tmp/svar2_mvp/prof_getitem.py $m germline 3 +done +``` +Expected: each prints `per_call_s=` with no exception. (haplotypes needs the reference — it is set; no BigWig is required for either mode.) + +- [ ] **Step 6: Commit** + +```bash +git add pixi.toml pixi.lock tmp/svar2_mvp/prof_getitem.py +git commit -m "perf(svar2): live read-bound Dataset.__getitem__ profiling driver + pyinstrument" +``` + +--- + +### Task A2: Capture the Python-layer baseline (cProfile + pyinstrument) + +**Files:** +- Create: `tmp/svar2_mvp/prof_python.py` +- Create (output): `tmp/svar2_mvp/prof_out/readbound/python_baseline.md` + +**Interfaces:** +- Consumes: `prof_getitem.make_call` (A1). +- Produces: committed per-(mode×cohort) cProfile top-cumulative tables + pyinstrument trees identifying the hottest **Python** functions on the read path. + +- [ ] **Step 1: Write the Python-layer profiler** + +Create `tmp/svar2_mvp/prof_python.py`: + +```python +"""cProfile + pyinstrument over the live read (Python-layer attribution). + + python tmp/svar2_mvp/prof_python.py + +cProfile ranks Python functions by cumulative time; pyinstrument gives a +low-overhead statistical wall-clock call tree as a cross-check (cProfile's own +per-call overhead can distort tiny hot loops).""" +import cProfile +import io +import pstats +import sys + +from prof_getitem import make_call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + + pr = cProfile.Profile() + pr.enable() + for _ in range(K): + call() + pr.disable() + s = io.StringIO() + pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats(30) + print(f"### cProfile {mode} {cohort} (K={K}), sort=cumulative\n") + print("```\n" + s.getvalue() + "```\n") + + from pyinstrument import Profiler + p = Profiler(interval=0.0005) + p.start() + for _ in range(K): + call() + p.stop() + print(f"### pyinstrument {mode} {cohort} (K={K})\n") + print("```\n" + p.output_text(unicode=False, color=False, show_all=False) + "```\n") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 2: Run all four combos into the baseline report** + +Run: +```bash +cd tmp/svar2_mvp +OUT=prof_out/readbound/python_baseline.md +echo "# SVAR2 read-bound Python-layer baseline ($(date -I))" > "$OUT" +for c in germline somatic; do for m in haplotypes variants; do + echo "== $m $c ==" + ../../.pixi/envs/dev/bin/python prof_python.py $m $c 200 >> "$OUT" 2>&1 +done; done +cd ../.. +``` +Expected: `python_baseline.md` populated with 4 cProfile tables + 4 pyinstrument trees, no tracebacks. (Lower K to ~50 for `somatic` if a combo is slow.) + +- [ ] **Step 3: Record the top Python functions** + +Read `tmp/svar2_mvp/prof_out/readbound/python_baseline.md`. Append a short `## Top Python functions (ranked)` section listing, per mode, the 3–5 highest-cumulative gvl Python functions. Expect for **variants**: `_reconstruct_variants`, `_gather_inputs`, `_ragged_arange_gather` / `_ragged_arange_gather_2level`, `_contig_groups`; for **haplotypes**: `get_haps_and_shifts`, `_gather_inputs`, `_assemble_haps`. Note which are pure-Python overhead vs. thin FFI wrappers. + +- [ ] **Step 4: Commit** + +```bash +git add tmp/svar2_mvp/prof_python.py tmp/svar2_mvp/prof_out/readbound/python_baseline.md +git commit --no-verify -m "perf(svar2): Python-layer baseline profile (cProfile + pyinstrument)" +``` + +--- + +### Task A3: Capture the native/Rust baseline (perf) + instruction-count reference + +**Files:** +- Create: `tmp/svar2_mvp/prof_perf.sh` +- Create (output): `tmp/svar2_mvp/prof_out/readbound/native_baseline.md` + +**Interfaces:** +- Consumes: `prof_getitem.py` (A1). +- Produces: committed per-(mode×cohort) DSO split, Rust symbol self-time, fp call-graph, and a `perf stat -e instructions,cycles` reference used as the Phase-B gate baseline. + +- [ ] **Step 1: Rebuild the `.so` with frame pointers** + +Run: `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` +Expected: build succeeds; this is the binary all A3/Phase-B perf captures use. + +- [ ] **Step 2: Write the perf capture script** + +Create `tmp/svar2_mvp/prof_perf.sh`: + +```bash +#!/usr/bin/env bash +# Native-layer attribution for the live read-bound path via perf (py-spy is +# unusable: ptrace_scope=2; Python 3.10 has no perf trampoline so Python frames +# are opaque -> DSO-level + Rust-symbol self-time is the split). +set -eu +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/readbound +mkdir -p "$OUT" +PERF=/carter/users/dlaub/.pixi/bin/perf +PY=.pixi/envs/dev/bin/python +FREQ=299 +REPORT="$OUT/native_baseline.md" +echo "# SVAR2 read-bound native baseline ($(date -I))" > "$REPORT" + +probe_K () { # mode cohort -> K sized to ~40s + local per + per=$("$PY" tmp/svar2_mvp/prof_getitem.py "$1" "$2" 5 | sed 's/per_call_s=//') + "$PY" -c "import math;print(max(20,math.ceil(40/max(float('$per'),1e-4))))" +} + +for c in germline somatic; do for m in haplotypes variants; do + tag="${m}_${c}"; K=$(probe_K "$m" "$c") + echo "## $tag (K=$K)" | tee -a "$REPORT" + # instruction-count reference (the Phase-B gate baseline) + echo '### perf stat' >> "$REPORT" + { "$PERF" stat -e instructions,cycles -- "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" ; } \ + 2>> "$REPORT" 1>/dev/null || echo "(perf stat HW counters unavailable)" >> "$REPORT" + # sampling profile + "$PERF" record -g --call-graph fp -F $FREQ -o "$OUT/$tag.data" -- \ + "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" >/dev/null 2>&1 + echo '### DSO split' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=dso --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -12 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### top Rust/native self-time symbols' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=symbol --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -20 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### call graph (top)' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=overhead,symbol -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -45 >> "$REPORT"; echo '```' >> "$REPORT" +done; done +echo "NATIVE_BASELINE_DONE -> $REPORT" +``` + +- [ ] **Step 3: Run it** + +Run: `bash tmp/svar2_mvp/prof_perf.sh` +Expected: prints `NATIVE_BASELINE_DONE`; `native_baseline.md` has DSO split + symbol + call-graph blocks for all 4 combos. Confirm the split shows **no** `SearchTree::build` / `dense_union` / `overlap_batch` samples (that would mean the union path is being hit — a driver bug). Expect `gather_haps_readbound`, `merge_keys`, `split_to_flat`, `decode_variants_from_split` (variants), `reconstruct_haplotypes_from_svar2` (haplotypes), and numpy cache-slice symbols (`PyArray_Repeat`, `mapiter_get`) instead. + +- [ ] **Step 4: Record the ranked native targets** + +Append a `## Ranked native targets` section to `native_baseline.md`: per mode, the top 3 native self-time symbols and their DSO (gvl vs genoray_core vs numpy). For **haplotypes**, note whether `gather_haps_readbound` + `split_to_flat` self-time is roughly **2× the diffs-only need** — that confirms the redundant double gather (Task B1). For **variants**, note `split_to_flat` / `decode_variants_from_split` allocation churn (`_int_malloc`/`SpecFromIter` under them → Task B2). + +- [ ] **Step 5: Commit** + +```bash +git add tmp/svar2_mvp/prof_perf.sh tmp/svar2_mvp/prof_out/readbound/native_baseline.md +git commit --no-verify -m "perf(svar2): native-layer baseline profile (perf DSO/symbol/callgraph + instr reference)" +``` + +--- + +## Phase B — Optimizations (each confirmed against Phase A, then parity + instruction-count gated) + +> **Before each Task below:** re-read the Phase A baseline and confirm the target actually ranks in the top few. If a candidate does NOT rank (e.g. B1 removed it, or the profile disagrees), skip that Task and instead apply the **same fix→gate template** to whichever function tops the profile, using `cargo asm`/`perf annotate` for native fns. Do not implement an optimization the profile does not justify. + +### Task B1: Eliminate the redundant pre-reconstruct gather (haplotypes) + +**Rationale (confirm against A3):** `Svar2Haps.get_haps_and_shifts` calls `hap_diffs_from_svar2_readbound` (full `gather_haps_readbound` + `split_to_flat` + `hap_diffs_svar2`) **and then** `reconstruct_haplotypes_from_svar2_readbound` (which repeats gather + split + diffs internally for output sizing). For a haplotypes read the pre-reconstruct diffs are needed **only** when jitter randomizes shifts; the warm `ds[:, :]` read is deterministic/ragged (`shifts = 0`), so the entire first gather is redundant there. `hap_lengths` (which uses `diffs`) is discarded on the haplotypes path (`__call__` keeps only `haps`); the tracks path (out of scope, but must stay green) reuses `get_haps_and_shifts` and *does* need `diffs`/`hap_lengths`, so the guard must preserve them there via an explicit flag. + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`get_haps_and_shifts`, ~lines 319-424; and its tracks caller) +- Test: `tests/dataset/test_svar2_readbound_haps.py`, `tests/dataset/test_svar2_dataset.py` (existing parity oracles — must stay green, including tracks) + +**Interfaces:** +- Consumes: `hap_diffs_from_svar2_readbound`, `reconstruct_haplotypes_from_svar2_readbound` (unchanged FFI). +- Produces: `get_haps_and_shifts(..., need_hap_lengths: bool = False)` computes the diffs loop only when its result is consumed (randomized shifts, OR a caller that needs `hap_lengths`/`diffs` — i.e. tracks passes `need_hap_lengths=True`). Same return tuple shape. + +- [ ] **Step 1: Add a failing micro-test asserting the diffs kernel is not called for a deterministic haps read** + +Add to `tests/dataset/test_svar2_readbound_haps.py`: + +```python +def test_deterministic_haps_read_skips_pre_reconstruct_diffs(monkeypatch): + """A deterministic (shifts=0) haplotypes read must NOT call the separate + hap_diffs readbound kernel — reconstruct sizes itself internally. Guards the + double-gather regression.""" + import genvarloader._dataset._svar2_haps as m + + calls = {"diffs": 0} + real = m.hap_diffs_from_svar2_readbound + def counting(*a, **k): + calls["diffs"] += 1 + return real(*a, **k) + monkeypatch.setattr(m, "hap_diffs_from_svar2_readbound", counting) + + # Build the same small live svar2 dataset the module parity tests use, then: + # ds.with_seqs("haplotypes")[:, :] + # (reuse this file's existing fixture that yields a ds2 Svar2Haps-backed view; + # if none is exposed, lift the _open_pair helper from test_svar2_dataset.py.) + ds2 = _svar2_haps_dataset() # existing/lifted fixture -> haplotypes view + ds2[:, :] + assert calls["diffs"] == 0 +``` + +If no fixture yields a live Svar2Haps haplotypes view in this file, lift `_open_pair` from `tests/dataset/test_svar2_dataset.py` into a local helper `_svar2_haps_dataset()` returning `ds2.with_seqs("haplotypes")`. + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_deterministic_haps_read_skips_pre_reconstruct_diffs -v` +Expected: FAIL with `assert 1 == 0` (the diffs kernel is currently called unconditionally). + +- [ ] **Step 3: Guard the diffs computation in `get_haps_and_shifts`** + +In `python/genvarloader/_dataset/_svar2_haps.py`, add `need_hap_lengths: bool = False` to `get_haps_and_shifts`'s signature, and replace the unconditional diffs block + shifts block (currently ~lines 352-384) with: + +```python + groups = self._contig_groups(contig_ids) + + # diffs are needed pre-reconstruct ONLY to (a) bound randomized jitter + # shifts, or (b) return hap_lengths/diffs to a caller that uses them + # (the tracks path). A deterministic/ragged haplotypes read needs + # neither: reconstruct sizes itself internally. Avoid the redundant + # gather+split+diffs in that (common warm-read) case. + randomized = not (deterministic or isinstance(output_length, str)) + need_diffs = randomized or need_hap_lengths + + if need_diffs: + diffs = np.empty((b, P), np.int32) + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + d = hap_diffs_from_svar2_readbound( + self.store, self.ds_contigs[ci], + gi[0], gi[1], gi[2], gi[3], gi[4], gi[5], gi[6], P, + ) + diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) + hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + else: + diffs = np.zeros((b, P), np.int32) # placeholder (unused downstream) + hap_lengths = np.broadcast_to( + lengths[:, None].astype(np.int32), (b, P) + ).copy() + + if randomized: + max_shift = diffs.clip(min=0) + max_shift = max_shift + (lengths - output_length).clip(min=0)[:, None] + shifts = rng.integers(0, max_shift + 1, dtype=np.int32) + else: + shifts = np.zeros((b, P), np.int32) +``` + +Then have the tracks caller pass `need_hap_lengths=True`. Find the caller: `grep -n "get_haps_and_shifts" python/genvarloader/_dataset/*.py` — it is invoked from `HapsTracks` dispatch (the tracks path) and from `Svar2Haps.__call__` (haplotypes). Update the tracks call site to `get_haps_and_shifts(..., need_hap_lengths=True)`; leave the haplotypes call site at the `False` default. + +> NOTE: when `randomized` is False but `output_length` is a fixed `int`, `hap_lengths` in the placeholder branch is `lengths` broadcast — but `reconstruct_haplotypes_from_svar2_readbound` with a fixed `output_length >= 0` recomputes per-hap lengths internally and ignores `hap_lengths`, so this is safe. Confirm no downstream consumer reads `hap_lengths` for a fixed-int deterministic haps read; if one does, set `need_hap_lengths=True` for that path too. + +- [ ] **Step 4: Python-only change — run the new test + full svar2 parity (incl. tracks)** + +Run: +```bash +pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_deterministic_haps_read_skips_pre_reconstruct_diffs -v +pixi run -e dev pytest tests/dataset -k svar2 -q +``` +Expected: the new test PASSES (`calls["diffs"] == 0`); all svar2 parity tests stay green (haplotypes/variants **and tracks** byte-identical to oracle — the tracks path still calls the diffs kernel via `need_hap_lengths=True`). + +- [ ] **Step 5: Full-tree parity + instruction-count delta** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +/carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py haplotypes germline 500 2>&1 | grep -E 'instructions|cycles' +``` +Expected: full tree green; instructions/read materially lower than the A3 baseline for `haplotypes` (record the % drop). If HW counters are unavailable, record cProfile primitive-call-count drop instead. + +- [ ] **Step 6: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_readbound_haps.py +git commit -m "perf(svar2): skip redundant pre-reconstruct gather for deterministic haplotype reads" +``` + +--- + +### Task B2: Pre-size `split_to_flat` + `decode_variants_from_split` allocations (gvl Rust) + +**Rationale (confirm against A3):** both functions build output `Vec`s with `Vec::new()` (no capacity) and grow them in hot loops — `split_to_flat` (`src/svar2/mod.rs:159`) for `dense_pos`/`dense_key`/`dense_present` (used by BOTH modes — variants decode and haplotype diffs/reconstruct call it), and `decode_variants_from_split` (`src/svar2/mod.rs:262`) for `pos`/`ilen`/`alt_bytes`/`str_off` (variants). `_int_malloc`/`SpecFromIter` were hot even in the old profile. Pre-sizing is byte-identical. **Only implement the parts that rank in the A3 native top symbols after B1.** + +**Files:** +- Modify: `src/svar2/mod.rs` (`split_to_flat` 159-237; `decode_variants_from_split` 262-320) +- Test: `cargo test` in gvl (`test_split_to_flat_*`, `test_decode_variants_from_split_*` in `src/svar2/mod.rs`) + the pytest parity suite + +**Interfaces:** +- Consumes/Produces: `split_to_flat(&BatchResultSplit) -> FlatChannels` and `decode_variants_from_split(&BatchResultSplit, &[u8], &[i64]) -> VariantsSoa` — signatures and output bytes unchanged; only allocation strategy changes. + +- [ ] **Step 1: Confirm the existing unit tests cover both** + +Run: `grep -n "fn test_split_to_flat\|fn test_decode_variants_from_split" src/svar2/mod.rs` +Expected: `test_split_to_flat_marshals_readbound_split`, `test_split_to_flat_trailing_zero_byte_is_allocated`, `test_decode_variants_from_split_merges_and_decodes` exist — the byte-identity guard. + +- [ ] **Step 2: Pre-size `split_to_flat`** + +In `split_to_flat`, compute the total dense entry count once and reserve; pre-size the presence bitstream and drop the on-set `resize` growth. Replace the `dense_pos`/`dense_key` init: + +```rust + let dense_total: usize = (0..n_q) + .map(|q| { + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_pos: Vec = Vec::with_capacity(dense_total); + let mut dense_key: Vec = Vec::with_capacity(dense_total); +``` + +and the presence bitstream: + +```rust + let total_bits: usize = (0..h_count) + .map(|h| { + let q = h / ploidy; + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_present: Vec = vec![0u8; total_bits.div_ceil(8)]; +``` + +Inside the hap loop, drop the `if dense_present.len() <= byte { resize }` guards and set bits directly (`dense_present[bit_acc / 8] |= 1 << (bit_acc % 8);`). Keep the final `dense_present.resize(bit_acc.div_ceil(8), 0);` as a no-op safety. + +- [ ] **Step 3: Pre-size `decode_variants_from_split`** + +Replace the `Vec::new()` inits (`src/svar2/mod.rs:272-275`) with capacity-reserved vectors (an over-estimate upper bound is fine for `with_capacity`): + +```rust + let flat = split_to_flat(br); + let ploidy = br.ploidy; + let n_q = br.n_regions; + let h_count = n_q * ploidy; + + // Upper bound on total merged variants across all haps: every vk entry plus + // every dense window entry (present or not). Over-reserving is harmless. + let vk_total = flat.vk_off[h_count] as usize; + let dense_bits = flat.dense_present_off[h_count] as usize; + let cap = vk_total + dense_bits; + let mut pos: Vec = Vec::with_capacity(cap); + let mut ilen: Vec = Vec::with_capacity(cap); + let mut alt_bytes: Vec = Vec::with_capacity(cap); + let mut str_off: Vec = Vec::with_capacity(cap + 1); + str_off.push(0); + let mut var_off: Vec = Vec::with_capacity(h_count + 1); + var_off.push(0); +``` + +(Leave the per-hap loop body unchanged.) + +- [ ] **Step 4: Rebuild + Rust unit tests** + +Run: +```bash +RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release +cargo test split_to_flat decode_variants_from_split 2>&1 | tail -20 # (set LD_LIBRARY_PATH to .pixi/envs/dev/lib if libpython load fails) +``` +Expected: the named unit tests PASS (byte-identical output). + +- [ ] **Step 5: Full-tree parity + instruction delta (both modes)** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +for m in variants haplotypes; do + echo "== $m ==" + /carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py $m somatic 300 2>&1 | grep -E 'instructions|cycles' +done +``` +Expected: full tree green; instructions/read down vs A3 baseline for both modes (record %). + +- [ ] **Step 6: Commit** + +```bash +git add src/svar2/mod.rs +git commit -m "perf(svar2): pre-size split_to_flat + decode_variants_from_split allocations (byte-identical)" +``` + +--- + +### Task B3: De-duplicate the twin ragged gather in `_reconstruct_variants` (gvl Python) + +**Rationale (confirm against A2):** in `_reconstruct_variants` (`python/genvarloader/_dataset/_svar2_haps.py:585-586`), `pos` and `ilen` are permuted back to global order by two separate `_ragged_arange_gather(pos_c, grouped_var_off, perm)` / `(ilen_c, grouped_var_off, perm)` calls with **identical** `offsets` and `perm` — so the `lens`/`new_off`/`within`/`src` index arrays are computed twice. Compute the source-index permutation once and apply it to both. **Only implement if `_ragged_arange_gather` / `_reconstruct_variants` ranks in the A2 variants top functions.** + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_ragged_arange_gather` region + `_reconstruct_variants`) +- Test: `tests/dataset/test_svar2_readbound_variants.py`, `tests/dataset/test_svar2_dataset.py` (variants parity — must stay green) + +**Interfaces:** +- Produces: a helper `_ragged_arange_src(offsets, perm) -> (src, new_off)` returning the 1-level reorder source index + new offsets; `_ragged_arange_gather` is re-expressed on it; `_reconstruct_variants` computes `src`/`var_off_g` once and does `pos_g = pos_c[src]`, `ilen_g = ilen_c[src]`. + +- [ ] **Step 1: Add the shared-index helper and re-express `_ragged_arange_gather`** + +In `python/genvarloader/_dataset/_svar2_haps.py`, add above `_ragged_arange_gather`: + +```python +def _ragged_arange_src( + offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray[np.int64], NDArray[np.int64]]: + """Source-row index + new offsets for a 1-level ragged reorder by ``perm``. + + ``new_data == data[src]``; ``src`` and ``new_off`` depend only on + ``(offsets, perm)`` — so callers reordering several parallel data arrays by + the same key compute this ONCE and index each array. + """ + offsets = np.asarray(offsets, np.int64) + lens = np.diff(offsets) + new_lens = lens[perm] + new_off = lengths_to_offsets(new_lens, np.int64) + n = int(new_off[-1]) + if n == 0: + return np.zeros(0, np.int64), new_off + within = np.arange(n, dtype=np.int64) - np.repeat(new_off[:-1], new_lens) + src = np.repeat(offsets[perm], new_lens) + within + return src, new_off +``` + +Then rewrite `_ragged_arange_gather` to use it: + +```python +def _ragged_arange_gather( + data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray, NDArray[np.int64]]: + """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``.""" + src, new_off = _ragged_arange_src(offsets, perm) + if src.size == 0: + return data[:0].copy(), new_off + return data[src], new_off +``` + +- [ ] **Step 2: Use the shared index for pos + ilen in `_reconstruct_variants`** + +Replace (`python/genvarloader/_dataset/_svar2_haps.py:585-586`): + +```python + pos_g, var_off_g = _ragged_arange_gather(pos_c, grouped_var_off, perm) + ilen_g, _ = _ragged_arange_gather(ilen_c, grouped_var_off, perm) +``` + +with: + +```python + src, var_off_g = _ragged_arange_src(grouped_var_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] +``` + +(The 2-level `alt` gather is left as-is; it is a different offset structure.) + +- [ ] **Step 3: Run variants parity (Python-only change, no rebuild)** + +Run: +```bash +pixi run -e dev pytest tests/dataset -k svar2 -q +``` +Expected: all svar2 variants parity tests green (byte-identical `pos`/`ilen`/`alt` reorder). + +- [ ] **Step 4: Full-tree parity + instruction delta** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +/carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py variants germline 500 2>&1 | grep -E 'instructions|cycles' +``` +Expected: full tree green; variants instructions/read down vs baseline (record %; if small, keep anyway — it removes a duplicated O(total_variants) pass and is a clarity win). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "perf(svar2): compute the pos/ilen ragged reorder index once in variants decode" +``` + +--- + +### Task B4: `cargo asm` pass on the top post-B1/B2/B3 native symbol (gvl or genoray) + +**Rationale:** after B1-B3, re-profile and take the #1 remaining native self-time symbol. Likely `genoray_core::spine::merge_keys` or `genoray_core::query::gather_haps_readbound` (genoray `svar-2`), gvl's `svar2::merge_hap`/`decode_alt` (variants) or `reconstruct_haplotypes_from_svar2` inner loop (haplotypes). This task inspects that function's assembly, finds a bounds-check / vectorization / allocation defect, and fixes it — the specific instruction change is *discovered from the asm*, not pre-guessed; the deliverable is a measured instruction-count reduction or a documented "no safe win." + +**Files:** +- Modify: the file owning the ranked function — gvl `src/reconstruct.rs` / `src/svar2/mod.rs`, **or** genoray `/carter/users/dlaub/projects/genoray/src/{spine.rs,query.rs}` (branch `svar-2`) +- Test: `cargo test` (owning repo) + gvl pytest parity suite + +**Interfaces:** +- Consumes: A3/post-B3 native profile ranking. +- Produces: same function signature and byte-identical output, fewer instructions on the hot loop. + +- [ ] **Step 1: Re-profile to pick the target** + +Run: `cp tmp/svar2_mvp/prof_out/readbound/native_baseline.md tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` then `bash tmp/svar2_mvp/prof_perf.sh` and read the top symbol per mode. Choose the highest gvl/genoray self-time symbol (ignore libc `_int_malloc`/`memmove` and numpy symbols — those are addressed structurally, not by asm). + +- [ ] **Step 2: Dump its assembly** + +For a gvl symbol: +```bash +cd "$(git rev-parse --show-toplevel)" +cargo asm --release --lib "genvarloader::svar2::merge_hap" 2>/dev/null | head -200 +# (adjust the fully-qualified path to the chosen symbol; `cargo asm --release --lib` +# then a fragment lists candidates if the exact path is ambiguous.) +``` +For a genoray symbol: +```bash +cd /carter/users/dlaub/projects/genoray +cargo asm --release --lib "genoray::spine::merge_keys" 2>/dev/null | head -200 +``` +Optionally cross-reference `perf annotate -i tmp/svar2_mvp/prof_out/readbound/.data --stdio` to see which source lines carry the samples. + +- [ ] **Step 3: Identify the defect** + +Look for: per-iteration `panic`/`bounds_check` calls in the hot loop (→ replace indexed access with iterators or `get_unchecked` where an invariant guarantees the bound), scalar (non-vectorized) copy/merge loops that could use `extend_from_slice`/`copy_from_slice`, or in-loop allocation. Write the specific instruction pattern in a comment on the fix. + +- [ ] **Step 4: Apply the minimal fix** (guided by Step 3 — e.g. hoist a slice bound so the compiler elides per-element checks, or switch a `for i in 0..n { out.push(a[i]) }` to `out.extend_from_slice(&a[..n])`). Keep behavior byte-identical. + +- [ ] **Step 5: Rebuild + parity** + +Run (genoray change picked up by gvl's rebuild via the crate path-dep): +```bash +# if genoray was edited, its unit tests first: +cd /carter/users/dlaub/projects/genoray && cargo test 2>&1 | tail -15; cd - +RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release +pixi run -e dev pytest tests -q +``` +Expected: owning-repo `cargo test` green; gvl full tree green (byte-identical parity). + +- [ ] **Step 6: Instruction-count delta + record** + +Run the same-session `perf stat` on the mode that exercises the changed symbol; append the before/after instruction counts and the asm observation to `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md`. If no safe win was found, record that conclusion instead and stop. + +- [ ] **Step 7: Commit (correct repo/branch)** + +```bash +# gvl change: +git add src/... && git commit -m "perf(svar2): asm fix — (byte-identical, -N% instr)" +# OR genoray change (in the genoray checkout, on svar-2): +cd /carter/users/dlaub/projects/genoray +git add src/... && git commit -m "perf(query): asm fix — (byte-identical, -N% instr)" +``` + +--- + +### Task B5: Consolidate results & update the profiling report + +**Files:** +- Create: `tmp/svar2_mvp/prof_out/readbound/RESULTS.md` +- Modify: genoray `docs/roadmap/svar-2.md` **only if** a kernel signature changed (record it under the relevant milestone note); gvl design spec §9 open-question resolution + +- [ ] **Step 1: Write the results summary** + +Create `tmp/svar2_mvp/prof_out/readbound/RESULTS.md` with a table: per mode (haplotypes, variants) × cohort, baseline vs final instructions/read (and cProfile fallback where HW counters were absent), the optimizations applied (B1/B2/B3/B4), and any candidate that was profiled-but-skipped (with the reason). State the parity result (full tree + svar2 suite green, incl. tracks) explicitly. Note the **deferred** items: tracks (out of scope this round; its `get_haps_and_shifts` double-gather is unaddressed and still runs the diffs kernel via `need_hap_lengths=True`), and variant-windows (guarded `NotImplementedError` — profile once implemented). + +- [ ] **Step 2: Reconcile docs if signatures changed** + +If any FFI/kernel signature changed, update the genoray roadmap note and the gvl design spec's §9 open-question resolution. If nothing signature-level changed, state "read-path internals only; no API/format/doc-surface change" in RESULTS.md. (The `get_haps_and_shifts` `need_hap_lengths` param is an internal signature, not public API — no skill/api.md/docs update needed.) + +- [ ] **Step 3: Commit** + +```bash +git add tmp/svar2_mvp/prof_out/readbound/RESULTS.md +git commit --no-verify -m "perf(svar2): read-bound getitem optimization results summary" +# + any genoray docs commit on svar-2 if applicable +``` + +--- + +## Self-Review + +**Spec coverage:** +- Profile the real read-bound path (not union oracle) → A1–A3 (driver forces `Dataset.open(svar2)`; A3 Step 3 asserts no `SearchTree`/`overlap_batch` samples). ✓ +- Tooling substitution (cProfile+pyinstrument / perf) → A2, A3; pyinstrument dep → A1. ✓ +- In-scope modes haplotypes + variants; tracks dropped from profiling but parity-protected; variant-windows deferred (guarded) → Global Constraints + A1 driver + B5 deferred note. ✓ +- Two-repo landing (gvl #266 / genoray svar-2) → Global Constraints + B4/B5 commit steps. ✓ +- Measure→confirm→fix→re-measure + parity + instr-delta → per-Task gates; Phase B preamble enforces "confirm before implementing." ✓ +- Haplotypes double-gather → B1 (concrete, tracks-safe via `need_hap_lengths`). Variants+shared alloc churn → B2 (concrete). Variants Python twin-gather → B3 (concrete). cargo asm on hot Rust → B4. ✓ +- No format/API change → B5 Step 2 reconciliation. ✓ + +**Placeholder scan:** B4 is investigation-then-fix by nature (asm-discovered), but specifies the exact function-selection rule, exact `cargo asm`/`perf annotate` commands, the defect classes to look for, and a concrete gate — not a hand-wave. B1/B2/B3 carry full code. No "TBD"/"similar to Task N". + +**Type consistency:** `make_call(mode, cohort)` defined in A1, reused verbatim in A2. `get_haps_and_shifts` gains `need_hap_lengths: bool = False` (B1) — threaded to the tracks caller in the same task; haplotypes caller uses the default. `_ragged_arange_src(offsets, perm) -> (src, new_off)` defined and consumed within B3; `_ragged_arange_gather` re-expressed on it (same return contract). `split_to_flat`/`decode_variants_from_split` signatures unchanged (B2). Store sample count via `SparseVar2.n_samples` (verified attribute). From b943607a3a9e721d1cec53396c9b50a1cf8c29fd Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 00:13:52 -0700 Subject: [PATCH 044/108] =?UTF-8?q?docs(plan,spec):=20parallelize=20cargo?= =?UTF-8?q?=20asm=20phase=20=E2=80=94=20subagent=20per=20hot=20fn=20+=20pe?= =?UTF-8?q?r-function=20parity=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cargo asm optimization (B4) fans out one Sonnet subagent per hot native function (worktree-isolated so same-file edits don't clobber), each producing a byte-identical fix + its own test__byte_identical parity test + instruction delta; winners merged sequentially behind the full-tree parity gate. Co-Authored-By: Claude Opus 4.8 --- ...2026-07-05-svar2-readbound-getitem-perf.md | 80 ++++++++++--------- ...-05-svar2-readbound-getitem-perf-design.md | 20 +++-- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md index 2931870c..3dc3fa28 100644 --- a/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md +++ b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md @@ -653,68 +653,70 @@ git commit -m "perf(svar2): compute the pos/ilen ragged reorder index once in va --- -### Task B4: `cargo asm` pass on the top post-B1/B2/B3 native symbol (gvl or genoray) +### Task B4: `cargo asm` pass on ALL hot native functions — parallel subagent fan-out -**Rationale:** after B1-B3, re-profile and take the #1 remaining native self-time symbol. Likely `genoray_core::spine::merge_keys` or `genoray_core::query::gather_haps_readbound` (genoray `svar-2`), gvl's `svar2::merge_hap`/`decode_alt` (variants) or `reconstruct_haplotypes_from_svar2` inner loop (haplotypes). This task inspects that function's assembly, finds a bounds-check / vectorization / allocation defect, and fixes it — the specific instruction change is *discovered from the asm*, not pre-guessed; the deliverable is a measured instruction-count reduction or a documented "no safe win." +**Rationale:** after B1-B3, re-profile and collect the **full set** of native functions that still carry meaningful self-time (gvl and genoray), not just the #1. Because these functions are independent, optimize them **in parallel — one subagent per function**, each inspecting that function's assembly, finding a bounds-check / vectorization / allocation defect, and applying a byte-identical fix guarded by its **own per-function parity test**. The specific instruction change is *discovered from the asm*, not pre-guessed; each subagent's deliverable is a measured instruction-count reduction (or a documented "no safe win"). The branch owner then merges the fixes sequentially behind the full-tree parity gate. + +**Execution model:** REQUIRED SUB-SKILLs — superpowers:dispatching-parallel-agents (fan-out) + superpowers:using-git-worktrees (isolation). Per project rules: subagent implementers run on **Sonnet or weaker**; worktrees live under `.claude/worktrees/` of the owning repo. Worktree isolation is REQUIRED because several candidates share a file (`src/svar2/mod.rs` holds `split_to_flat`/`merge_hap`/`decode_alt`; genoray `src/spine.rs` holds `merge_keys`) — parallel in-place edits to the same file would clobber each other. **Files:** -- Modify: the file owning the ranked function — gvl `src/reconstruct.rs` / `src/svar2/mod.rs`, **or** genoray `/carter/users/dlaub/projects/genoray/src/{spine.rs,query.rs}` (branch `svar-2`) -- Test: `cargo test` (owning repo) + gvl pytest parity suite +- Modify (per subagent, in its own worktree): the file owning that function — gvl `src/reconstruct.rs` / `src/svar2/mod.rs`, **or** genoray `/carter/users/dlaub/projects/genoray/src/{spine.rs,query.rs}` (branch `svar-2`) +- Test (per subagent): a focused `cargo test` in the owning repo asserting byte-identical output for that function **Interfaces:** -- Consumes: A3/post-B3 native profile ranking. -- Produces: same function signature and byte-identical output, fewer instructions on the hot loop. +- Consumes: A3/post-B3 native profile ranking (the hot-function set). +- Produces: per function — same signature, byte-identical output, fewer hot-loop instructions, and a committed per-function parity test. -- [ ] **Step 1: Re-profile to pick the target** +- [ ] **Step 1: Re-profile and enumerate the hot-function set** -Run: `cp tmp/svar2_mvp/prof_out/readbound/native_baseline.md tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` then `bash tmp/svar2_mvp/prof_perf.sh` and read the top symbol per mode. Choose the highest gvl/genoray self-time symbol (ignore libc `_int_malloc`/`memmove` and numpy symbols — those are addressed structurally, not by asm). +Run: `cp tmp/svar2_mvp/prof_out/readbound/native_baseline.md tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` then `bash tmp/svar2_mvp/prof_perf.sh`. From the per-mode symbol tables, list every gvl/genoray self-time symbol above a cutoff (e.g. ≥1.5% self across any mode). **Exclude** libc (`_int_malloc`/`memmove`/`memset`) and numpy symbols — those are structural, not asm-fixable. Write the resulting list (function → owning repo/file → which mode exercises it) into `tmp/svar2_mvp/prof_out/readbound/asm_targets.md`. This list is the fan-out work-list. -- [ ] **Step 2: Dump its assembly** +- [ ] **Step 2: Create one worktree per target function** -For a gvl symbol: +For each function `F` in the work-list, create an isolated worktree off the correct branch. gvl functions: ```bash cd "$(git rev-parse --show-toplevel)" -cargo asm --release --lib "genvarloader::svar2::merge_hap" 2>/dev/null | head -200 -# (adjust the fully-qualified path to the chosen symbol; `cargo asm --release --lib` -# then a fragment lists candidates if the exact path is ambiguous.) +git worktree add ".claude/worktrees/asm-" svar2-m6b-kernel ``` -For a genoray symbol: +genoray functions: ```bash cd /carter/users/dlaub/projects/genoray -cargo asm --release --lib "genoray::spine::merge_keys" 2>/dev/null | head -200 +git worktree add ".claude/worktrees/asm-" svar-2 ``` -Optionally cross-reference `perf annotate -i tmp/svar2_mvp/prof_out/readbound/.data --stdio` to see which source lines carry the samples. +Record the worktree path per function in `asm_targets.md`. + +- [ ] **Step 3: Dispatch one Sonnet subagent per function (in parallel)** + +Dispatch all subagents in a single batch (superpowers:dispatching-parallel-agents). Give each subagent this exact brief, filled in for its function `F`, file, worktree path, and the mode/cohort that exercises it: -- [ ] **Step 3: Identify the defect** +> You are optimizing exactly ONE Rust function, `F`, in worktree `` (do not touch any other file). Model: Sonnet. +> 1. Dump its assembly: `cargo asm --release --lib "" 2>/dev/null | head -200` (if the path is ambiguous, run `cargo asm --release --lib` and grep the candidate list). Cross-reference `perf annotate -i tmp/svar2_mvp/prof_out/readbound/.data --stdio` for the hot source lines. +> 2. Identify a byte-identical defect: per-iteration `panic`/bounds-check in the hot loop (→ iterators or `get_unchecked` behind a proven invariant), scalar copy/merge loops (→ `extend_from_slice`/`copy_from_slice`), or in-loop allocation (→ `with_capacity`). Comment the specific instruction pattern on the fix. +> 3. Apply the minimal fix. Behavior MUST stay byte-identical. +> 4. Write a **per-function parity test** in the owning repo: a `#[test]` that runs `F` on representative inputs and asserts the exact output (compare against a hand-checked expected value, or against a slow reference implementation of `F` inline in the test). Name it `test__byte_identical`. +> 5. `cargo test test__byte_identical` (set `LD_LIBRARY_PATH` to `.pixi/envs/dev/lib` if libpython fails to load in gvl). It MUST pass. +> 6. Measure: `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` (from the gvl worktree; for genoray targets, build gvl against the genoray worktree path-dep), then same-session `perf stat -e instructions,cycles -- .pixi/envs/dev/bin/python tmp/svar2_mvp/prof_getitem.py ` before vs after. Report the instruction delta. +> 7. Commit in your worktree: `perf(...): asm fix — (byte-identical, -N% instr)`. If you found NO safe byte-identical win, revert your edits, commit nothing, and report "no safe win for F" with the asm reason. -Look for: per-iteration `panic`/`bounds_check` calls in the hot loop (→ replace indexed access with iterators or `get_unchecked` where an invariant guarantees the bound), scalar (non-vectorized) copy/merge loops that could use `extend_from_slice`/`copy_from_slice`, or in-loop allocation. Write the specific instruction pattern in a comment on the fix. +- [ ] **Step 4: Collect results** -- [ ] **Step 4: Apply the minimal fix** (guided by Step 3 — e.g. hoist a slice bound so the compiler elides per-element checks, or switch a `for i in 0..n { out.push(a[i]) }` to `out.extend_from_slice(&a[..n])`). Keep behavior byte-identical. +Gather each subagent's report (fix applied? instruction delta? or no-safe-win) into `asm_targets.md`. Discard worktrees for no-win functions: `git worktree remove .claude/worktrees/asm-`. -- [ ] **Step 5: Rebuild + parity** +- [ ] **Step 5: Merge the winning fixes sequentially behind the parity gate** -Run (genoray change picked up by gvl's rebuild via the crate path-dep): +For each winning function, in turn (gvl fixes onto `svar2-m6b-kernel`, genoray fixes onto `svar-2`): ```bash -# if genoray was edited, its unit tests first: -cd /carter/users/dlaub/projects/genoray && cargo test 2>&1 | tail -15; cd - +# gvl example (from the main worktree): +cd "$(git rev-parse --show-toplevel)" +git cherry-pick # or merge the worktree branch RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release -pixi run -e dev pytest tests -q +pixi run -e dev pytest tests -q # full-tree parity after EACH merge ``` -Expected: owning-repo `cargo test` green; gvl full tree green (byte-identical parity). +If a merge conflicts (two fixes in the same file) or the full tree goes red, resolve the conflict / drop the offending fix, re-run, and record the decision. genoray fixes: cherry-pick onto `svar-2` in the genoray checkout, then rebuild gvl (crate path-dep) and re-run the gvl full tree. After all merges, remove the worktrees. -- [ ] **Step 6: Instruction-count delta + record** +- [ ] **Step 6: Record** -Run the same-session `perf stat` on the mode that exercises the changed symbol; append the before/after instruction counts and the asm observation to `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md`. If no safe win was found, record that conclusion instead and stop. - -- [ ] **Step 7: Commit (correct repo/branch)** - -```bash -# gvl change: -git add src/... && git commit -m "perf(svar2): asm fix — (byte-identical, -N% instr)" -# OR genoray change (in the genoray checkout, on svar-2): -cd /carter/users/dlaub/projects/genoray -git add src/... && git commit -m "perf(query): asm fix — (byte-identical, -N% instr)" -``` +Append the final per-function outcomes (merged / dropped / no-win) and cumulative instruction deltas per mode to `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md`. --- @@ -750,9 +752,9 @@ git commit --no-verify -m "perf(svar2): read-bound getitem optimization results - In-scope modes haplotypes + variants; tracks dropped from profiling but parity-protected; variant-windows deferred (guarded) → Global Constraints + A1 driver + B5 deferred note. ✓ - Two-repo landing (gvl #266 / genoray svar-2) → Global Constraints + B4/B5 commit steps. ✓ - Measure→confirm→fix→re-measure + parity + instr-delta → per-Task gates; Phase B preamble enforces "confirm before implementing." ✓ -- Haplotypes double-gather → B1 (concrete, tracks-safe via `need_hap_lengths`). Variants+shared alloc churn → B2 (concrete). Variants Python twin-gather → B3 (concrete). cargo asm on hot Rust → B4. ✓ +- Haplotypes double-gather → B1 (concrete, tracks-safe via `need_hap_lengths`). Variants+shared alloc churn → B2 (concrete). Variants Python twin-gather → B3 (concrete). cargo asm on ALL hot Rust fns → B4 (parallel subagent fan-out, per-function parity tests, worktree-isolated). ✓ - No format/API change → B5 Step 2 reconciliation. ✓ -**Placeholder scan:** B4 is investigation-then-fix by nature (asm-discovered), but specifies the exact function-selection rule, exact `cargo asm`/`perf annotate` commands, the defect classes to look for, and a concrete gate — not a hand-wave. B1/B2/B3 carry full code. No "TBD"/"similar to Task N". +**Placeholder scan:** B4 is investigation-then-fix by nature (asm-discovered) and now a parallel fan-out, but specifies the exact hot-set enumeration rule + cutoff, worktree-per-function isolation, the verbatim per-subagent brief (with exact `cargo asm`/`perf annotate`/`perf stat` commands, the defect classes, the per-function `test__byte_identical` requirement), and a sequential merge-behind-parity gate — not a hand-wave. B1/B2/B3 carry full code. No "TBD"/"similar to Task N". **Type consistency:** `make_call(mode, cohort)` defined in A1, reused verbatim in A2. `get_haps_and_shifts` gains `need_hap_lengths: bool = False` (B1) — threaded to the tracks caller in the same task; haplotypes caller uses the default. `_ragged_arange_src(offsets, perm) -> (src, new_off)` defined and consumed within B3; `_ragged_arange_gather` re-expressed on it (same return contract). `split_to_flat`/`decode_variants_from_split` signatures unchanged (B2). Store sample count via `SparseVar2.n_samples` (verified attribute). diff --git a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md index 48cbf4a1..e2b5f29a 100644 --- a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md +++ b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md @@ -113,10 +113,17 @@ concrete candidates to **confirm against the profile before touching**: `_ragged_arange_gather` permutation passes — vectorize / drop copies where the profile justifies. - **Rust hot fns via `cargo asm`:** likely `svar2::split_to_flat` (AoS→SoA copy), - `gather_haps_readbound`, `merge_keys`, `hap_diffs_svar2`, and the - `reconstruct_haplotypes_from_svar2` inner kernel. `cargo asm` targets: - bounds-check elision, autovectorization of copy/merge loops, alloc churn - (`SpecFromIter`/`_int_malloc` were hot even in the old run). + `svar2::merge_hap`/`decode_alt`, `gather_haps_readbound`, `merge_keys`, + `hap_diffs_svar2`, and the `reconstruct_haplotypes_from_svar2` inner kernel. + `cargo asm` targets: bounds-check elision, autovectorization of copy/merge + loops, alloc churn (`SpecFromIter`/`_int_malloc` were hot even in the old run). + Because these functions are independent, the `cargo asm` phase **fans out one + subagent per hot function in parallel** (Sonnet implementers, each in its own + git worktree so same-file edits don't clobber), and **each fix carries its own + per-function parity test** (a focused `cargo test` asserting byte-identical + output on representative inputs) alongside the instruction-count delta. The + branch owner then merges the worktree fixes sequentially behind the full-tree + parity gate, dropping any that regress or fail parity. ## 7. Measurement discipline & parity gate @@ -140,7 +147,10 @@ Per hard-won project lessons (shared-node noise): tables, per mode × cohort) under `tmp/svar2_mvp/prof_out/`. 2. Confirmed optimizations implemented: gvl-side on `svar2-m6b-kernel` (#266), genoray-side on `svar-2` — each parity-gated, each with a recorded - same-session instruction-count delta. + same-session instruction-count delta. The `cargo asm` phase is executed as a + **parallel subagent fan-out** (one worktree-isolated Sonnet subagent per hot + function, each with a per-function parity test), merged sequentially behind + the full-tree parity gate. 3. `pyinstrument` added to `pixi.toml`. 4. No format/API/doc-surface changes (read-path internals only), so the skill/api.md/docs gates do not apply; the genoray roadmap needs no milestone From 676c5253b7574ed200ce2342a361fdc16a839f82 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 00:20:52 -0700 Subject: [PATCH 045/108] perf(svar2): live read-bound Dataset.__getitem__ profiling driver + pyinstrument --- pixi.lock | 311 ++++++++++++++++++++++++---------- pixi.toml | 1 + tmp/svar2_mvp/prof_getitem.py | 64 +++++++ 3 files changed, 291 insertions(+), 85 deletions(-) create mode 100644 tmp/svar2_mvp/prof_getitem.py diff --git a/pixi.lock b/pixi.lock index 7d7bba80..d3c8746b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -88,6 +88,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -176,6 +177,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -213,7 +215,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl @@ -325,6 +326,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -349,6 +351,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl @@ -389,7 +392,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -482,6 +484,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -564,6 +567,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -623,7 +627,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl @@ -743,6 +746,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -767,6 +771,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl @@ -803,7 +808,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -903,6 +907,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -1215,6 +1220,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -1444,6 +1450,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -1775,6 +1782,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -1983,6 +1991,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -1997,6 +2006,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz @@ -2016,7 +2026,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl @@ -2031,7 +2040,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl @@ -2092,6 +2100,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl @@ -2102,6 +2111,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz @@ -2122,7 +2132,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl @@ -2140,7 +2149,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -2293,6 +2301,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.2.2-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.11.0-py310hb9dbd67_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -2439,6 +2448,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl @@ -2473,7 +2483,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl @@ -2642,6 +2651,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.2.2-py310haea493c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pthread-stubs-0.4-hd74edd7_1002.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -2676,6 +2686,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl @@ -2709,7 +2720,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -2809,6 +2819,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -2897,6 +2908,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -2934,7 +2946,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl @@ -3046,6 +3057,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -3070,6 +3082,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl @@ -3110,7 +3123,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -3209,6 +3221,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py311h49ec1c0_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py311haee01d2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.15-hd63d673_0_cpython.conda @@ -3442,6 +3455,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py311hc949640_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py311h50b1a05_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.15-h8561d8f_0_cpython.conda @@ -3607,6 +3621,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -3840,6 +3855,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -4005,6 +4021,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py313h07c4f96_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py313h54dd161_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.13-h6add32d_100_cp313.conda @@ -4239,6 +4256,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py313h0997733_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py313h6fa1262_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.13-h20e6be0_100_cp313.conda @@ -4455,6 +4473,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.2.2-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.11.0-py310hb9dbd67_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -4607,6 +4626,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -4663,7 +4683,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl @@ -4839,6 +4858,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.2.2-py310haea493c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pthread-stubs-0.4-hd74edd7_1002.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -4873,6 +4893,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl @@ -4906,7 +4927,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl @@ -7355,6 +7375,66 @@ packages: purls: [] size: 1759386 timestamp: 1777312019188 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda + sha256: 5315f974ccb4a664aaacdf3cdbcf8619221f6fb211b44045a9bcf2ec6f2eec16 + md5: 3c8b1312ed395879899491168f503d00 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 160187 + timestamp: 1767565456971 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py311h49ec1c0_0.conda + sha256: 37293d78effcfd7958d7e467bf961e0abb017ddd49eb75fa3cbdefe489d04b2f + md5: 9ec77f7ef80f25bc19d8eeb8ba12f142 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.11,<3.12.0a0 + - python_abi 3.11.* *_cp311 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 191363 + timestamp: 1767565475544 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda + sha256: d355a34b127d5ffbf716e19c8beaf4031251ccb029285618437e344c12a2d0b0 + md5: 0b365b8629970ebf8486485fefa5f8d5 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 187624 + timestamp: 1767565456324 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py313h07c4f96_0.conda + sha256: ba206c26c6555a08b8fce977a31f848c78fa30531688d3dd5789e99613ff311a + md5: 572bb387712a5ed27f860aec04d3f5a0 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 190051 + timestamp: 1767565452479 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda sha256: 225d491c94a6962b92377154b33b424cb0cc30aa2fe0b433c6c110d05da7283d md5: 132a1bc9d31e17d6e12d7188ca7ddfb8 @@ -10991,6 +11071,66 @@ packages: purls: [] size: 1176168 timestamp: 1777313031417 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda + sha256: 5155281bd659b1c87a2621739e2cb4cad5643239b1238cd51ba344e029cd7377 + md5: 4fd4fd1a923ab4d04a31beffe986f0e3 + depends: + - __osx >=11.0 + - python >=3.10,<3.11.0a0 + - python >=3.10,<3.11.0a0 *_cpython + - python_abi 3.10.* *_cp310 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 159914 + timestamp: 1767565719097 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py311hc949640_0.conda + sha256: 963ced7b3506a1817c0db28cd647395d41d7cd4c9c3da9dfd7f107565c4c91a9 + md5: c6bda0caf8d63adbe088ac8181fb1960 + depends: + - __osx >=11.0 + - python >=3.11,<3.12.0a0 + - python >=3.11,<3.12.0a0 *_cpython + - python_abi 3.11.* *_cp311 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 189747 + timestamp: 1767565960069 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda + sha256: b31076b97840fb2e983413e4034090cfaf7128cd06f32d3e94d0f2b397fac350 + md5: 84fd6824786da8cb8391c2038186b74c + depends: + - __osx >=11.0 + - python >=3.12,<3.13.0a0 + - python >=3.12,<3.13.0a0 *_cpython + - python_abi 3.12.* *_cp312 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 187474 + timestamp: 1767565653461 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py313h0997733_0.conda + sha256: ce2592818d1de99a561df62f505b2b8ed8a9bd2f6bc22f57ac517f3587afbf83 + md5: 36016d88b9275f390d39434d0c041662 + depends: + - __osx >=11.0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 188912 + timestamp: 1767565820081 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda sha256: 34dd964342267b5d93674eefe37055b8915943de60d43b4e36a08b9642319f46 md5: b0b2e5d34dba70dfad3955a59e611b46 @@ -12557,6 +12697,22 @@ packages: - pytest-httpserver ; extra == 'test' - pytest-localftpserver ; extra == 'test' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + name: rich-rst + version: 2.1.0 + sha256: 7ecd1343ee12c879d0e7ae74c3eb6d263b023d2929c6d114212eb1fd91057255 + requires_dist: + - rich>=12.0.0 + - pygments>=2.0.0 + - docutils ; extra == 'docs' + - sphinx ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - ruff ; extra == 'dev' + - mypy ; extra == 'dev' + - pre-commit ; extra == 'dev' - pypi: https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl name: mistune version: 3.2.1 @@ -13554,6 +13710,62 @@ packages: version: 0.3.25 sha256: ba907b069ad498647661cc376ba79438728a321afb61a9eca333d60ca9c6d9a6 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + name: setuptools + version: 83.0.0 + sha256: 29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - virtualenv>=13.0.0 ; extra == 'test' + - wheel>=0.44.0 ; extra == 'test' + - pip>=19.1 ; extra == 'test' + - packaging>=24.2 ; extra == 'test' + - jaraco-envs>=2.2 ; extra == 'test' + - pytest-xdist>=3 ; extra == 'test' + - jaraco-path>=3.7.2 ; extra == 'test' + - build[virtualenv]>=1.0.3 ; extra == 'test' + - filelock>=3.4.0 ; extra == 'test' + - ini2toml[lite]>=0.14 ; extra == 'test' + - tomli-w>=1.0.0 ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' + - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' + - pytest-home>=0.5 ; extra == 'test' + - pytest-subprocess ; extra == 'test' + - pyproject-hooks!=1.1 ; extra == 'test' + - jaraco-test>=5.5 ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pygments-github-lexers==0.0.5 ; extra == 'doc' + - sphinx-favicon ; extra == 'doc' + - sphinx-inline-tabs ; extra == 'doc' + - sphinx-reredirects ; extra == 'doc' + - sphinxcontrib-towncrier ; extra == 'doc' + - sphinx-notfound-page>=1,<2 ; extra == 'doc' + - pyproject-hooks!=1.1 ; extra == 'doc' + - towncrier<24.7 ; extra == 'doc' + - packaging>=24.2 ; extra == 'core' + - more-itertools>=8.8 ; extra == 'core' + - jaraco-text>=3.7 ; extra == 'core' + - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' + - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' + - wheel>=0.43.0 ; extra == 'core' + - jaraco-functools>=4 ; extra == 'core' + - more-itertools ; extra == 'core' + - pytest-checkdocs>=2.14 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - ruff>=0.13.0 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=3.4 ; extra == 'enabler' + - pytest-mypy>=1.0.1 ; platform_python_implementation != 'PyPy' and extra == 'type' + - mypy==1.18.* ; extra == 'type' + - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' + - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl name: sphinxcontrib-applehelp version: 2.0.0 @@ -15072,62 +15284,6 @@ packages: requires_dist: - numpy>=1.21.3 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - name: setuptools - version: 82.0.1 - sha256: a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb - requires_dist: - - pytest>=6,!=8.1.* ; extra == 'test' - - virtualenv>=13.0.0 ; extra == 'test' - - wheel>=0.44.0 ; extra == 'test' - - pip>=19.1 ; extra == 'test' - - packaging>=24.2 ; extra == 'test' - - jaraco-envs>=2.2 ; extra == 'test' - - pytest-xdist>=3 ; extra == 'test' - - jaraco-path>=3.7.2 ; extra == 'test' - - build[virtualenv]>=1.0.3 ; extra == 'test' - - filelock>=3.4.0 ; extra == 'test' - - ini2toml[lite]>=0.14 ; extra == 'test' - - tomli-w>=1.0.0 ; extra == 'test' - - pytest-timeout ; extra == 'test' - - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' - - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' - - pytest-home>=0.5 ; extra == 'test' - - pytest-subprocess ; extra == 'test' - - pyproject-hooks!=1.1 ; extra == 'test' - - jaraco-test>=5.5 ; extra == 'test' - - sphinx>=3.5 ; extra == 'doc' - - jaraco-packaging>=9.3 ; extra == 'doc' - - rst-linker>=1.9 ; extra == 'doc' - - furo ; extra == 'doc' - - sphinx-lint ; extra == 'doc' - - jaraco-tidelift>=1.4 ; extra == 'doc' - - pygments-github-lexers==0.0.5 ; extra == 'doc' - - sphinx-favicon ; extra == 'doc' - - sphinx-inline-tabs ; extra == 'doc' - - sphinx-reredirects ; extra == 'doc' - - sphinxcontrib-towncrier ; extra == 'doc' - - sphinx-notfound-page>=1,<2 ; extra == 'doc' - - pyproject-hooks!=1.1 ; extra == 'doc' - - towncrier<24.7 ; extra == 'doc' - - packaging>=24.2 ; extra == 'core' - - more-itertools>=8.8 ; extra == 'core' - - jaraco-text>=3.7 ; extra == 'core' - - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' - - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' - - wheel>=0.43.0 ; extra == 'core' - - jaraco-functools>=4 ; extra == 'core' - - more-itertools ; extra == 'core' - - pytest-checkdocs>=2.4 ; extra == 'check' - - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' - - ruff>=0.13.0 ; sys_platform != 'cygwin' and extra == 'check' - - pytest-cov ; extra == 'cover' - - pytest-enabler>=2.2 ; extra == 'enabler' - - pytest-mypy ; extra == 'type' - - mypy==1.18.* ; extra == 'type' - - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' - - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl name: nvidia-nvjitlink-cu12 version: 12.6.85 @@ -16303,21 +16459,6 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' -- pypi: https://files.pythonhosted.org/packages/cf/5f/62fdf0f574ec75af7ada8ad27c1a67e3723d491143bc88d5371c25673949/rich_rst-2.0.2-py3-none-any.whl - name: rich-rst - version: 2.0.2 - sha256: 7bc2923929c9bbc61aadfef2069b742446823b306af84cf9e0f56b965a3d3035 - requires_dist: - - rich>=12.0.0 - - pygments>=2.0.0 - - docutils ; extra == 'docs' - - rich ; extra == 'docs' - - rich-rst ; extra == 'docs' - - sphinx ; extra == 'docs' - - sphinx-copybutton ; extra == 'docs' - - sphinx-rtd-theme ; extra == 'docs' - - pytest ; extra == 'tests' - - pytest-cov ; extra == 'tests' - pypi: https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: watchfiles version: 1.1.1 diff --git a/pixi.toml b/pixi.toml index 18df7eeb..68ce2992 100644 --- a/pixi.toml +++ b/pixi.toml @@ -40,6 +40,7 @@ icecream = "*" hypothesis = "*" filelock = "*" memray = "*" +pyinstrument = "*" patchelf = "*" pyrefly = "*" diff --git a/tmp/svar2_mvp/prof_getitem.py b/tmp/svar2_mvp/prof_getitem.py new file mode 100644 index 00000000..22b5c133 --- /dev/null +++ b/tmp/svar2_mvp/prof_getitem.py @@ -0,0 +1,64 @@ +"""Profile the LIVE SVAR2 read-bound Dataset.__getitem__ path (not the union +oracle) for the in-scope modes. One (mode, cohort) per process so cProfile/perf +attribute cleanly. + + python tmp/svar2_mvp/prof_getitem.py + +gvl.write + Dataset.open run ONCE (we profile the READ, not the write). Prints +per_call_s over K warm calls. Tracks mode is out of scope; variant-windows is +guarded NotImplementedError in Svar2Haps and cannot be profiled yet.""" +import sys +import time +from pathlib import Path + +import polars as pl + +STORE_DIR = Path("/carter/users/dlaub/projects/svar2_mvp") +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] +WORK = Path("tmp/svar2_mvp/prof_out/readbound") + + +def _bed(): + return pl.DataFrame({ + "chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS], + }) + + +def make_call(mode, cohort): + import genvarloader as gvl + from genoray import SparseVar2 + + prefix = STORE_DIR / cohort + sv2 = SparseVar2(f"{prefix}.svar2") + n_s = sv2.n_samples + ds_path = WORK / f"{cohort}_{mode}.gvl" + WORK.mkdir(parents=True, exist_ok=True) + + gvl.write(ds_path, _bed(), variants=SparseVar2(f"{prefix}.svar2"), + samples=None, max_jitter=0, overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF) + view = ds.with_seqs(mode) # "haplotypes" or "variants" + + R = len(REGIONS) + + def call(): + view[:R, :n_s] + + return call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + print(f"per_call_s={(time.perf_counter() - t0) / K:.5f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) From 9ccc46b0727fa09bfd775ffb3d7d3eda2c3db853 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 00:36:14 -0700 Subject: [PATCH 046/108] perf(svar2): Python-layer baseline profile (cProfile + pyinstrument) --- .../prof_out/readbound/python_baseline.md | 445 ++++++++++++++++++ tmp/svar2_mvp/prof_python.py | 41 ++ 2 files changed, 486 insertions(+) create mode 100644 tmp/svar2_mvp/prof_out/readbound/python_baseline.md create mode 100644 tmp/svar2_mvp/prof_python.py diff --git a/tmp/svar2_mvp/prof_out/readbound/python_baseline.md b/tmp/svar2_mvp/prof_out/readbound/python_baseline.md new file mode 100644 index 00000000..386dba42 --- /dev/null +++ b/tmp/svar2_mvp/prof_out/readbound/python_baseline.md @@ -0,0 +1,445 @@ +# SVAR2 read-bound Python-layer baseline (2026-07-06) +2026-07-06 00:30:19.871 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl +2026-07-06 00:30:19.889 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. +2026-07-06 00:30:19.889 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 + + ncalls tottime percall cumtime percall filename:lineno(function) + 200 0.738 0.004 43.452 0.217 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) + 200 0.001 0.000 42.714 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) + 200 0.004 0.000 42.713 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) + 200 0.002 0.000 42.706 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) + 200 0.044 0.000 42.681 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) + 200 0.006 0.000 42.607 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) + 200 0.146 0.001 42.599 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:319(get_haps_and_shifts) + 200 4.692 0.023 38.910 0.195 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:714(_assemble_haps) + 200 8.965 0.045 34.167 0.171 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) + 1200 0.003 0.000 16.360 0.014 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) + 600 0.002 0.000 16.352 0.027 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) + 600 16.346 0.027 16.346 0.027 {method 'repeat' of 'numpy.ndarray' objects} + 600 8.851 0.015 8.851 0.015 {built-in method numpy.arange} + 200 1.972 0.010 1.972 0.010 {built-in method genvarloader.genvarloader.reconstruct_haplotypes_from_svar2_readbound} + 200 1.271 0.006 1.271 0.006 {built-in method genvarloader.genvarloader.hap_diffs_from_svar2_readbound} + 400 0.008 0.000 0.253 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) + 1600 0.209 0.000 0.214 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) + 200 0.034 0.000 0.037 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) + 3200 0.030 0.000 0.030 0.000 {built-in method numpy.ascontiguousarray} + 200 0.013 0.000 0.023 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) + 200 0.004 0.000 0.022 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) + 400 0.000 0.000 0.017 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() + 200 0.001 0.000 0.016 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) + 200 0.001 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) + 200 0.004 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) + 200 0.001 0.000 0.014 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) + 200 0.004 0.000 0.013 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) + 400 0.002 0.000 0.012 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) + 600 0.001 0.000 0.011 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2979(prod) + 400 0.001 0.000 0.009 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) + + +``` + +### pyinstrument haplotypes germline (K=200) + +``` + + _ ._ __/__ _ _ _ _ _/_ Recorded: 00:31:04 Samples: 2811 + /_//_/// /_\ / //_// / //_'/ // Duration: 43.543 CPU time: 44.033 +/ _/ v5.1.2 + +Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 + +43.5427 main prof_python.py:16 +`- 43.5427 call prof_getitem.py:48 + |- 42.7615 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 + | `- 42.7615 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 + | `- 42.7615 getitem genvarloader/_dataset/_query.py:66 + | `- 42.7615 _getitem_unspliced genvarloader/_dataset/_query.py:154 + | `- 42.7594 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 + | `- 42.7594 Svar2Haps.get_haps_and_shifts genvarloader/_dataset/_svar2_haps.py:319 + | |- 39.0037 Svar2Haps._assemble_haps genvarloader/_dataset/_svar2_haps.py:714 + | | |- 34.2766 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 + | | | |- 16.3452 repeat numpy/core/fromnumeric.py:423 + | | | | `- 16.3452 _wrapfunc numpy/core/fromnumeric.py:53 + | | | | `- 16.3452 ndarray.repeat + | | | |- 8.9989 [self] genvarloader/_dataset/_svar2_haps.py + | | | `- 8.9325 arange + | | `- 4.7255 [self] genvarloader/_dataset/_svar2_haps.py + | |- 1.9794 reconstruct_haplotypes_from_svar2_readbound + | `- 1.3297 hap_diffs_from_svar2_readbound + `- 0.7812 [self] prof_getitem.py + +``` + +2026-07-06 00:31:54.920 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_variants.gvl +2026-07-06 00:31:54.941 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. +2026-07-06 00:31:54.941 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 + + ncalls tottime percall cumtime percall filename:lineno(function) + 200 0.000 0.000 1.064 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) + 200 0.000 0.000 1.064 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) + 200 0.001 0.000 1.063 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) + 200 0.001 0.000 1.061 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) + 200 0.037 0.000 1.031 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) + 200 0.001 0.000 0.975 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) + 200 0.041 0.000 0.973 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:527(_reconstruct_variants) + 200 0.542 0.003 0.542 0.003 {built-in method genvarloader.genvarloader.decode_variants_from_svar2_readbound} + 3000 0.002 0.000 0.134 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) + 1600 0.001 0.000 0.116 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) + 1600 0.114 0.000 0.114 0.000 {method 'repeat' of 'numpy.ndarray' objects} + 400 0.030 0.000 0.105 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) + 200 0.003 0.000 0.104 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) + 200 0.029 0.000 0.099 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:101(_ragged_arange_gather_2level) + 800 0.086 0.000 0.088 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) + 200 0.031 0.000 0.033 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) + 400 0.000 0.000 0.028 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() + 200 0.000 0.000 0.028 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) + 800/200 0.001 0.000 0.027 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) + 800/200 0.006 0.000 0.027 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) + 1200 0.002 0.000 0.022 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) + 1200 0.001 0.000 0.019 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) + 400 0.003 0.000 0.018 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:188(from_fields) + 200 0.001 0.000 0.018 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1437() + 1200 0.017 0.000 0.017 0.000 {method 'cumsum' of 'numpy.ndarray' objects} + 200 0.002 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) + 200 0.001 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_rag_variants.py:210(__init__) + 200 0.009 0.000 0.014 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) + 1200 0.013 0.000 0.013 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/function_base.py:1324(diff) + 1400 0.012 0.000 0.012 0.000 {built-in method numpy.ascontiguousarray} + + +``` + +### pyinstrument variants germline (K=200) + +``` + + _ ._ __/__ _ _ _ _ _/_ Recorded: 00:31:56 Samples: 1200 + /_//_/// /_\ / //_// / //_'/ // Duration: 1.070 CPU time: 1.064 +/ _/ v5.1.2 + +Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 + +1.0697 main prof_python.py:16 +`- 1.0697 call prof_getitem.py:48 + `- 1.0697 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 + `- 1.0697 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 + `- 1.0692 getitem genvarloader/_dataset/_query.py:66 + |- 0.9705 _getitem_unspliced genvarloader/_dataset/_query.py:154 + | `- 0.9705 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 + | `- 0.9705 Svar2Haps._reconstruct_variants genvarloader/_dataset/_svar2_haps.py:527 + | |- 0.5452 decode_variants_from_svar2_readbound + | |- 0.1100 [self] genvarloader/_dataset/_svar2_haps.py + | |- 0.1062 Svar2Haps._gather_inputs genvarloader/_dataset/_svar2_haps.py:632 + | | |- 0.0954 ascontiguousarray + | | `- 0.0108 memmap.__getitem__ numpy/core/memmap.py:334 + | |- 0.1040 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 + | | `- 0.1040 repeat numpy/core/fromnumeric.py:423 + | | `- 0.1040 _wrapfunc numpy/core/fromnumeric.py:53 + | | `- 0.1040 ndarray.repeat + | `- 0.1020 _ragged_arange_gather_2level genvarloader/_dataset/_svar2_haps.py:101 + | `- 0.0974 repeat numpy/core/fromnumeric.py:423 + | `- 0.0974 _wrapfunc numpy/core/fromnumeric.py:53 + | `- 0.0969 ndarray.repeat + `- 0.0978 genvarloader/_dataset/_query.py:119 + `- 0.0968 _reshape_outer genvarloader/_dataset/_query.py:131 + `- 0.0963 RaggedVariants.reshape seqpro/rag/_core.py:1428 + `- 0.0948 RaggedVariants._reshape_impl seqpro/rag/_core.py:1434 + `- 0.0903 from_fields seqpro/rag/_core.py:188 + |- 0.0617 seqpro/rag/_core.py:206 + | `- 0.0577 array_equal numpy/core/numeric.py:2378 + | |- 0.0417 [self] numpy/core/numeric.py + | `- 0.0125 _all numpy/core/_methods.py:61 + `- 0.0160 seqpro/rag/_core.py:213 + +``` + +2026-07-06 00:32:02.542 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl +2026-07-06 00:32:02.562 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. +2026-07-06 00:32:02.563 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 + + ncalls tottime percall cumtime percall filename:lineno(function) + 50 0.338 0.007 53.447 1.069 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) + 50 0.000 0.000 53.109 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) + 50 0.001 0.000 53.108 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) + 50 0.001 0.000 53.106 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) + 50 0.047 0.001 53.099 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) + 50 0.360 0.007 53.036 1.061 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) + 50 0.554 0.011 52.675 1.053 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:319(get_haps_and_shifts) + 50 6.248 0.125 49.781 0.996 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:714(_assemble_haps) + 50 11.425 0.228 43.483 0.870 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) + 300 0.001 0.000 20.497 0.068 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) + 150 0.001 0.000 20.489 0.137 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) + 150 20.487 0.137 20.487 0.137 {method 'repeat' of 'numpy.ndarray' objects} + 150 11.570 0.077 11.570 0.077 {built-in method numpy.arange} + 50 1.400 0.028 1.400 0.028 {built-in method genvarloader.genvarloader.reconstruct_haplotypes_from_svar2_readbound} + 50 0.551 0.011 0.551 0.011 {built-in method genvarloader.genvarloader.hap_diffs_from_svar2_readbound} + 100 0.013 0.000 0.343 0.003 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) + 400 0.293 0.001 0.294 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) + 50 0.041 0.001 0.044 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) + 800 0.036 0.000 0.036 0.000 {built-in method numpy.ascontiguousarray} + 50 0.002 0.000 0.026 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) + 50 0.000 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) + 50 0.002 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) + 200 0.012 0.000 0.012 0.000 {method 'astype' of 'numpy.ndarray' objects} + 50 0.009 0.000 0.012 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) + 50 0.011 0.000 0.011 0.000 {method 'sort' of 'numpy.ndarray' objects} + 100 0.001 0.000 0.009 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) + 100 0.000 0.000 0.008 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) + 100 0.007 0.000 0.007 0.000 {method 'cumsum' of 'numpy.ndarray' objects} + 100 0.007 0.000 0.007 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/function_base.py:1324(diff) + 100 0.000 0.000 0.005 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() + + +``` + +### pyinstrument haplotypes somatic (K=50) + +``` + + _ ._ __/__ _ _ _ _ _/_ Recorded: 00:33:03 Samples: 1451 + /_//_/// /_\ / //_// / //_'/ // Duration: 53.412 CPU time: 53.899 +/ _/ v5.1.2 + +Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 + +53.4118 main prof_python.py:16 +`- 53.4118 call prof_getitem.py:48 + `- 53.0665 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 + `- 53.0665 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 + `- 53.0665 getitem genvarloader/_dataset/_query.py:66 + `- 53.0665 _getitem_unspliced genvarloader/_dataset/_query.py:154 + `- 53.0170 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 + `- 52.6492 Svar2Haps.get_haps_and_shifts genvarloader/_dataset/_svar2_haps.py:319 + |- 49.7006 Svar2Haps._assemble_haps genvarloader/_dataset/_svar2_haps.py:714 + | |- 43.3816 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 + | | |- 20.5025 repeat numpy/core/fromnumeric.py:423 + | | | `- 20.5025 _wrapfunc numpy/core/fromnumeric.py:53 + | | | `- 20.5025 ndarray.repeat + | | |- 11.5908 arange + | | `- 11.2883 [self] genvarloader/_dataset/_svar2_haps.py + | `- 6.2733 [self] genvarloader/_dataset/_svar2_haps.py + |- 1.4053 reconstruct_haplotypes_from_svar2_readbound + |- 0.5750 [self] genvarloader/_dataset/_svar2_haps.py + `- 0.5739 hap_diffs_from_svar2_readbound + +``` + +2026-07-06 00:34:04.651 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_variants.gvl +2026-07-06 00:34:04.732 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. +2026-07-06 00:34:04.732 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 + + ncalls tottime percall cumtime percall filename:lineno(function) + 200 0.175 0.001 4.216 0.021 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) + 200 0.000 0.000 4.041 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) + 200 0.003 0.000 4.041 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) + 200 0.001 0.000 4.036 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) + 200 0.167 0.001 3.994 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) + 200 0.071 0.000 3.780 0.019 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) + 200 0.270 0.001 3.709 0.019 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:527(_reconstruct_variants) + 200 2.010 0.010 2.010 0.010 {built-in method genvarloader.genvarloader.decode_variants_from_svar2_readbound} + 200 0.033 0.000 0.809 0.004 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) + 800 0.693 0.001 0.696 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) + 3000 0.002 0.000 0.224 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) + 400 0.068 0.000 0.223 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) + 1600 0.001 0.000 0.172 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) + 1600 0.169 0.000 0.169 0.000 {method 'repeat' of 'numpy.ndarray' objects} + 200 0.152 0.001 0.160 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) + 200 0.035 0.000 0.113 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:101(_ragged_arange_gather_2level) + 1400 0.080 0.000 0.080 0.000 {built-in method numpy.ascontiguousarray} + 200 0.005 0.000 0.065 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) + 1200 0.003 0.000 0.053 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) + 200 0.001 0.000 0.052 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) + 200 0.006 0.000 0.051 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) + 1200 0.001 0.000 0.049 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) + 1200 0.047 0.000 0.047 0.000 {method 'cumsum' of 'numpy.ndarray' objects} + 200 0.042 0.000 0.042 0.000 {method 'sort' of 'numpy.ndarray' objects} + 400 0.000 0.000 0.039 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() + 200 0.000 0.000 0.039 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) + 200 0.031 0.000 0.038 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) + 800/200 0.001 0.000 0.038 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) + 800/200 0.007 0.000 0.037 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) + 400 0.004 0.000 0.036 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:188(from_fields) + + +``` + +### pyinstrument variants somatic (K=200) + +``` + + _ ._ __/__ _ _ _ _ _/_ Recorded: 00:34:14 Samples: 2807 + /_//_/// /_\ / //_// / //_'/ // Duration: 4.232 CPU time: 4.210 +/ _/ v5.1.2 + +Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 + +4.2318 main prof_python.py:16 +`- 4.2318 call prof_getitem.py:48 + |- 4.0152 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 + | `- 4.0152 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 + | `- 4.0152 getitem genvarloader/_dataset/_query.py:66 + | `- 4.0137 _getitem_unspliced genvarloader/_dataset/_query.py:154 + | |- 3.8348 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 + | | |- 3.6962 Svar2Haps._reconstruct_variants genvarloader/_dataset/_svar2_haps.py:527 + | | | |- 2.0787 decode_variants_from_svar2_readbound + | | | |- 0.7419 Svar2Haps._gather_inputs genvarloader/_dataset/_svar2_haps.py:632 + | | | | `- 0.7413 memmap.__getitem__ numpy/core/memmap.py:334 + | | | |- 0.3715 [self] genvarloader/_dataset/_svar2_haps.py + | | | |- 0.2237 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 + | | | | |- 0.1215 [self] genvarloader/_dataset/_svar2_haps.py + | | | | `- 0.1012 repeat numpy/core/fromnumeric.py:423 + | | | | `- 0.1012 _wrapfunc numpy/core/fromnumeric.py:53 + | | | | `- 0.1012 ndarray.repeat + | | | |- 0.1660 _inverse_row_perm genvarloader/_dataset/_svar2_haps.py:695 + | | | `- 0.1110 _ragged_arange_gather_2level genvarloader/_dataset/_svar2_haps.py:101 + | | | |- 0.0569 repeat numpy/core/fromnumeric.py:423 + | | | | `- 0.0569 _wrapfunc numpy/core/fromnumeric.py:53 + | | | | `- 0.0569 ndarray.repeat + | | | `- 0.0541 [self] genvarloader/_dataset/_svar2_haps.py + | | `- 0.1386 [self] genvarloader/_dataset/_svar2_haps.py + | `- 0.1784 [self] genvarloader/_dataset/_query.py + `- 0.2166 [self] prof_getitem.py + +``` + + +## Top Python functions (ranked) + +Ranked by cProfile cumulative time (`cumtime`), cross-checked against the +pyinstrument call trees. "gvl Python" = defined in `python/genvarloader/`. +FFI wrappers = `{built-in method genvarloader.genvarloader.*}` frames — these +are single-call-boundary hops into the compiled Rust extension; their +`tottime` is Rust execution time, not Python interpreter overhead. + +### haplotypes mode (germline K=200, somatic K=50) + +| rank | function | cumtime (germline / somatic) | tottime (germline / somatic) | kind | +|---|---|---|---|---| +| 1 | `Svar2Haps.get_haps_and_shifts` (`_svar2_haps.py:319`) | 42.599s / 52.675s | 0.146s / 0.554s | pure-Python dispatcher (thin — nearly all time is in callees) | +| 2 | `Svar2Haps._assemble_haps` (`_svar2_haps.py:714`) | 38.910s / 49.781s | 4.692s / 6.248s | pure-Python, real work — non-trivial self-time plus calls into `_ragged_arange_gather` | +| 3 | `_ragged_arange_gather` (`_svar2_haps.py:80`) | 34.167s / 43.483s | 8.965s / 11.425s | pure-Python hot loop — dominates via the numpy calls it issues (`ndarray.repeat` 16.3s/20.5s, `numpy.arange` 8.9s/11.6s of tottime); this, not `_gather_inputs`, is the second-largest gvl-Python cost center | +| 4 | `reconstruct_haplotypes_from_svar2_readbound` (built-in) | 1.972s / 1.400s | 1.972s / 1.400s | **FFI wrapper** (Rust) — thin, self-time is Rust-side | +| 5 | `hap_diffs_from_svar2_readbound` (built-in) | 1.271s / 0.551s | 1.271s / 0.551s | **FFI wrapper** (Rust) — thin, self-time is Rust-side | +| — | `Svar2Haps._gather_inputs` (`_svar2_haps.py:632`) | 0.253s / 0.343s | 0.008s / 0.013s | pure-Python — **much smaller than expected**; see note below | + +**Deviation from expectation:** the brief expected `_gather_inputs` to rank among the top 3 +alongside `get_haps_and_shifts` and `_assemble_haps`. It does not — observed cumtime for +`_gather_inputs` is 0.25–0.34s vs. 38.9–49.8s for `_assemble_haps`, i.e. ~2 orders of magnitude +smaller in haplotypes mode. The actual #2/#3 hot spots are `_assemble_haps` itself (self-time) +and `_ragged_arange_gather`, which together account for essentially all non-FFI time; the +`numpy.ndarray.repeat` and `numpy.arange` calls issued *from inside* `_ragged_arange_gather` +are the single largest tottime consumers of the whole profile (16–20s and 9–12s respectively +at K=200/50). `_inverse_row_perm` and `_contig_groups` are present but negligible (<0.05s cum). + +### variants mode (germline K=200, somatic K=200) + +| rank | function | cumtime (germline / somatic) | tottime (germline / somatic) | kind | +|---|---|---|---|---| +| 1 | `Svar2Haps._reconstruct_variants` (`_svar2_haps.py:527`) | 0.973s / 3.709s | 0.041s / 0.270s | pure-Python dispatcher, thin | +| 2 | `decode_variants_from_svar2_readbound` (built-in) | 0.542s / 2.010s | 0.542s / 2.010s | **FFI wrapper** (Rust) — largest single tottime consumer in this mode | +| 3 | `Svar2Haps._gather_inputs` (`_svar2_haps.py:632`) | 0.104s / 0.809s | 0.003s / 0.033s | pure-Python — scales up sharply with cohort size (germline 3202 vs. somatic 16007 samples); dominated by `numpy.memmap.__getitem__` (0.696s tottime at somatic scale) | +| 4 | `_ragged_arange_gather` (`_svar2_haps.py:80`) | 0.105s / 0.223s | 0.030s / 0.068s | pure-Python hot loop (same as haplotypes mode, smaller absolute magnitude here) | +| 5 | `_ragged_arange_gather_2level` (`_svar2_haps.py:101`) | 0.099s / 0.113s | 0.029s / 0.035s | pure-Python hot loop, 2-level variant used for variants mode | +| — | `_contig_groups` (`_svar2_haps.py:619`) | 0.015s / 0.065s | 0.002s / 0.006s | pure-Python, small but present as expected | + +**Matches expectation:** the ranking generally confirms the brief's predicted list +(`_reconstruct_variants`, `_gather_inputs`, `_ragged_arange_gather`/`_2level`, `_contig_groups`), +with one addition — `decode_variants_from_svar2_readbound` (Rust FFI) is actually the single +largest *tottime* contributor in variants mode, ahead of any pure-Python gvl function. Also +notable: `_gather_inputs` is proportionally much more expensive for variants (rank 3, scales +with cohort/memmap-read size) than for haplotypes (negligible), the inverse of the brief's +haplotypes-mode expectation. + +### Cross-cutting observation + +In **both** modes, the true CPU-bound hot path is the pure-Python `_ragged_arange_gather` +(and its `_2level` sibling) issuing repeated small `numpy.arange`/`ndarray.repeat` calls in a +Python loop — this is the clearest "vectorize or push to Rust" candidate surfaced by this +baseline. The Rust FFI calls (`reconstruct_haplotypes_from_svar2_readbound`, +`hap_diffs_from_svar2_readbound`, `decode_variants_from_svar2_readbound`) are thin +single-hop wrappers whose cost is compiled-code execution time, not Python overhead, and are +out of scope for Python-layer optimization. diff --git a/tmp/svar2_mvp/prof_python.py b/tmp/svar2_mvp/prof_python.py new file mode 100644 index 00000000..ea54bc17 --- /dev/null +++ b/tmp/svar2_mvp/prof_python.py @@ -0,0 +1,41 @@ +"""cProfile + pyinstrument over the live read (Python-layer attribution). + + python tmp/svar2_mvp/prof_python.py + +cProfile ranks Python functions by cumulative time; pyinstrument gives a +low-overhead statistical wall-clock call tree as a cross-check (cProfile's own +per-call overhead can distort tiny hot loops).""" +import cProfile +import io +import pstats +import sys + +from prof_getitem import make_call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + + pr = cProfile.Profile() + pr.enable() + for _ in range(K): + call() + pr.disable() + s = io.StringIO() + pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats(30) + print(f"### cProfile {mode} {cohort} (K={K}), sort=cumulative\n") + print("```\n" + s.getvalue() + "```\n") + + from pyinstrument import Profiler + p = Profiler(interval=0.0005) + p.start() + for _ in range(K): + call() + p.stop() + print(f"### pyinstrument {mode} {cohort} (K={K})\n") + print("```\n" + p.output_text(unicode=False, color=False, show_all=False) + "```\n") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) From 1e28077109fdf771811346659aa05a269ff569eb Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 01:04:04 -0700 Subject: [PATCH 047/108] perf(svar2): native-layer baseline profile (perf DSO/symbol/callgraph + instr reference) --- .../prof_out/readbound/native_baseline.md | 555 ++++++++++++++++++ tmp/svar2_mvp/prof_perf.sh | 41 ++ 2 files changed, 596 insertions(+) create mode 100644 tmp/svar2_mvp/prof_out/readbound/native_baseline.md create mode 100644 tmp/svar2_mvp/prof_perf.sh diff --git a/tmp/svar2_mvp/prof_out/readbound/native_baseline.md b/tmp/svar2_mvp/prof_out/readbound/native_baseline.md new file mode 100644 index 00000000..7f39c035 --- /dev/null +++ b/tmp/svar2_mvp/prof_out/readbound/native_baseline.md @@ -0,0 +1,555 @@ +# SVAR2 read-bound native baseline (2026-07-06) +## haplotypes_germline (K=178) +### perf stat +2026-07-06 00:45:55.272 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl +2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. +2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - + 1.42% [.] __rustc::__rust_dealloc - - + 1.17% [.] arr_unravel_index - - +``` +### call graph (top) +``` + 40.02% 0.00% [.] 0x00005555558b5300 - - + | + ---0x5555558b5300 + | + --39.84%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold + pyo3::impl_::trampoline::fastcall_cfunction_with_keywords + pyo3::impl_::trampoline::trampoline + genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound + | + --39.77%--genvarloader::ffi::decode_variants_from_svar2_readbound + | + |--38.02%--pyo3::marker::Python::detach + | | + | |--17.43%--genoray_core::query::gather_haps_readbound + | | | + | | |--1.61%--genoray_core::spine::merge_keys + | | | + | | |--0.98%--__rustc::__rust_dealloc + | | | + | | --0.69%--_int_free + | | + | |--17.04%--genvarloader::svar2::decode_variants_from_split + | | | + | | |--5.05%--genvarloader::svar2::split_to_flat + | | | + | | |--2.35%--svar2_codec::decode_key + | | | + | | --1.04%--alloc::raw_vec::RawVec::grow_one + | | + | --0.58%--__memmove_avx_unaligned_erms + | + --1.55%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter + 39.85% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - + | + ---genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound + | + --39.78%--genvarloader::ffi::decode_variants_from_svar2_readbound + | + |--38.03%--pyo3::marker::Python::detach + | | + | |--17.43%--genoray_core::query::gather_haps_readbound + | | | + | | |--1.61%--genoray_core::spine::merge_keys + | | | + | | |--0.98%--__rustc::__rust_dealloc +``` +## haplotypes_somatic (K=38) +### perf stat +2026-07-06 00:49:32.456 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl +2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. +2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 `gather_haps_readbound`, confirmed via +`src/ffi/mod.rs`). Confirmed via a dedicated symbol-usage check, not by inspection of this +report alone. + +Also note: the readbound haplotype FFI entry point is +`genvarloader::ffi::reconstruct_haplotypes_from_svar2_readbound` (`nm` confirms this is a +distinct symbol from the union-path `reconstruct_haplotypes_from_svar2`); neither +appears with meaningful self-time because they are thin PyO3 wrapper functions - the real +work happens in `gather_haps_readbound` (genoray_core) underneath. This is expected, not a +driver bug. + +### haplotypes (germline K=178, somatic K=38) + +Top 3 self-time symbols, by DSO: + +1. `mapiter_trivial_get` - numpy (`_multiarray_umath...so`) - 10.74% (germline) / 9.76% (somatic) +2. `LONG_subtract_AVX2` / `LONG_add_AVX2` - numpy - 10.04%/9.98% (germline) / 7.74%/7.88% (somatic) +3. `genoray_core::query::gather_haps_readbound` - genoray_core (statically linked into `genvarloader.abi3.so`) - 6.70% (germline) / 0.52% (somatic, see caveat below) + +Rust self-time in the gvl/genoray_core layer (`gather_haps_readbound` + `split_to_flat` +0.60% + `hap_diffs_svar2` 0.55%, germline) sums to ~7.85%, dwarfed by the ~30% combined +numpy fancy-indexing/arithmetic self-time (`mapiter_trivial_get` + `LONG_add_AVX2` + +`LONG_subtract_AVX2`) and by a large unresolved kernel/`[unknown]` component (31-32% DSO +share, germline; 28-29%, somatic) that lines up with sys time dominating user time in +`perf stat` (30.0s sys vs 15.9s user germline; 32.6s sys vs 19.8s user somatic) - almost +certainly page-fault/mmap overhead (reference-genome memmap and/or per-sample-scale array +allocation), not Rust reconstruction. `variants` mode by contrast has ~1-14s sys time, +confirming this kernel-time cost is haplotypes-specific. + +**B1 double-gather hint (gather_haps_readbound + split_to_flat vs diffs-only need):** +PARTIALLY SUPPORTED / INCONCLUSIVE ON THE EXACT RATIO. Within the Rust layer, +`gather_haps_readbound` (6.70%, germline) self-time is ~12x `hap_diffs_svar2`'s (0.55%), +qualitatively consistent with "gathering full haplotype sequences" costing far more than +computing diffs alone - i.e. supports that a diffs-only reconstruction path would be much +cheaper than the current full-gather. But the profiled ratio is far larger than the +brief's "~2x" expectation, and self-time percentages alone can't distinguish "redundant +double work" from "gather legitimately touches more bytes than diffs do" - call-count +instrumentation (not just perf self-time) is needed to pin the exact redundancy factor. +Bigger caveat: for haplotypes, the gvl/genoray Rust layer is a minority contributor +(~8%) of total native time; the numpy fancy-indexing arithmetic (~30%) and kernel/mmap +overhead (~30%) are larger targets by self-time. **Verdict: directionally supports B1 but +does not confirm the specific "~2x" claim; a B1 fix should be scoped alongside (not +instead of) investigating the numpy-level indexing arithmetic and the mmap/page-fault +kernel time, or its net effect on wall-clock will be modest.** + +The somatic cohort's `gather_haps_readbound` self-time (0.52%) is much lower than +germline's (6.70%) despite germline and somatic using the same code path; the somatic +haplotypes capture is instead dominated by `PyUnicode_RichCompare`/`PyObject_RichCompare`/ +`list_contains`/`list_index` (python3.10, ~9.6%/3.3%/3.2%/1.9%/0.8%) - likely +sample-name/list matching overhead scaling with the 16007-sample cohort, worth a separate +look but out of scope for the B1/B2 hints named in this brief. + +### variants (germline K=6547, somatic K=1922) + +Top 3 self-time symbols, by DSO: + +1. `genoray_core::query::gather_haps_readbound` - genoray_core (in `genvarloader.abi3.so`) - 12.85% (germline) / 6.75% (somatic) +2. `genvarloader::svar2::decode_variants_from_split` + `genvarloader::svar2::split_to_flat` - gvl `.so` (`genvarloader::svar2`) - 6.59%+5.05%=11.64% (germline) / 2.73%+~ (somatic, split_to_flat drops out of top-20) +3. `PyArray_Repeat` - numpy - 9.13% (germline) / 4.98% (somatic) + +**B2 allocation-churn hint (split_to_flat/decode_variants_from_split with `_int_malloc`/`SpecFromIter` beneath):** +SUPPORTED. The call graph shows `decode_variants_from_split` as a direct child of +`gather_haps_readbound`'s sibling call (via `pyo3::marker::Python::detach` -> +`genvarloader::ffi::decode_variants_from_svar2_readbound`), with `split_to_flat`, +`svar2_codec::decode_key`, and `alloc::raw_vec::RawVec::grow_one` nested directly +beneath it - `grow_one` is a live allocation-growth event, not incidental. Allocator +self-time symbols appear prominently at the top level in both cohorts: `_int_free` +(4.31% germline / 4.09% somatic), `_int_malloc` (1.71%/2.15%), `malloc`/`__libc_calloc` +(2.06%+1.56% germline / 2.37% somatic), and +` as SpecFromIterNested>::from_iter` (1.55% germline, a Rust +Vec-from-iterator allocation pattern) and `__rustc::__rust_dealloc` (1.42%/1.46%) - all +consistent with the decode/flatten step allocating and freeing many small buffers per +call rather than reusing/pre-sizing them. This is a clear Task B2 target: reduce +allocation count in `split_to_flat`/`decode_variants_from_split` (pre-size `Vec`s, +avoid `from_iter` on hot paths, reuse buffers across calls). diff --git a/tmp/svar2_mvp/prof_perf.sh b/tmp/svar2_mvp/prof_perf.sh new file mode 100644 index 00000000..81755ae8 --- /dev/null +++ b/tmp/svar2_mvp/prof_perf.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Native-layer attribution for the live read-bound path via perf (py-spy is +# unusable: ptrace_scope=2; Python 3.10 has no perf trampoline so Python frames +# are opaque -> DSO-level + Rust-symbol self-time is the split). +set -eu +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/readbound +mkdir -p "$OUT" +PERF=/carter/users/dlaub/.pixi/bin/perf +PY=.pixi/envs/dev/bin/python +FREQ=299 +REPORT="$OUT/native_baseline.md" +echo "# SVAR2 read-bound native baseline ($(date -I))" > "$REPORT" + +probe_K () { # mode cohort -> K sized to ~40s + local per + per=$("$PY" tmp/svar2_mvp/prof_getitem.py "$1" "$2" 5 | sed 's/per_call_s=//') + "$PY" -c "import math;print(max(20,math.ceil(40/max(float('$per'),1e-4))))" +} + +for c in germline somatic; do for m in haplotypes variants; do + tag="${m}_${c}"; K=$(probe_K "$m" "$c") + echo "## $tag (K=$K)" | tee -a "$REPORT" + # instruction-count reference (the Phase-B gate baseline) + echo '### perf stat' >> "$REPORT" + { "$PERF" stat -e instructions,cycles -- "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" ; } \ + 2>> "$REPORT" 1>/dev/null || echo "(perf stat HW counters unavailable)" >> "$REPORT" + # sampling profile + "$PERF" record -g --call-graph fp -F $FREQ -o "$OUT/$tag.data" -- \ + "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" >/dev/null 2>&1 + echo '### DSO split' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=dso --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -12 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### top Rust/native self-time symbols' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=symbol --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -20 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### call graph (top)' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=overhead,symbol -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -45 >> "$REPORT"; echo '```' >> "$REPORT" +done; done +echo "NATIVE_BASELINE_DONE -> $REPORT" From 914a8b9d519e73f10ff3c9b464db00cfe3d6bdf7 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 01:34:07 -0700 Subject: [PATCH 048/108] perf(svar2): skip redundant pre-reconstruct gather for deterministic haplotype reads --- python/genvarloader/_dataset/_svar2_haps.py | 56 +++++++++++++-------- tests/dataset/test_svar2_readbound_haps.py | 43 ++++++++++++++++ 2 files changed, 79 insertions(+), 20 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 679492f9..6c56a14a 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -313,6 +313,7 @@ def __call__( deterministic=deterministic, splice_plan=splice_plan, to_rc=to_rc, + need_hap_lengths=False, ) return cast(_H, haps) @@ -325,6 +326,7 @@ def get_haps_and_shifts( deterministic: bool, splice_plan: "SplicePlan | None" = None, to_rc: "NDArray[np.bool_] | None" = None, + need_hap_lengths: bool = True, ) -> tuple[ Ragged[np.bytes_], NDArray[np.intp], @@ -351,33 +353,47 @@ def get_haps_and_shifts( groups = self._contig_groups(contig_ids) + # diffs are needed pre-reconstruct ONLY to (a) bound randomized jitter + # shifts, or (b) return hap_lengths/diffs to a caller that uses them + # (the tracks path). A deterministic/ragged haplotypes read needs + # neither: reconstruct sizes itself internally. Avoid the redundant + # gather+split+diffs in that (common warm-read) case. + randomized = not (deterministic or isinstance(output_length, str)) + need_diffs = randomized or need_hap_lengths + # --- diffs (per contig group, stitched back to (b, P) query order) --- - diffs = np.empty((b, P), np.int32) - for ci, qsel in groups: - gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) - d = hap_diffs_from_svar2_readbound( - self.store, - self.ds_contigs[ci], - gi[0], - gi[1], - gi[2], - gi[3], - gi[4], - gi[5], - gi[6], - P, - ) - diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) + if need_diffs: + diffs = np.empty((b, P), np.int32) + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + d = hap_diffs_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + P, + ) + diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) - hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + else: + diffs = np.zeros((b, P), np.int32) # placeholder (unused downstream) + hap_lengths = np.broadcast_to( + lengths[:, None].astype(np.int32), (b, P) + ).copy() # --- shifts (single rng draw; mirrors Haps._prepare_request) --- - if deterministic or isinstance(output_length, str): - shifts = np.zeros((b, P), np.int32) - else: + if randomized: max_shift = diffs.clip(min=0) max_shift = max_shift + (lengths - output_length).clip(min=0)[:, None] shifts = rng.integers(0, max_shift + 1, dtype=np.int32) + else: + shifts = np.zeros((b, P), np.int32) ffi_out_len = ( np.int64(-1) if isinstance(output_length, str) else np.int64(output_length) diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py index 7ffbfc2a..94b05e93 100644 --- a/tests/dataset/test_svar2_readbound_haps.py +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -290,3 +290,46 @@ def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): assert np.array_equal( np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") ) + + +def _svar2_haps_dataset(tmp_path: Path, svar2_store: Path): + """Build a full gvl Dataset over the ``svar2_store`` fixture and return its + haplotypes view (Svar2Haps-backed). + + Lifted/adapted from ``test_svar2_dataset.py::_open_pair`` -- this file has no + existing fixture that yields a live gvl.Dataset (only bare genoray stores), + so this helper builds the minimal one needed to exercise Svar2Haps through + the public Dataset API. + """ + import polars as pl + from genoray import SparseVar2 + + import genvarloader as gvl + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ref = svar2_store.parent / "ref.fa" + d = tmp_path / "ds.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_store), samples=None, overwrite=True) + return gvl.Dataset.open(d, reference=ref).with_seqs("haplotypes") + + +def test_deterministic_haps_read_skips_pre_reconstruct_diffs( + tmp_path: Path, svar2_store: Path, monkeypatch: pytest.MonkeyPatch +): + """A deterministic (shifts=0) haplotypes read must NOT call the separate + hap_diffs readbound kernel -- reconstruct sizes itself internally. Guards the + double-gather regression.""" + import genvarloader._dataset._svar2_haps as m + + calls = {"diffs": 0} + real = m.hap_diffs_from_svar2_readbound + + def counting(*a, **k): + calls["diffs"] += 1 + return real(*a, **k) + + monkeypatch.setattr(m, "hap_diffs_from_svar2_readbound", counting) + + ds2 = _svar2_haps_dataset(tmp_path, svar2_store) + ds2[:, :] + assert calls["diffs"] == 0 From ee9c219d4138340500f9408b6627d87612f570a9 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 02:14:05 -0700 Subject: [PATCH 049/108] perf(svar2): pre-size split_to_flat + decode_variants_from_split allocations (byte-identical) --- src/svar2/mod.rs | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 97da8d72..b2bb4408 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -165,8 +165,15 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { let vk_key: Vec = br.vk.iter().map(|k| k.key as i32).collect(); let vk_off: Vec = br.vk_off.iter().map(|&o| o as i64).collect(); - let mut dense_pos: Vec = Vec::new(); - let mut dense_key: Vec = Vec::new(); + let dense_total: usize = (0..n_q) + .map(|q| { + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_pos: Vec = Vec::with_capacity(dense_total); + let mut dense_key: Vec = Vec::with_capacity(dense_total); let mut dense_range: Vec = Vec::with_capacity(n_q * 2); for q in 0..n_q { let base = dense_pos.len() as i32; @@ -184,7 +191,15 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { dense_range.push(dense_pos.len() as i32); } - let mut dense_present: Vec = Vec::new(); + let total_bits: usize = (0..h_count) + .map(|h| { + let q = h / ploidy; + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_present: Vec = vec![0u8; total_bits.div_ceil(8)]; let mut dense_present_off: Vec = Vec::with_capacity(h_count + 1); let mut bit_acc: usize = 0; dense_present_off.push(0); @@ -195,22 +210,14 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { let snp_base = br.dense_snp_present_off[h]; for k in 0..(se - ss) { if genoray_core::bits_get_bit(&br.dense_snp_present, snp_base + k) { - let byte = bit_acc / 8; - if dense_present.len() <= byte { - dense_present.resize(byte + 1, 0); - } - dense_present[byte] |= 1 << (bit_acc % 8); + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); } bit_acc += 1; } let indel_base = br.dense_indel_present_off[h]; for k in 0..(ie - is_) { if genoray_core::bits_get_bit(&br.dense_indel_present, indel_base + k) { - let byte = bit_acc / 8; - if dense_present.len() <= byte { - dense_present.resize(byte + 1, 0); - } - dense_present[byte] |= 1 << (bit_acc % 8); + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); } bit_acc += 1; } @@ -269,10 +276,16 @@ pub fn decode_variants_from_split( let n_q = br.n_regions; let h_count = n_q * ploidy; - let mut pos: Vec = Vec::new(); - let mut ilen: Vec = Vec::new(); - let mut alt_bytes: Vec = Vec::new(); - let mut str_off: Vec = vec![0]; + // Upper bound on total merged variants across all haps: every vk entry plus + // every dense window entry (present or not). Over-reserving is harmless. + let vk_total = flat.vk_off[h_count] as usize; + let dense_bits = flat.dense_present_off[h_count] as usize; + let cap = vk_total + dense_bits; + let mut pos: Vec = Vec::with_capacity(cap); + let mut ilen: Vec = Vec::with_capacity(cap); + let mut alt_bytes: Vec = Vec::with_capacity(cap); + let mut str_off: Vec = Vec::with_capacity(cap + 1); + str_off.push(0); let mut var_off: Vec = Vec::with_capacity(h_count + 1); var_off.push(0); From cc4ee6ed8713d269f4da73d533b0b218dc711e43 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 02:24:12 -0700 Subject: [PATCH 050/108] perf(svar2): compute the pos/ilen ragged reorder index once in variants decode --- python/genvarloader/_dataset/_svar2_haps.py | 37 +++++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 6c56a14a..62b18ec6 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -77,14 +77,14 @@ class _Svar2Cache: sample_cols: NDArray[np.int64] -def _ragged_arange_gather( - data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] -) -> tuple[NDArray, NDArray[np.int64]]: - """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``. - - ``offsets`` has length ``n_rows + 1``; ``perm`` is the new row order - (final row ``i`` == old row ``perm[i]``). Fully vectorized (no Python loop - over rows). Returns ``(new_data, new_offsets)``. +def _ragged_arange_src( + offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray[np.int64], NDArray[np.int64]]: + """Source-row index + new offsets for a 1-level ragged reorder by ``perm``. + + ``new_data == data[src]``; ``src`` and ``new_off`` depend only on + ``(offsets, perm)`` — so callers reordering several parallel data arrays by + the same key compute this ONCE and index each array. """ offsets = np.asarray(offsets, np.int64) lens = np.diff(offsets) @@ -92,9 +92,19 @@ def _ragged_arange_gather( new_off = lengths_to_offsets(new_lens, np.int64) n = int(new_off[-1]) if n == 0: - return data[:0].copy(), new_off + return np.zeros(0, np.int64), new_off within = np.arange(n, dtype=np.int64) - np.repeat(new_off[:-1], new_lens) src = np.repeat(offsets[perm], new_lens) + within + return src, new_off + + +def _ragged_arange_gather( + data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray, NDArray[np.int64]]: + """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``.""" + src, new_off = _ragged_arange_src(offsets, perm) + if src.size == 0: + return data[:0].copy(), new_off return data[src], new_off @@ -598,8 +608,13 @@ def _reconstruct_variants( perm = self._inverse_row_perm(cat_query_order, b, P) - pos_g, var_off_g = _ragged_arange_gather(pos_c, grouped_var_off, perm) - ilen_g, _ = _ragged_arange_gather(ilen_c, grouped_var_off, perm) + src, var_off_g = _ragged_arange_src(grouped_var_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] alt_g, alt_var_off_g, alt_str_off_g = _ragged_arange_gather_2level( alt_c, grouped_var_off, grouped_str_off, perm ) From fa7ca9a1d6742ebcfc941dab1792971edb933ee1 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 02:44:00 -0700 Subject: [PATCH 051/108] perf(svar2): re-profile native layer after B1-B3; enumerate cargo-asm work-list --- .../prof_out/readbound/asm_targets.md | 98 +++ .../prof_out/readbound/native_after_b1b3.md | 555 ++++++++++++++ .../prof_out/readbound/native_baseline.md | 722 ++++++++---------- 3 files changed, 971 insertions(+), 404 deletions(-) create mode 100644 tmp/svar2_mvp/prof_out/readbound/asm_targets.md create mode 100644 tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md diff --git a/tmp/svar2_mvp/prof_out/readbound/asm_targets.md b/tmp/svar2_mvp/prof_out/readbound/asm_targets.md new file mode 100644 index 00000000..3de82502 --- /dev/null +++ b/tmp/svar2_mvp/prof_out/readbound/asm_targets.md @@ -0,0 +1,98 @@ +# SVAR2 read-bound: cargo-asm work-list (post B1-B3) + +Captured 2026-07-06 via `tmp/svar2_mvp/prof_perf.sh` after rebuilding with +`RUSTFLAGS="-C force-frame-pointers=yes"` on top of commits through +`a297d24` (B2: pre-size split_to_flat/decode_variants_from_split). Source +data: `tmp/svar2_mvp/prof_out/readbound/native_baseline.md` (fresh, +post-B1-B3). Pre-B1-B3 numbers preserved in +`tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` (misleadingly named — +it holds the OLD/pre-B1-B3 capture). + +Sanity re-check (A3-style): grep for `overlap_batch`/`dense_union` in the +fresh `native_baseline.md` returned **no matches** — union oracle confirmed +absent from the read-bound path. `SearchTree::build` tops out at 1.54% +(haplotypes_germline), consistent with the benign per-region `find_ranges` +search, not a regression. + +## Per-mode top-5 native symbols (including excluded numpy/libc/kernel), with asm-fixable-Rust characterization + +### haplotypes_germline (K=191) +| self% | symbol | owner | +|---|---|---| +| 12.31% | `[k] 0xffffffffb1a0f327` | kernel (unresolved, page-fault/mmap path) | +| 11.64% | `mapiter_trivial_get` | numpy | +| 11.11% | `LONG_add_AVX2` | numpy | +| 10.86% | `LONG_subtract_AVX2` | numpy | +| 3.86% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | + +Characterization: dominated by numpy int64 add/sub kernels + kernel-side +paging; only `gather_haps_readbound` (3.86%) + `SearchTree::build` (1.54%, +outside top-5) clear our cutoff → **~5.4% of self-time is asm-fixable Rust** +in this mode. B1 (skip redundant haplotype gather) appears to have already +squeezed most of the Rust cost out of the haplotypes path. + +### variants_germline (K=7143) +| self% | symbol | owner | +|---|---|---| +| 18.31% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | +| 6.79% | `PyArray_Repeat` | numpy | +| 5.43% | `genvarloader::svar2::decode_variants_from_split` | gvl (Rust, asm-fixable) | +| 5.21% | `genvarloader::svar2::split_to_flat` | gvl (Rust, asm-fixable) | +| 4.65% | `_int_free` | libc | + +Characterization: this is the mode with the most asm-fixable Rust left — +`gather_haps_readbound` + `decode_variants_from_split` + `split_to_flat` + +`merge_keys` (2.06%) + `svar2_codec::decode_key` (2.33%) sum to **~33.3% of +self-time in gvl/genoray Rust**, the single largest optimization target of +the four modes. + +### haplotypes_somatic (K=37) +| self% | symbol | owner | +|---|---|---| +| 11.02% | `mapiter_trivial_get` | numpy | +| 10.07% | `[k] 0xffffffffb1a0f327` | kernel | +| 9.09% | `PyUnicode_RichCompare` | python | +| 8.09% | `LONG_add_AVX2` | numpy | +| 8.07% | `LONG_subtract_AVX2` | numpy | + +Characterization: essentially **no asm-fixable Rust remains** — only +`SearchTree::build` appears at all (0.55%, below cutoff). Entirely +numpy/python/kernel-structural at this point. + +### variants_somatic (K=1792) +| self% | symbol | owner | +|---|---|---| +| 6.86% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | +| 5.74% | `PyUnicode_RichCompare` | python | +| 4.41% | `[k] 0xffffffffb1a0f327` | kernel | +| 4.02% | `mapiter_get` | numpy | +| 3.89% | `_int_free` | libc | + +Characterization: `gather_haps_readbound` + `decode_variants_from_split` +(2.56%) + `split_to_flat` (1.79%) + `merge_keys` (2.31%) sum to **~13.5% of +self-time in Rust** — about half of variants_germline's asm-fixable budget. + +## Work-list: gvl/genoray native symbols with self-time ≥1.5% in ANY mode + +| symbol | repo/file:line | max self% | modes (self%) | perf.data tag(s) | +|---|---|---|---|---| +| `genoray_core::query::gather_haps_readbound` | genoray `src/query.rs:1086` | 18.31% | haplotypes_germline (3.86%), variants_germline (18.31%), variants_somatic (6.86%) | `haplotypes_germline`, `variants_germline`, `variants_somatic` | +| `genvarloader::svar2::decode_variants_from_split` | gvl `src/svar2/mod.rs:269` | 5.43% | variants_germline (5.43%), variants_somatic (2.56%) | `variants_germline`, `variants_somatic` | +| `genvarloader::svar2::split_to_flat` | gvl `src/svar2/mod.rs:159` | 5.21% | variants_germline (5.21%), variants_somatic (1.79%) | `variants_germline`, `variants_somatic` | +| `svar2_codec::decode_key` | genoray `svar2-codec/src/lib.rs:237` | 2.33% | variants_germline (2.33%) | `variants_germline` | +| `genoray_core::spine::merge_keys` | genoray `src/spine.rs:63` | 2.31% | variants_germline (2.06%), variants_somatic (2.31%) | `variants_germline`, `variants_somatic` | +| `genoray_core::search::SearchTree::build` | genoray `src/search.rs:93` | 1.54% | haplotypes_germline (1.54%), haplotypes_somatic (0.55%, below cutoff) | `haplotypes_germline` | + +**6 functions clear the ≥1.5% cutoff** → fan-out size of 6 for the parallel +cargo-asm pass. All are genoray-owned except `decode_variants_from_split` and +`split_to_flat` (gvl-owned, `src/svar2/mod.rs`). + +Sub-cutoff, noted for completeness (do not fan out on these): `genvarloader::svar2::hap_diffs_svar2` +peaks at 0.58% (haplotypes_germline) — below the 1.5% bar in every mode. + +## Excluded categories (structural, not cargo-asm-fixable) +- libc: `_int_malloc`/`_int_free`/`__memmove_avx_unaligned_erms`/`__memcmp_avx2_movbe`/`malloc`/`__libc_calloc` +- numpy: `_multiarray_umath` internals — `mapiter_trivial_get`/`mapiter_get`/`LONG_add_AVX2`/`LONG_subtract_AVX2`/`PyArray_Repeat`/`npyiter_buffered_iternext`/`_contig_to_contig` +- Python interpreter / GC: `_PyEval_EvalFrameDefault`, `gc_collect_main`, `deduce_unreachable`, `visit_reachable`, `dict_traverse`, `PyUnicode_RichCompare`, `PyObject_RichCompare(Bool)`, `list_contains`, `list_index` +- Rust std/alloc/toolchain (not gvl/genoray application code): `alloc::vec::Vec::from_iter` variants, `__rustc::__rust_dealloc`, `__rustc::__rust_no_alloc_shim_is_unstable_v2`, `alloc::raw_vec::RawVec::grow_one` +- unresolved kernel samples: `[k] 0xffffffffb1a0f327`, `[k] 0xffffffffb14fa26d`, `[k] 0xffffffffb1c011e0` diff --git a/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md b/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md new file mode 100644 index 00000000..7f39c035 --- /dev/null +++ b/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md @@ -0,0 +1,555 @@ +# SVAR2 read-bound native baseline (2026-07-06) +## haplotypes_germline (K=178) +### perf stat +2026-07-06 00:45:55.272 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl +2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. +2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - + 1.42% [.] __rustc::__rust_dealloc - - + 1.17% [.] arr_unravel_index - - +``` +### call graph (top) +``` + 40.02% 0.00% [.] 0x00005555558b5300 - - + | + ---0x5555558b5300 + | + --39.84%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold + pyo3::impl_::trampoline::fastcall_cfunction_with_keywords + pyo3::impl_::trampoline::trampoline + genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound + | + --39.77%--genvarloader::ffi::decode_variants_from_svar2_readbound + | + |--38.02%--pyo3::marker::Python::detach + | | + | |--17.43%--genoray_core::query::gather_haps_readbound + | | | + | | |--1.61%--genoray_core::spine::merge_keys + | | | + | | |--0.98%--__rustc::__rust_dealloc + | | | + | | --0.69%--_int_free + | | + | |--17.04%--genvarloader::svar2::decode_variants_from_split + | | | + | | |--5.05%--genvarloader::svar2::split_to_flat + | | | + | | |--2.35%--svar2_codec::decode_key + | | | + | | --1.04%--alloc::raw_vec::RawVec::grow_one + | | + | --0.58%--__memmove_avx_unaligned_erms + | + --1.55%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter + 39.85% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - + | + ---genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound + | + --39.78%--genvarloader::ffi::decode_variants_from_svar2_readbound + | + |--38.03%--pyo3::marker::Python::detach + | | + | |--17.43%--genoray_core::query::gather_haps_readbound + | | | + | | |--1.61%--genoray_core::spine::merge_keys + | | | + | | |--0.98%--__rustc::__rust_dealloc +``` +## haplotypes_somatic (K=38) +### perf stat +2026-07-06 00:49:32.456 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl +2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. +2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. + 0%| | 0/3 [00:00 `gather_haps_readbound`, confirmed via +`src/ffi/mod.rs`). Confirmed via a dedicated symbol-usage check, not by inspection of this +report alone. + +Also note: the readbound haplotype FFI entry point is +`genvarloader::ffi::reconstruct_haplotypes_from_svar2_readbound` (`nm` confirms this is a +distinct symbol from the union-path `reconstruct_haplotypes_from_svar2`); neither +appears with meaningful self-time because they are thin PyO3 wrapper functions - the real +work happens in `gather_haps_readbound` (genoray_core) underneath. This is expected, not a +driver bug. + +### haplotypes (germline K=178, somatic K=38) + +Top 3 self-time symbols, by DSO: + +1. `mapiter_trivial_get` - numpy (`_multiarray_umath...so`) - 10.74% (germline) / 9.76% (somatic) +2. `LONG_subtract_AVX2` / `LONG_add_AVX2` - numpy - 10.04%/9.98% (germline) / 7.74%/7.88% (somatic) +3. `genoray_core::query::gather_haps_readbound` - genoray_core (statically linked into `genvarloader.abi3.so`) - 6.70% (germline) / 0.52% (somatic, see caveat below) + +Rust self-time in the gvl/genoray_core layer (`gather_haps_readbound` + `split_to_flat` +0.60% + `hap_diffs_svar2` 0.55%, germline) sums to ~7.85%, dwarfed by the ~30% combined +numpy fancy-indexing/arithmetic self-time (`mapiter_trivial_get` + `LONG_add_AVX2` + +`LONG_subtract_AVX2`) and by a large unresolved kernel/`[unknown]` component (31-32% DSO +share, germline; 28-29%, somatic) that lines up with sys time dominating user time in +`perf stat` (30.0s sys vs 15.9s user germline; 32.6s sys vs 19.8s user somatic) - almost +certainly page-fault/mmap overhead (reference-genome memmap and/or per-sample-scale array +allocation), not Rust reconstruction. `variants` mode by contrast has ~1-14s sys time, +confirming this kernel-time cost is haplotypes-specific. + +**B1 double-gather hint (gather_haps_readbound + split_to_flat vs diffs-only need):** +PARTIALLY SUPPORTED / INCONCLUSIVE ON THE EXACT RATIO. Within the Rust layer, +`gather_haps_readbound` (6.70%, germline) self-time is ~12x `hap_diffs_svar2`'s (0.55%), +qualitatively consistent with "gathering full haplotype sequences" costing far more than +computing diffs alone - i.e. supports that a diffs-only reconstruction path would be much +cheaper than the current full-gather. But the profiled ratio is far larger than the +brief's "~2x" expectation, and self-time percentages alone can't distinguish "redundant +double work" from "gather legitimately touches more bytes than diffs do" - call-count +instrumentation (not just perf self-time) is needed to pin the exact redundancy factor. +Bigger caveat: for haplotypes, the gvl/genoray Rust layer is a minority contributor +(~8%) of total native time; the numpy fancy-indexing arithmetic (~30%) and kernel/mmap +overhead (~30%) are larger targets by self-time. **Verdict: directionally supports B1 but +does not confirm the specific "~2x" claim; a B1 fix should be scoped alongside (not +instead of) investigating the numpy-level indexing arithmetic and the mmap/page-fault +kernel time, or its net effect on wall-clock will be modest.** + +The somatic cohort's `gather_haps_readbound` self-time (0.52%) is much lower than +germline's (6.70%) despite germline and somatic using the same code path; the somatic +haplotypes capture is instead dominated by `PyUnicode_RichCompare`/`PyObject_RichCompare`/ +`list_contains`/`list_index` (python3.10, ~9.6%/3.3%/3.2%/1.9%/0.8%) - likely +sample-name/list matching overhead scaling with the 16007-sample cohort, worth a separate +look but out of scope for the B1/B2 hints named in this brief. + +### variants (germline K=6547, somatic K=1922) + +Top 3 self-time symbols, by DSO: + +1. `genoray_core::query::gather_haps_readbound` - genoray_core (in `genvarloader.abi3.so`) - 12.85% (germline) / 6.75% (somatic) +2. `genvarloader::svar2::decode_variants_from_split` + `genvarloader::svar2::split_to_flat` - gvl `.so` (`genvarloader::svar2`) - 6.59%+5.05%=11.64% (germline) / 2.73%+~ (somatic, split_to_flat drops out of top-20) +3. `PyArray_Repeat` - numpy - 9.13% (germline) / 4.98% (somatic) + +**B2 allocation-churn hint (split_to_flat/decode_variants_from_split with `_int_malloc`/`SpecFromIter` beneath):** +SUPPORTED. The call graph shows `decode_variants_from_split` as a direct child of +`gather_haps_readbound`'s sibling call (via `pyo3::marker::Python::detach` -> +`genvarloader::ffi::decode_variants_from_svar2_readbound`), with `split_to_flat`, +`svar2_codec::decode_key`, and `alloc::raw_vec::RawVec::grow_one` nested directly +beneath it - `grow_one` is a live allocation-growth event, not incidental. Allocator +self-time symbols appear prominently at the top level in both cohorts: `_int_free` +(4.31% germline / 4.09% somatic), `_int_malloc` (1.71%/2.15%), `malloc`/`__libc_calloc` +(2.06%+1.56% germline / 2.37% somatic), and +` as SpecFromIterNested>::from_iter` (1.55% germline, a Rust +Vec-from-iterator allocation pattern) and `__rustc::__rust_dealloc` (1.42%/1.46%) - all +consistent with the decode/flatten step allocating and freeing many small buffers per +call rather than reusing/pre-sizing them. This is a clear Task B2 target: reduce +allocation count in `split_to_flat`/`decode_variants_from_split` (pre-size `Vec`s, +avoid `from_iter` on hot paths, reuse buffers across calls). diff --git a/tmp/svar2_mvp/prof_out/readbound/native_baseline.md b/tmp/svar2_mvp/prof_out/readbound/native_baseline.md index 7f39c035..cedf6190 100644 --- a/tmp/svar2_mvp/prof_out/readbound/native_baseline.md +++ b/tmp/svar2_mvp/prof_out/readbound/native_baseline.md @@ -1,12 +1,12 @@ # SVAR2 read-bound native baseline (2026-07-06) -## haplotypes_germline (K=178) +## haplotypes_germline (K=191) ### perf stat -2026-07-06 00:45:55.272 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl -2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. -2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - 0%| | 0/3 [00:00 as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - - 1.42% [.] __rustc::__rust_dealloc - - - 1.17% [.] arr_unravel_index - - + 18.31% [.] genoray_core::query::gather_haps_readbound - - + 6.79% [.] PyArray_Repeat - - + 5.43% [.] genvarloader::svar2::decode_variants_from_split - - + 5.21% [.] genvarloader::svar2::split_to_flat - - + 4.65% [.] _int_free - - + 4.16% [.] mapiter_get - - + 3.78% [.] __memmove_avx_unaligned_erms - - + 3.07% [.] mapiter_trivial_get - - + 2.33% [.] malloc - - + 2.33% [.] svar2_codec::decode_key - - + 2.23% [.] npyiter_buffered_iternext - - + 2.06% [.] genoray_core::spine::merge_keys - - + 2.04% [.] _int_malloc - - + 1.85% [.] _PyEval_EvalFrameDefault - - + 1.70% [.] _contig_to_contig - - + 1.60% [.] LONG_add_AVX2 - - + 1.57% [.] as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - + 1.48% [.] __libc_calloc - - + 1.35% [.] __rustc::__rust_dealloc - - + 1.01% [.] __rustc::__rust_no_alloc_shim_is_unstable_v2 - - ``` ### call graph (top) ``` - 40.02% 0.00% [.] 0x00005555558b5300 - - + 44.40% 0.00% [.] 0x00005555558b5300 - - | ---0x5555558b5300 | - --39.84%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold + --44.26%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold pyo3::impl_::trampoline::fastcall_cfunction_with_keywords pyo3::impl_::trampoline::trampoline - genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound | - --39.77%--genvarloader::ffi::decode_variants_from_svar2_readbound + --44.25%--genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound | - |--38.02%--pyo3::marker::Python::detach - | | - | |--17.43%--genoray_core::query::gather_haps_readbound - | | | - | | |--1.61%--genoray_core::spine::merge_keys - | | | - | | |--0.98%--__rustc::__rust_dealloc - | | | - | | --0.69%--_int_free - | | - | |--17.04%--genvarloader::svar2::decode_variants_from_split - | | | - | | |--5.05%--genvarloader::svar2::split_to_flat - | | | - | | |--2.35%--svar2_codec::decode_key - | | | - | | --1.04%--alloc::raw_vec::RawVec::grow_one - | | - | --0.58%--__memmove_avx_unaligned_erms - | - --1.55%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - 39.85% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - + --44.21%--genvarloader::ffi::decode_variants_from_svar2_readbound + | + |--42.43%--pyo3::marker::Python::detach + | | + | |--23.19%--genoray_core::query::gather_haps_readbound + | | | + | | |--1.90%--genoray_core::spine::merge_keys + | | | + | | |--0.84%--__rustc::__rust_dealloc + | | | + | | |--0.66%--cfree@GLIBC_2.2.5 + | | | + | | --0.62%--_int_free + | | + | |--15.91%--genvarloader::svar2::decode_variants_from_split + | | | + | | |--5.22%--genvarloader::svar2::split_to_flat + | | | + | | |--2.10%--svar2_codec::decode_key + | | | + | | |--1.22%--alloc::raw_vec::RawVec::grow_one + | | | + | | --0.66%-- as alloc::vec::spec_from_iter::SpecFromIter>::from_iter + | | + | --0.51%--__rustc::__rust_dealloc + | + --1.57%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter + 44.26% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - | ---genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound | - --39.78%--genvarloader::ffi::decode_variants_from_svar2_readbound + --44.23%--genvarloader::ffi::decode_variants_from_svar2_readbound | - |--38.03%--pyo3::marker::Python::detach + |--42.44%--pyo3::marker::Python::detach | | - | |--17.43%--genoray_core::query::gather_haps_readbound - | | | - | | |--1.61%--genoray_core::spine::merge_keys - | | | - | | |--0.98%--__rustc::__rust_dealloc ``` -## haplotypes_somatic (K=38) +## haplotypes_somatic (K=37) ### perf stat -2026-07-06 00:49:32.456 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl -2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. -2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - 0%| | 0/3 [00:00 `gather_haps_readbound`, confirmed via -`src/ffi/mod.rs`). Confirmed via a dedicated symbol-usage check, not by inspection of this -report alone. - -Also note: the readbound haplotype FFI entry point is -`genvarloader::ffi::reconstruct_haplotypes_from_svar2_readbound` (`nm` confirms this is a -distinct symbol from the union-path `reconstruct_haplotypes_from_svar2`); neither -appears with meaningful self-time because they are thin PyO3 wrapper functions - the real -work happens in `gather_haps_readbound` (genoray_core) underneath. This is expected, not a -driver bug. - -### haplotypes (germline K=178, somatic K=38) - -Top 3 self-time symbols, by DSO: - -1. `mapiter_trivial_get` - numpy (`_multiarray_umath...so`) - 10.74% (germline) / 9.76% (somatic) -2. `LONG_subtract_AVX2` / `LONG_add_AVX2` - numpy - 10.04%/9.98% (germline) / 7.74%/7.88% (somatic) -3. `genoray_core::query::gather_haps_readbound` - genoray_core (statically linked into `genvarloader.abi3.so`) - 6.70% (germline) / 0.52% (somatic, see caveat below) - -Rust self-time in the gvl/genoray_core layer (`gather_haps_readbound` + `split_to_flat` -0.60% + `hap_diffs_svar2` 0.55%, germline) sums to ~7.85%, dwarfed by the ~30% combined -numpy fancy-indexing/arithmetic self-time (`mapiter_trivial_get` + `LONG_add_AVX2` + -`LONG_subtract_AVX2`) and by a large unresolved kernel/`[unknown]` component (31-32% DSO -share, germline; 28-29%, somatic) that lines up with sys time dominating user time in -`perf stat` (30.0s sys vs 15.9s user germline; 32.6s sys vs 19.8s user somatic) - almost -certainly page-fault/mmap overhead (reference-genome memmap and/or per-sample-scale array -allocation), not Rust reconstruction. `variants` mode by contrast has ~1-14s sys time, -confirming this kernel-time cost is haplotypes-specific. - -**B1 double-gather hint (gather_haps_readbound + split_to_flat vs diffs-only need):** -PARTIALLY SUPPORTED / INCONCLUSIVE ON THE EXACT RATIO. Within the Rust layer, -`gather_haps_readbound` (6.70%, germline) self-time is ~12x `hap_diffs_svar2`'s (0.55%), -qualitatively consistent with "gathering full haplotype sequences" costing far more than -computing diffs alone - i.e. supports that a diffs-only reconstruction path would be much -cheaper than the current full-gather. But the profiled ratio is far larger than the -brief's "~2x" expectation, and self-time percentages alone can't distinguish "redundant -double work" from "gather legitimately touches more bytes than diffs do" - call-count -instrumentation (not just perf self-time) is needed to pin the exact redundancy factor. -Bigger caveat: for haplotypes, the gvl/genoray Rust layer is a minority contributor -(~8%) of total native time; the numpy fancy-indexing arithmetic (~30%) and kernel/mmap -overhead (~30%) are larger targets by self-time. **Verdict: directionally supports B1 but -does not confirm the specific "~2x" claim; a B1 fix should be scoped alongside (not -instead of) investigating the numpy-level indexing arithmetic and the mmap/page-fault -kernel time, or its net effect on wall-clock will be modest.** - -The somatic cohort's `gather_haps_readbound` self-time (0.52%) is much lower than -germline's (6.70%) despite germline and somatic using the same code path; the somatic -haplotypes capture is instead dominated by `PyUnicode_RichCompare`/`PyObject_RichCompare`/ -`list_contains`/`list_index` (python3.10, ~9.6%/3.3%/3.2%/1.9%/0.8%) - likely -sample-name/list matching overhead scaling with the 16007-sample cohort, worth a separate -look but out of scope for the B1/B2 hints named in this brief. - -### variants (germline K=6547, somatic K=1922) - -Top 3 self-time symbols, by DSO: - -1. `genoray_core::query::gather_haps_readbound` - genoray_core (in `genvarloader.abi3.so`) - 12.85% (germline) / 6.75% (somatic) -2. `genvarloader::svar2::decode_variants_from_split` + `genvarloader::svar2::split_to_flat` - gvl `.so` (`genvarloader::svar2`) - 6.59%+5.05%=11.64% (germline) / 2.73%+~ (somatic, split_to_flat drops out of top-20) -3. `PyArray_Repeat` - numpy - 9.13% (germline) / 4.98% (somatic) - -**B2 allocation-churn hint (split_to_flat/decode_variants_from_split with `_int_malloc`/`SpecFromIter` beneath):** -SUPPORTED. The call graph shows `decode_variants_from_split` as a direct child of -`gather_haps_readbound`'s sibling call (via `pyo3::marker::Python::detach` -> -`genvarloader::ffi::decode_variants_from_svar2_readbound`), with `split_to_flat`, -`svar2_codec::decode_key`, and `alloc::raw_vec::RawVec::grow_one` nested directly -beneath it - `grow_one` is a live allocation-growth event, not incidental. Allocator -self-time symbols appear prominently at the top level in both cohorts: `_int_free` -(4.31% germline / 4.09% somatic), `_int_malloc` (1.71%/2.15%), `malloc`/`__libc_calloc` -(2.06%+1.56% germline / 2.37% somatic), and -` as SpecFromIterNested>::from_iter` (1.55% germline, a Rust -Vec-from-iterator allocation pattern) and `__rustc::__rust_dealloc` (1.42%/1.46%) - all -consistent with the decode/flatten step allocating and freeing many small buffers per -call rather than reusing/pre-sizing them. This is a clear Task B2 target: reduce -allocation count in `split_to_flat`/`decode_variants_from_split` (pre-size `Vec`s, -avoid `from_iter` on hot paths, reuse buffers across calls). From 98873907e92583ca6ec28b3c597daa924111b5a3 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 09:28:32 -0700 Subject: [PATCH 052/108] =?UTF-8?q?perf(svar2):=20decode=5Fvariants=5Ffrom?= =?UTF-8?q?=5Fsplit=20asm=20fix=20=E2=80=94=20hoist=20q=3Dh/ploidy=20div,?= =?UTF-8?q?=20unchecked=20present=5Fbit=20(byte-identical)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two asm-visible wins in decode_variants_from_split's own loop body (not the inlined split_to_flat/merge_hap/decode_alt callees): 1. `q = h / ploidy` was a per-h-iteration integer division (ploidy is a runtime value LLVM can't strength-reduce). Restructured the loop as `for q in 0..n_q { for _hap in 0..ploidy { ... } }` with a plain running `h` counter — visits the exact same (h, q) pairs in the exact same order, so it's byte-identical, and removes the div entirely (cargo asm: div/idiv count in the compiled symbol goes from 2 to 0, and the div-by-zero panic path disappears). Also hoists the per-query ds/de dense-window lookup out of the ploidy-many-times-redundant per-hap reload. 2. The `present_bit` closure's read of `flat.dense_present[bit / 8]` carried a bounds check on every call (once per dense-window entry per hap, i.e. the hottest part of the loop). `split_to_flat` sizes `dense_present` to `total_bits.div_ceil(8)` bytes where `total_bits = dense_present_off[h_count]`, and `merge_hap` only calls `present_bit(k)` for `k` in `0..(de - ds)`, which by construction is exactly `dense_present_off[h+1] - dense_present_off[h]` bits wide — so `bit / 8` is always in bounds. Switched to `get_unchecked` with a safety comment stating the invariant; cargo asm confirms the `cmp`+`jae`-to-panic pair before the byte load is gone. Added a focused test (mixed present/absent bits, ploidy 2, a hap window crossing a byte boundary) since the existing merge/decode test only exercised ploidy=1 / single-byte presence and wouldn't have caught a regression in either change. Verified: `cargo test decode_variants_from_split` (2/2), svar2 pytest suite 32/32, full tree 559 passed / 46 failed / 428 errors (matches pre-existing baseline, no new failures). Same-session perf stat instruction counts (noisy node, cycles unreliable, instructions deterministic): ~53,465M before -> ~53,317M after, a reproducible ~0.28% reduction across repeated runs. --- src/svar2/mod.rs | 149 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 120 insertions(+), 29 deletions(-) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index b2bb4408..eec5d315 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -289,38 +289,62 @@ pub fn decode_variants_from_split( let mut var_off: Vec = Vec::with_capacity(h_count + 1); var_off.push(0); - for h in 0..h_count { - let q = h / ploidy; - let vk_lo = flat.vk_off[h] as usize; - let vk_hi = flat.vk_off[h + 1] as usize; + // `q = h / ploidy` is loop-invariant across each hap's ploidy-many + // iterations; the original per-h division (`h / ploidy`, `ploidy` a + // runtime value LLVM can't strength-reduce) recomputed it on every + // iteration via a full integer division. Iterate `q` in the outer loop + // and track `h` with a plain running counter instead — this visits the + // exact same `(h, q)` pairs in the exact same order (h = 0, 1, 2, ..., + // h_count-1, with q = h/ploidy for each), so it's byte-identical, and it + // also hoists the per-query `ds`/`de` window lookup out of the + // ploidy-many-times-redundant per-hap load. + let mut h = 0usize; + for q in 0..n_q { let ds = flat.dense_range[q * 2] as usize; let de = flat.dense_range[q * 2 + 1] as usize; - let base_bit = flat.dense_present_off[h] as usize; - let present_bit = |k: usize| -> bool { - let bit = base_bit + k; - (flat.dense_present[bit / 8] >> (bit % 8)) & 1 == 1 - }; - - let merged = merge_hap( - &flat.vk_pos, - &flat.vk_key, - vk_lo, - vk_hi, - &flat.dense_pos, - &flat.dense_key, - ds, - de, - present_bit, - ); - - for &(p, key) in &merged { - let (il, alt) = decode_alt(key, lut_bytes, lut_off); - pos.push(p as i32); - ilen.push(il as i32); - alt_bytes.extend_from_slice(&alt); - str_off.push(alt_bytes.len() as i64); + for _hap in 0..ploidy { + let vk_lo = flat.vk_off[h] as usize; + let vk_hi = flat.vk_off[h + 1] as usize; + let base_bit = flat.dense_present_off[h] as usize; + let present_bit = |k: usize| -> bool { + let bit = base_bit + k; + // SAFETY: `merge_hap` only ever calls `present_bit(k)` for `k` + // in `0..(de - ds)` (see its `(ds..de).enumerate()` loop), so + // `bit` ranges over `[base_bit, base_bit + (de - ds))` = + // `[dense_present_off[h], dense_present_off[h + 1])`. Per + // `split_to_flat`, `de - ds` (from `dense_range`, built per + // query `q`) and `dense_present_off[h + 1] - + // dense_present_off[h]` (built per hap `h`, replicated + // `ploidy` times per query) are both exactly the same + // per-query dense-window width, so this range is always `<= + // dense_present_off[h_count] = total_bits`, and + // `dense_present` is sized `total_bits.div_ceil(8)` bytes — + // so `bit / 8` is always in bounds. + (unsafe { *flat.dense_present.get_unchecked(bit / 8) } >> (bit % 8)) & 1 == 1 + }; + + let merged = merge_hap( + &flat.vk_pos, + &flat.vk_key, + vk_lo, + vk_hi, + &flat.dense_pos, + &flat.dense_key, + ds, + de, + present_bit, + ); + + for &(p, key) in &merged { + let (il, alt) = decode_alt(key, lut_bytes, lut_off); + pos.push(p as i32); + ilen.push(il as i32); + alt_bytes.extend_from_slice(&alt); + str_off.push(alt_bytes.len() as i64); + } + var_off.push(pos.len() as i64); + h += 1; } - var_off.push(pos.len() as i64); } VariantsSoa { @@ -587,4 +611,71 @@ mod tests { // Pure-del ALT is empty -> the 3rd variant's [start, end) is [2, 2). assert_eq!(soa.str_off, vec![0, 1, 2, 2]); } + + /// Exercises the two things the asm-inspection pass touched: (1) `q = + /// h / ploidy` computed via an incrementing counter instead of a division + /// (needs `ploidy > 1` so some consecutive haps share a `q`, which the + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's now-`get_unchecked` read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte + /// boundary (hap 1's 5-bit window covers global bits 5..10, i.e. bytes 0 + /// and 1). + #[test] + fn test_decode_variants_from_split_byte_identical_presence_edge() { + use genoray_core::query::KeyRef; + + let k = |b: &[u8]| svar2_codec::encode_alt_inline(b, 0); + + // 2 regions x ploidy 2 = 4 haps, no var_key entries (isolates the + // dense/present-bit path). Region 0's dense window is 3 snp + 2 indel + // (width 5, shared by haps 0 & 1); region 1's is 3 snp + 0 indel + // (width 3, shared by haps 2 & 3). Combined presence bitstream is 16 + // bits = 2 bytes, with hap 1's window (global bits 5..10) crossing + // the byte-0/byte-1 boundary at bit 8. + let br = BatchResultSplit { + n_regions: 2, + n_samples: 1, + ploidy: 2, + vk: vec![], + vk_off: vec![0, 0, 0, 0, 0], + dense_snp: vec![ + KeyRef { position: 10, key: k(b"A") }, + KeyRef { position: 11, key: k(b"C") }, + KeyRef { position: 12, key: k(b"G") }, + KeyRef { position: 50, key: k(b"T") }, + KeyRef { position: 51, key: k(b"A") }, + KeyRef { position: 52, key: k(b"C") }, + ], + dense_snp_range: vec![(0, 3), (3, 6)], + // Per-hap snp-bit widths 3,3,3,3 -> offsets 0,3,6,9,12. Bitstream + // (idx0..11): 1,0,1, 0,1,0, 1,1,0, 0,0,1 -> byte0 = 0b1101_0101 + // (bits0-7: 1,0,1,0,1,0,1,1 -> 1+4+16+64+128=213), byte1 low + // nibble (bits8-11: 0,0,0,1 -> 8). + dense_snp_present: vec![213, 8], + dense_snp_present_off: vec![0, 3, 6, 9, 12], + dense_indel: vec![ + KeyRef { position: 13, key: svar2_codec::encode_pure_del(-2) }, + KeyRef { position: 14, key: svar2_codec::encode_pure_del(-5) }, + ], + dense_indel_range: vec![(0, 2), (2, 2)], + // Per-hap indel-bit widths 2,2,0,0 -> offsets 0,2,4,4,4. + // Bitstream (idx0..3): 1,0, 0,1 -> byte0 = 0b1001 (1+8=9). + dense_indel_present: vec![9], + dense_indel_present_off: vec![0, 2, 4, 4, 4], + }; + + let soa = decode_variants_from_split(&br, &[], &[0]); + + // hap0 (q0): snp present [1,0,1] -> keeps pos10("A"),pos12("G"); + // indel present [1,0] -> keeps pos13(ilen -2). + // hap1 (q0): snp present [0,1,0] -> keeps pos11("C"); + // indel present [0,1] -> keeps pos14(ilen -5). + // hap2 (q1): snp present [1,1,0] -> keeps pos50("T"),pos51("A"). + // hap3 (q1): snp present [0,0,1] -> keeps pos52("C"). + assert_eq!(soa.pos, vec![10, 12, 13, 11, 14, 50, 51, 52]); + assert_eq!(soa.ilen, vec![0, 0, -2, 0, -5, 0, 0, 0]); + assert_eq!(soa.alt_bytes, b"AGCTAC".to_vec()); + assert_eq!(soa.str_off, vec![0, 1, 2, 2, 3, 3, 4, 5, 6]); + assert_eq!(soa.var_off, vec![0, 3, 5, 7, 8]); + } } From 768572326bf9b3a4ffe4f2164202d4128f375ce8 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 09:36:58 -0700 Subject: [PATCH 053/108] perf(svar2): debug_assert guard for decode_variants_from_split get_unchecked invariant --- src/svar2/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index eec5d315..2e13d278 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -308,6 +308,12 @@ pub fn decode_variants_from_split( let base_bit = flat.dense_present_off[h] as usize; let present_bit = |k: usize| -> bool { let bit = base_bit + k; + debug_assert!( + bit / 8 < flat.dense_present.len(), + "decode_variants_from_split present_bit OOB: bit/8={} len={}", + bit / 8, + flat.dense_present.len() + ); // SAFETY: `merge_hap` only ever calls `present_bit(k)` for `k` // in `0..(de - ds)` (see its `(ds..de).enumerate()` loop), so // `bit` ranges over `[base_bit, base_bit + (de - ds))` = From 3a92ec76722282081ad380312bead560267c3399 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 09:57:40 -0700 Subject: [PATCH 054/108] =?UTF-8?q?perf(svar2):=20split=5Fto=5Fflat=20asm?= =?UTF-8?q?=20fix=20=E2=80=94=20hoist=20q=3Dh/ploidy=20division=20out=20of?= =?UTF-8?q?=20hot=20loops=20(byte-identical)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/svar2/mod.rs | 112 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 94 insertions(+), 18 deletions(-) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 2e13d278..4e769857 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -191,37 +191,52 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { dense_range.push(dense_pos.len() as i32); } - let total_bits: usize = (0..h_count) - .map(|h| { - let q = h / ploidy; + // Per query `q`, every one of its `ploidy` haps has the exact same dense + // window width `(se - ss) + (ie - is_)` — so summing per-h widths over + // `0..h_count` (which recomputes `q = h / ploidy` via a runtime-value + // hardware division on *every* h) is equivalent to summing the per-q + // width once and multiplying by `ploidy`. Same total, no per-h division. + let total_bits: usize = (0..n_q) + .map(|q| { let (ss, se) = br.dense_snp_range[q]; let (is_, ie) = br.dense_indel_range[q]; - (se - ss) + (ie - is_) + ((se - ss) + (ie - is_)) * ploidy }) .sum(); let mut dense_present: Vec = vec![0u8; total_bits.div_ceil(8)]; let mut dense_present_off: Vec = Vec::with_capacity(h_count + 1); let mut bit_acc: usize = 0; dense_present_off.push(0); - for h in 0..h_count { - let q = h / ploidy; + // Same hoist as `decode_variants_from_split`: `q = h / ploidy` is + // loop-invariant across each hap's ploidy-many iterations, so iterate `q` + // in the outer loop and track `h` with a plain running counter instead of + // recomputing `h / ploidy` (a runtime division) on every hap. Visits the + // exact same `(h, q)` pairs in the exact same order (h = 0, 1, ..., + // h_count-1, q = h/ploidy for each) — byte-identical — and also hoists the + // per-query `(ss, se)`/`(is_, ie)` window lookup out of the + // ploidy-many-times-redundant per-hap load. + let mut h = 0usize; + for q in 0..n_q { let (ss, se) = br.dense_snp_range[q]; let (is_, ie) = br.dense_indel_range[q]; - let snp_base = br.dense_snp_present_off[h]; - for k in 0..(se - ss) { - if genoray_core::bits_get_bit(&br.dense_snp_present, snp_base + k) { - dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + for _hap in 0..ploidy { + let snp_base = br.dense_snp_present_off[h]; + for k in 0..(se - ss) { + if genoray_core::bits_get_bit(&br.dense_snp_present, snp_base + k) { + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + } + bit_acc += 1; } - bit_acc += 1; - } - let indel_base = br.dense_indel_present_off[h]; - for k in 0..(ie - is_) { - if genoray_core::bits_get_bit(&br.dense_indel_present, indel_base + k) { - dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + let indel_base = br.dense_indel_present_off[h]; + for k in 0..(ie - is_) { + if genoray_core::bits_get_bit(&br.dense_indel_present, indel_base + k) { + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + } + bit_acc += 1; } - bit_acc += 1; + dense_present_off.push(bit_acc as i64); + h += 1; } - dense_present_off.push(bit_acc as i64); } // The reused kernels read `dense_present[bit/8]` for EVERY window entry of // every hap, so the buffer must always be ceil(total_bits/8) bytes — even @@ -570,6 +585,67 @@ mod tests { } } + #[test] + fn test_split_to_flat_ploidy_gt1_reuses_per_query_window_across_haps() { + use genoray_core::query::KeyRef; + + // Regression for the q=h/ploidy hoist: ploidy=2, n_regions=2 (h_count=4) + // so each query's dense_snp/dense_indel window is shared by 2 haps, but + // each hap has its own presence bits at its own `*_present_off` offset. + // This exercises the outer-q/inner-hap loop restructuring (the old code + // recomputed `q = h / ploidy` per hap via a runtime division; the new + // code hoists the per-query window lookup out of the hap loop) — proving + // it doesn't mix up which hap's presence bits attach to which query's + // window. The 12 combined presence bits (3/hap * 4 haps) also cross a + // byte boundary with mixed set/unset bits, like the trailing-zero test, + // but here across queries+haps instead of a single-bit-per-hap window. + let dense_snp: Vec = (0..4) + .map(|i| KeyRef { + position: 10 + i, + key: 200 + i, + }) + .collect(); + let dense_indel: Vec = (0..2) + .map(|i| KeyRef { + position: 50 + i, + key: 500 + i, + }) + .collect(); + + let br = BatchResultSplit { + n_regions: 2, + n_samples: 1, + ploidy: 2, + vk: vec![], + vk_off: vec![0; 5], + dense_snp, + // query 0 owns snp[0..2), query 1 owns snp[2..4) — width 2/query, + // shared by both of that query's haps. + dense_snp_range: vec![(0, 2), (2, 4)], + // hap0 bits(0,1)=(1,0), hap1 bits(2,3)=(0,1), hap2 bits(4,5)=(1,1), + // hap3 bits(6,7)=(0,0) -> byte 0b0011_1001. + dense_snp_present: vec![0b0011_1001], + dense_snp_present_off: vec![0, 2, 4, 6, 8], + dense_indel, + // query 0 owns indel[0..1), query 1 owns indel[1..2) — width 1/query. + dense_indel_range: vec![(0, 1), (1, 2)], + // hap0=1, hap1=0, hap2=1, hap3=1 -> byte 0b0000_1101. + dense_indel_present: vec![0b0000_1101], + dense_indel_present_off: vec![0, 1, 2, 3, 4], + }; + + let flat = split_to_flat(&br); + + // snp-then-indel per query, queries in order. + assert_eq!(flat.dense_pos, vec![10, 11, 50, 12, 13, 51]); + assert_eq!(flat.dense_key, vec![200, 201, 500, 202, 203, 501]); + assert_eq!(flat.dense_range, vec![0, 3, 3, 6]); + + // 4 haps * 3 bits/hap = 12 bits -> 2 bytes, spanning a byte boundary. + assert_eq!(flat.dense_present_off, vec![0, 3, 6, 9, 12]); + assert_eq!(flat.dense_present, vec![0b1101_0101, 0b0000_1001]); + } + #[test] fn test_decode_variants_from_split_merges_and_decodes() { use genoray_core::query::KeyRef; From e4a37c992413a4438bc460886533e7b9cd5f0c80 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 10:08:17 -0700 Subject: [PATCH 055/108] perf(svar2): read-bound getitem optimization results summary --- tmp/svar2_mvp/prof_out/readbound/RESULTS.md | 214 ++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 tmp/svar2_mvp/prof_out/readbound/RESULTS.md diff --git a/tmp/svar2_mvp/prof_out/readbound/RESULTS.md b/tmp/svar2_mvp/prof_out/readbound/RESULTS.md new file mode 100644 index 00000000..9237fa0a --- /dev/null +++ b/tmp/svar2_mvp/prof_out/readbound/RESULTS.md @@ -0,0 +1,214 @@ +# SVAR2 read-bound `Dataset.__getitem__` optimization — consolidated results (2026-07-06) + +Plan: `docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md` +Branches: gvl `svar2-m6b-kernel` (PR #266) + genoray `svar-2`. + +This report consolidates Phase A (baselining) and Phase B (B1-B4 optimization) +of the SVAR2 read-bound `getitem` perf effort. It supersedes any per-task +summary for headline numbers; per-task detail lives in +`.superpowers/sdd/task-{A2,A3,B1,B2,B3,B4-step1,B4a,B4b,B4c}-report.md` and the +condensed ledger in `.superpowers/sdd/progress.md` (section "SDD Progress — +SVAR2 read-bound getitem perf"). + +## 1. Correction to the A3 baseline note (dense_union) + +The original A3 native-baseline commentary is sometimes paraphrased as "the +union oracle (`dense_union()`) is never invoked on the read-bound path." That +overstates what was found and is corrected here: + +- `dense_union()` **is** called on the read-bound path (genoray + `src/query.rs:771`) — it is not absent. +- What A3 actually established (verified against the real call graph, not + grep-only) is that it's **cheap and below the sampling threshold**: it never + shows up with measurable self-time in any of the 4 `perf` captures, and the + disqualifying whole-cohort entry points (`overlap_batch`/`overlap_sample`) + are genuinely absent from the read-bound call chain + (`SparseVar2.find_ranges` → `gather_haps_readbound`/`gather_ranges_readbound`). + `genoray_core::search::SearchTree::build` (1.54% haplotypes_germline, + 0.55% haplotypes_somatic) is the benign per-region `find_ranges` search + phase, not the whole-cohort union oracle. +- **Correct statement for future reference:** *`dense_union()` is called on + the read-bound path but is cheap (below sampling threshold); the + whole-cohort union/oracle path (`overlap_batch`/`overlap_sample`) is what's + absent, not `dense_union()` itself.* + +## 2. Baseline reference numbers (Phase A) + +`perf stat -e instructions,cycles` on the whole profiled process (one +`gvl.write` + `Dataset.open` + K warm `ds[:, :]` calls), frame-pointer build, +committed in `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` — this +file is misleadingly named; **it holds the PRE-B1-B3 (original A3) capture**, +not a post-B1-B3 one (naming preserved as-is per the task record; do not +invert this when reading raw files). + +| combo | K | instructions | cycles | insn/cycle | +|---|---|---|---|---| +| haplotypes_germline | 178 | 136,108,783,319 | 63,090,572,307 | 2.16 | +| variants_germline | 6547 | 434,768,134,979 | 142,981,599,031 | 3.04 | +| haplotypes_somatic | 38 | 211,440,945,021 | 78,077,607,534 | 2.71 | +| variants_somatic | 1922 | 470,732,727,589 | 138,279,404,076 | 3.40 | + +A3 also profiled the Python layer (A2) and native layer (A3) to rank hot +functions. Python-layer hot path (both modes): pure-Python `_ragged_arange_gather` +(and `_2level`) issuing repeated small `numpy.arange`/`ndarray.repeat` calls — +the clearest vectorize/push-to-Rust candidate. Native-layer hot path: haplotypes +dominated by numpy int64 add/sub kernels + kernel/mmap page-fault time (~30% +each), with gvl/genoray Rust only ~8% of self-time; variants dominated by +`gather_haps_readbound` (12.85% germline) + `decode_variants_from_split` + +`split_to_flat` (11.64% combined, germline) + numpy `PyArray_Repeat` (9.13%). + +`tmp/svar2_mvp/prof_out/readbound/native_baseline.md` (also misleadingly +named) holds a **second, POST-B1-B3 re-profile** captured in Task B4 Step 1 to +enumerate the cargo-asm work-list (not a matched-K comparison against the +table above — K differs run-to-run because the harness doesn't pin sample +count): + +| combo | K | instructions | cycles | insn/cycle | +|---|---|---|---|---| +| haplotypes_germline | 191 | 135,051,821,403 | 64,154,539,866 | 2.11 | +| variants_germline | 7143 | 456,803,347,416 | 165,527,068,344 | 2.76 | +| haplotypes_somatic | 37 | 203,147,989,079 | 76,904,197,639 | 2.64 | +| variants_somatic | 1792 | 435,990,647,839 | 139,669,338,811 | 3.12 | + +**Because K is not matched between these two captures (different runs, +different cohort sizes drawn each time), they are not diffed directly as a +before/after number.** The reliable before/after deltas are the per-task, +matched-K, same-session measurements below. + +## 3. Optimizations applied — per-task matched-K deltas + +All changes are **byte-identical**: the svar2 pytest suite +(`pytest tests/dataset -k svar2`, 32 tests reading the real `svar2_mvp` +stores — haplotypes, variants, and tracks parity vs the SVAR1 oracle) stayed +32/32 green through every task, with zero new failures/errors introduced in +the full tree at any step (see §5, Parity). + +Each row below is its own same-session, same-K, git-stash-based before/after +measurement (not one cumulative run) — presented per-task as instructed, +since no single cumulative baseline-vs-final run was captured. + +| Task | Repo | Change | Mode/cohort (K) | Instructions before → after | Δ instructions | Status | +|---|---|---|---|---|---|---| +| **B1** | gvl | Skip redundant pre-reconstruct diffs gather for deterministic haplotype reads (`need_hap_lengths` inverted-default) | haplotypes_germline (K=178) | 136,108,783,319 → 127,529,646,381 | **−6.3%** (cycles −4.7%) | byte-identical | +| **B2** | gvl | Pre-size `split_to_flat` + `decode_variants_from_split` allocations | variants_somatic (K=300) | 160,637,370,802 → 161,125,582,889 | **+0.30% (noise)** | byte-identical, kept anyway | +| **B2** | gvl | (same change, haplotypes) | haplotypes_somatic (K=300) | 895,401,688,408 → 895,815,473,126 | **+0.05% (noise)** | byte-identical, kept anyway | +| **B3** | gvl | De-dup twin ragged reorder-index computation in `_reconstruct_variants` | variants_germline (K=500) | 59,472,976,343 → 58,383,771,799 | **−1.83%** | byte-identical | +| **B4a** | genoray (`svar-2`) | `gather_haps_readbound` asm fix: skip-prefix→slice, bounds-check hoist, inline 2-pointer merge (replaces per-hap `merge_keys` allocation) | variants_germline (K=500) | 58.36e9 → 53.41e9 (avg of 3 runs/side) | **≈−8.5%** (−4.95B instr) | byte-identical (proven + tie-break test) | +| **B4b** | gvl | `decode_variants_from_split` asm fix: hoist per-iter `q = h/ploidy` division; `get_unchecked` on presence bit (proven in-bounds, `debug_assert`-guarded) | variants_germline (K=500) | ≈53,464.9M → ≈53,317.4M (avg of 2 runs/side) | **≈−0.28%** | byte-identical (proven) | +| **B4c** | gvl | `split_to_flat` asm fix: hoist `q = h/ploidy` division out of both hot loops (4 div → 0), no `unsafe` | variants_germline (K=500) | ≈53,296.4M → ≈53,212.6M (avg of 3 runs/side) | **≈−0.15%** | byte-identical | + +**Headline result: B4a (genoray `gather_haps_readbound` asm fix) is the +single largest byte-identical win, ~8.5% instructions on the +variants_germline (K=500) workload** — larger than B1's 6.3% haplotypes win +and an order of magnitude larger than B4b/B4c. B2 is a null-delta-but-kept +scalability improvement (see reasoning below); B1 and B3 are Python-layer +structural/DRY wins with real, smaller, matched-K deltas. + +### Why B2 shows no measurable win here + +`decode_variants_from_split`/`split_to_flat` pre-sizing is a genuine +allocation-count reduction (proven via cargo unit tests and code review), but +the harness's fixed `REGIONS` (3 windows, ~500-1000bp each) keep +`dense_total`/`total_bits`/`cap` tiny per call, so few `Vec` reallocations are +actually eliminated per call. The eliminated cost is a rounding error against +the multi-billion-instruction total dominated by write+open setup and +Python/numpy glue at this harness's scale. The optimization is kept as a +zero-risk scalability improvement: its payoff should scale with region +size/variant density, which this benchmark doesn't exercise, and both targets +independently rank high in the A3/B4-Step-1 native profiles (`split_to_flat` +5.05-5.21% self-time, `decode_variants_from_split` 6.59-5.43% self-time). + +## 4. Profiled-but-deferred candidates + +Task B4 Step 1 re-profiled the native layer after B1-B3 and enumerated every +gvl/genoray Rust symbol with ≥1.5% self-time in any mode (6 functions). The +user approved a scoped **sequential** asm pass on the top 3 by ROI (B4a/b/c +above, all done). The remaining 3 were profiled and explicitly **deferred**, +not forgotten, as a controller-approved scope decision (do only the top-3 asm +targets this round; leave the rest for a follow-up pass): + +| Symbol | Repo | Max self-time | Reason deferred | +|---|---|---|---| +| `svar2_codec::decode_key` | genoray `svar2-codec/src/lib.rs:237` | 2.33% (variants_germline) | Below the top-3 ROI cutoff; scope decision to do only the top-3 sequentially this round | +| `genoray_core::spine::merge_keys` | genoray `src/spine.rs:63` | 2.31% (variants_somatic) | Same; note B4a's inline merge in `gather_haps_readbound` already eliminated one call site of this pattern, but the standalone `merge_keys` function itself (used elsewhere) was not asm-optimized | +| `genoray_core::search::SearchTree::build` | genoray `src/search.rs:93` | 1.54% (haplotypes_germline) | Same; also note this is the **benign** per-region `find_ranges` search phase, not the whole-cohort union oracle (see §1) — deferring it is a pure perf scope call, not a correctness concern | + +## 5. Parity + +- **svar2 suite: 32/32 byte-identical, held through every task** (B1, B2, B3, + B4a, B4b, B4c) — covers haplotypes, variants, and tracks paths against the + SVAR1 oracle (`pytest tests/dataset -k svar2`). +- **Full tree: NOT fully green, but no regressions.** `pixi run -e dev gen` + (ground-truth fixture generation) is **broken pre-existing to this plan**: + `VcfBuilder.__init__() got an unexpected keyword argument 'fileformat'` + (`tests/_builders/case.py:324`), a vcfixture (unpinned, `>=0.5.0`) API-drift + issue unrelated to svar2 or this perf work. This produces a fixed ~428 + errors / 46 failed baseline in `pytest tests -q` that held constant, + unchanged, through every task in this plan. The passed count held at 559 + (+1 from B1's added micro-test) with **zero new failures or errors** + introduced at any step — confirmed by identical failing-test-ID sets with + each change stashed vs applied. +- **This is a known issue / CI blocker requiring a separate fix** + (pin vcfixture or update the `VcfBuilder` call site at + `tests/_builders/case.py:324`), tracked here for visibility. It is **out of + scope** for this perf plan (dependency/test-infra issue, not a code + regression) and should be filed/fixed separately before branch CI can go + green. + +## 6. Deferred features (out of scope this round, by design) + +- **Tracks**: out of scope for this round's profiling and optimization. + `Svar2Haps.get_haps_and_shifts`'s tracks caller still runs the diffs kernel + via the `need_hap_lengths=True` default (B1's inverted default preserves + this — only the pure-haplotypes `__call__` entry point opts out with + `need_hap_lengths=False`). The B1 double-gather is therefore **unaddressed + for tracks by design**; tracks parity (`test_svar2_tracks_match_svar1*`) + was verified unaffected/green throughout, but no tracks-path perf work was + done. +- **Variant-windows**: guarded with `NotImplementedError` in `Svar2Haps` — + cannot be profiled or optimized until implemented. Deferred until that + feature lands. + +## 7. API / format / docs reconciliation (B5 Step 2) + +**Read-path internals only; no API/format/doc-surface change.** + +- `need_hap_lengths` (B1) is an internal parameter on + `Svar2Haps.get_haps_and_shifts`, not a public API — it is not exported and + is not part of `genvarloader.__all__`. +- All B2/B3/B4a/B4b/B4c changes are Rust/Python read-path internals + (allocation pre-sizing, asm-level instruction elimination, a de-duplicated + index computation) with no change to any public symbol, function signature + exposed to users, on-disk dataset format, or CLI/bcftools/plink2 + preprocessing requirement. +- No genoray kernel FFI signature changed (`gather_haps_readbound`, + `decode_variants_from_split`, `split_to_flat` all keep their existing + signatures — only their bodies changed). +- Per the "Maintaining the `genvarloader` skill" and "Docs audit" rules in + `CLAUDE.md`: since nothing in `__all__`, `gvl.write`, `Dataset.open`, or any + `Dataset.with_*` signature/default changed, **no update to + `skills/genvarloader/SKILL.md`, `docs/source/api.md`, or any other prose doc + is required**, and none was made. No genoray docs (`docs/roadmap/svar-2.md`) + needed updating either, since no genoray kernel signature changed — the + genoray commits below are pure asm/allocation fixes on existing functions. + +## 8. Two-repo commit list + +**gvl** (`svar2-m6b-kernel`, PR #266), in order: +- `56c7b36` — B1: skip redundant pre-reconstruct gather for deterministic haplotype reads +- `a297d24` — B2: pre-size split_to_flat + decode_variants_from_split allocations +- `ecfc057` — B3: compute the pos/ilen ragged reorder index once in variants decode +- `2bdee38` — B4 Step 1: re-profile native layer after B1-B3; enumerate cargo-asm work-list +- `1e894d2` — B4b: decode_variants_from_split asm fix (byte-identical) +- `85a8925` — B4b follow-up: debug_assert guard for the get_unchecked invariant +- `e99a0d9` — B4c: split_to_flat asm fix — hoist q=h/ploidy division out of hot loops (byte-identical) + +**genoray** (`svar-2`): +- `a7c32b3` — B4a: gather_haps_readbound asm fix — kill skip/take waste, elide bounds checks, inline the per-hap merge (byte-identical) +- `69b3c97` — B4a follow-up: test covering gather_haps_readbound same-position SNP+indel tie; cross-ref merge_keys + +## 9. Sources + +Per-task detail, TDD evidence, and full profiler output: `.superpowers/sdd/task-{A2,A3,B1,B2,B3,B4-step1,B4a,B4b,B4c}-report.md`. +Condensed ledger: `.superpowers/sdd/progress.md` (section "SDD Progress — SVAR2 read-bound getitem perf"). +Raw captures: `tmp/svar2_mvp/prof_out/readbound/{python_baseline.md,native_after_b1b3.md,native_baseline.md,asm_targets.md}`. From 5a2283f12ac0fac00b39b5eb56c4283914226bc4 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 10:23:47 -0700 Subject: [PATCH 056/108] docs(svar2): correct stale/imprecise comments in split_to_flat + get_haps_and_shifts skip branch --- python/genvarloader/_dataset/_svar2_haps.py | 7 ++++++- src/svar2/mod.rs | 10 ++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 62b18ec6..ed4f7f0a 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -392,7 +392,12 @@ def get_haps_and_shifts( hap_lengths = (lengths[:, None] + diffs).astype(np.int32) else: - diffs = np.zeros((b, P), np.int32) # placeholder (unused downstream) + diffs = np.zeros( + (b, P), np.int32 + ) # diffs discarded by __call__ (only caller reaching this branch) + # hap_lengths below still feeds g_total -> should_parallelize (a perf + # heuristic only; never affects output bytes), so the ref-length + # placeholder is byte-identical-safe. hap_lengths = np.broadcast_to( lengths[:, None].astype(np.int32), (b, P) ).copy() diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 4e769857..c4fb7777 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -238,12 +238,10 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { h += 1; } } - // The reused kernels read `dense_present[bit/8]` for EVERY window entry of - // every hap, so the buffer must always be ceil(total_bits/8) bytes — even - // when the last hap's window bits are all zero (the in-loop grow-on-set - // above only extends up to the highest SET bit, leaving a trailing all-zero - // byte unallocated → OOB panic downstream). genoray byte-sizes its presence - // arrays via div_ceil unconditionally; match that here. + // `dense_present` was pre-sized to `total_bits.div_ceil(8)` above, and the + // fill loop increments `bit_acc` exactly `total_bits` times, so this resize + // is a defensive no-op (kept to document the ceil-byte invariant the reused + // kernels rely on: they read `dense_present[bit/8]` for every window entry). dense_present.resize(bit_acc.div_ceil(8), 0); FlatChannels { From fb29166cdc453dc34df40638a913c0c7bb4f01f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:56:37 +0000 Subject: [PATCH 057/108] chore(pre-commit): auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tmp/svar2_mvp/prof_getitem.py | 25 +++++++++++++------ .../prof_out/readbound/native_after_b1b3.md | 24 +++++++++++++++--- .../prof_out/readbound/native_baseline.md | 24 +++++++++++++++--- .../prof_out/readbound/python_baseline.md | 24 +++++++++++++++--- tmp/svar2_mvp/prof_python.py | 2 ++ 5 files changed, 79 insertions(+), 20 deletions(-) diff --git a/tmp/svar2_mvp/prof_getitem.py b/tmp/svar2_mvp/prof_getitem.py index 22b5c133..e212eead 100644 --- a/tmp/svar2_mvp/prof_getitem.py +++ b/tmp/svar2_mvp/prof_getitem.py @@ -7,6 +7,7 @@ gvl.write + Dataset.open run ONCE (we profile the READ, not the write). Prints per_call_s over K warm calls. Tracks mode is out of scope; variant-windows is guarded NotImplementedError in Svar2Haps and cannot be profiled yet.""" + import sys import time from pathlib import Path @@ -21,11 +22,13 @@ def _bed(): - return pl.DataFrame({ - "chrom": [CHROM] * len(REGIONS), - "chromStart": [s for s, _ in REGIONS], - "chromEnd": [e for _, e in REGIONS], - }) + return pl.DataFrame( + { + "chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS], + } + ) def make_call(mode, cohort): @@ -38,10 +41,16 @@ def make_call(mode, cohort): ds_path = WORK / f"{cohort}_{mode}.gvl" WORK.mkdir(parents=True, exist_ok=True) - gvl.write(ds_path, _bed(), variants=SparseVar2(f"{prefix}.svar2"), - samples=None, max_jitter=0, overwrite=True) + gvl.write( + ds_path, + _bed(), + variants=SparseVar2(f"{prefix}.svar2"), + samples=None, + max_jitter=0, + overwrite=True, + ) ds = gvl.Dataset.open(ds_path, reference=REF) - view = ds.with_seqs(mode) # "haplotypes" or "variants" + view = ds.with_seqs(mode) # "haplotypes" or "variants" R = len(REGIONS) diff --git a/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md b/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md index 7f39c035..ea292b71 100644 --- a/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md +++ b/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md @@ -4,7 +4,11 @@ 2026-07-06 00:45:55.272 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl 2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. 2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - 0%| | 0/3 [00:00 Date: Mon, 6 Jul 2026 12:51:44 -0700 Subject: [PATCH 058/108] fix(write): copy variant index across filesystems on EXDEV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _write_from_vcf/_write_from_pgen hardlinked the genoray index into /genotypes/variants.arrow via Path.hardlink_to, which raises OSError(EXDEV) when the dataset output and the variant source live on different filesystems (e.g. writing to /tmp from a networked mount) — the write path then errored out instead of producing a dataset. Add _link_or_copy(src, dst): attempt the zero-copy hardlink, fall back to shutil.copyfile only on EXDEV. Same-filesystem writes keep the hardlink (byte-identical), and cross-device writes now succeed. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_write.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 4d95b98b..f05f9614 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1,5 +1,7 @@ +import errno import gc import json +import shutil import warnings from collections.abc import Callable, Iterator, Sequence from importlib.metadata import version @@ -629,6 +631,23 @@ def _reject_unsupported_variants(index: pl.DataFrame, source: str) -> None: ) +def _link_or_copy(src: Path, dst: Path) -> None: + """Hardlink ``src`` → ``dst`` to avoid duplicating the (possibly large) variant + index, falling back to a copy when the two live on different filesystems. + + ``Path.hardlink_to`` raises ``OSError(EXDEV)`` across filesystem boundaries + (e.g. writing a dataset under ``/tmp`` from a source on a network mount). The + copy produces byte-identical content, so same-filesystem writes keep the + zero-copy hardlink and cross-device writes still succeed. + """ + try: + dst.hardlink_to(src) + except OSError as e: + if e.errno != errno.EXDEV: + raise + shutil.copyfile(src, dst) + + def _write_from_vcf( path: Path, bed: pl.DataFrame, vcf: VCF, max_mem: int, extend_to_length: bool ): @@ -641,7 +660,7 @@ def _write_from_vcf( out_dir = path / "genotypes" out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "variants.arrow").hardlink_to(vcf._index_path()) + _link_or_copy(vcf._index_path(), out_dir / "variants.arrow") return _write_phased_chunked( out_dir, bed, _vcf_region_chunks(bed, vcf, max_mem, extend_to_length) @@ -820,7 +839,7 @@ def _write_from_pgen( out_dir = path / "genotypes" out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "variants.arrow").hardlink_to(pgen._index_path()) + _link_or_copy(pgen._index_path(), out_dir / "variants.arrow") return _write_phased_chunked( out_dir, bed, _pgen_region_chunks(bed, pgen, max_mem, extend_to_length) From e3c26b4a14704b984fce59112b4ff24a262d3052 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 12:52:00 -0700 Subject: [PATCH 059/108] test(build): adopt vcfixture 0.6 API; fix env LD_LIBRARY_PATH on linux vcfixture 0.6 dropped the VcfBuilder(fileformat=) str kwarg (use version=VcfVersion; VCFv4.0 is no longer offered, so pin to V4_1) and now requires typed Allele instances for ALT (alt=[Seq("C")], not alt=["C"]). Update the session/case fixture builders so `pixi run -e dev gen` and the fixture-backed tests run again. Bump the pin to >=0.6.0,<0.7 and unify vcfixture to 0.6.0 across environments in the lock. Add [target.linux-64.activation.env] prepending $CONDA_PREFIX/lib to LD_LIBRARY_PATH so the env's libstdc++/libpython win over HPC-module system libs; fixes an llvmlite GLIBCXX_3.4.30 crash in spawned multiprocessing workers and libpython loading in standalone cargo-test binaries. Co-Authored-By: Claude Opus 4.8 --- pixi.lock | 54 +++++++++++++++--------------------- pixi.toml | 11 ++++++-- tests/_builders/case.py | 36 ++++++++++++------------ tests/_builders/test_case.py | 8 ++++-- 4 files changed, 55 insertions(+), 54 deletions(-) diff --git a/pixi.lock b/pixi.lock index d3c8746b..be4ac2a9 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1033,6 +1033,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl @@ -1069,7 +1070,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl @@ -1277,6 +1277,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl @@ -1310,7 +1311,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -1572,6 +1572,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -1617,7 +1618,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl @@ -1838,6 +1838,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl @@ -1871,7 +1872,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -1986,7 +1986,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl @@ -1996,6 +1995,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/39/f8/a9af1b0e78a4d2a3974eac8289d97f83526346571f5ef3edda13bcff262d/charset_normalizer-3.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl @@ -2094,9 +2094,9 @@ environments: - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/55/f7590fd3a3b9c80b7106876bc868827304462cbf9b40b0f3d664c04c4e67/charset_normalizer-3.4.8-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - - pypi: https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl @@ -3321,6 +3321,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/0b/0615dbedb98f5b32a35a53290fbdc6e22306968109278d7e58df82d7a9f6/numba-0.65.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/bb/3d3e0a9350a81f3cf38e0bf69c4ad0c17e2f4bfe142c98202294fe0cfa92/selectolax-0.4.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3340,7 +3341,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl @@ -3486,6 +3486,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5a/b0/a4ffc4ae74d2d822200dcc46898987d8eb6032d1e2b219cae39da6f5cbcc/pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3503,7 +3504,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/96/b3/650500c2eab4534d98e9166f4298e0f3c69c742afdf24e6eabccd1f16ad8/numba-0.65.1-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -3718,6 +3718,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -3738,7 +3739,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl @@ -3886,6 +3886,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3902,7 +3903,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -4117,6 +4117,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/42/2b2b00b0df934353c8e3ebfb2be68f63d7aa6dfe5eb5d5f6427b092e0814/awkward_cpp-52-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/e6/01ada46425bd120b36e7c2f6fe74a980de6912d416f07c9c85b22fadd515/pybigwig-0.3.25-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/5f/dd/0c6a5a36ec132665f85e5e33f0480b58cf5aa8af8fbe1d5971410d789558/ncls-0.0.70.tar.gz @@ -4137,7 +4138,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/22/b8d873f6466b20aa563fc9b33acd48dec89a07803ddaa2f1c8ca1cd33126/numba-0.65.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -4288,6 +4288,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/dd/0c6a5a36ec132665f85e5e33f0480b58cf5aa8af8fbe1d5971410d789558/ncls-0.0.70.tar.gz - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -4308,7 +4309,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl @@ -12548,6 +12548,11 @@ packages: - polars-config-meta[polars]>=0.3.2 - attrs requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/1e/55/f7590fd3a3b9c80b7106876bc868827304462cbf9b40b0f3d664c04c4e67/charset_normalizer-3.4.8-cp310-cp310-macosx_10_9_universal2.whl + name: charset-normalizer + version: 3.4.8 + sha256: ad320cccde0bbe430fe3a0e5ef3f17b16ef7cfe496603d7b3a153aa204e093e5 + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl name: idna version: '3.18' @@ -12571,11 +12576,6 @@ packages: requires_dist: - numpy>=1.19.3 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: charset-normalizer - version: 3.4.7 - sha256: cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 - requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: pysam version: 0.24.0 @@ -12649,11 +12649,6 @@ packages: - jsonargparse[signatures]>=4.38.0 ; extra == 'cli' - tomlkit ; extra == 'cli' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl - name: charset-normalizer - version: 3.4.7 - sha256: cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d - requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl name: sphinxcontrib-qthelp version: 2.0.0 @@ -13061,6 +13056,11 @@ packages: version: 0.5.2 sha256: b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/39/f8/a9af1b0e78a4d2a3974eac8289d97f83526346571f5ef3edda13bcff262d/charset_normalizer-3.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.8 + sha256: 511851c35e4ec8f4be773acd2c0222a7ec6b63971fea8988103632896f174b4b + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl name: zipp version: 4.1.0 @@ -14984,16 +14984,6 @@ packages: - coloredlogs - click requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - name: vcfixture - version: 0.5.0 - sha256: 9d9823022e11e8d20b3191046627dbaf0be21a7db578f49636089fdb70cbe3a7 - requires_dist: - - numpy>=1.24 - - pysam>=0.22 - - hypothesis>=6.100 - - typing-extensions>=4.4 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl name: pyzmq version: 27.1.0 diff --git a/pixi.toml b/pixi.toml index 68ce2992..a68966de 100644 --- a/pixi.toml +++ b/pixi.toml @@ -19,6 +19,13 @@ splice = { features = ["notebook", "basenji2", "py310", "splice"] } [target.osx-arm64.activation.env] RUSTFLAGS = "-C link-arg=-rpath -C link-arg=$CONDA_PREFIX/lib" +# Prepend the env's lib dir so its libstdc++/libpython win over any system libs +# an HPC module may have placed earlier on LD_LIBRARY_PATH (e.g. an older gcc +# libstdc++ lacking GLIBCXX_3.4.30, which breaks llvmlite in spawned workers and +# libpython loading in `cargo test` binaries). +[target.linux-64.activation.env] +LD_LIBRARY_PATH = "$CONDA_PREFIX/lib:$LD_LIBRARY_PATH" + [dependencies] python = "<3.14" uv = "*" @@ -51,7 +58,7 @@ plink2 = "*" genvarloader = { path = ".", editable = true, extras = ["cli"] } # Test-only: needed by the property/fixture tests across the whole py3xx matrix # (not a genvarloader runtime dep, so it must live here, not just in py310). -vcfixture = ">=0.5.0" +vcfixture = ">=0.6.0,<0.7" [feature.docs.dependencies] pandoc = "*" @@ -118,7 +125,7 @@ pooch = "*" awkward = "*" pydantic = ">=2,<3" hypothesis = "*" -vcfixture = ">=0.5.0" +vcfixture = ">=0.6.0,<0.7" filelock = "*" [feature.py311.dependencies] diff --git a/tests/_builders/case.py b/tests/_builders/case.py index 3298ee1e..5b96f6ee 100644 --- a/tests/_builders/case.py +++ b/tests/_builders/case.py @@ -14,7 +14,7 @@ import numpy as np import polars as pl -from vcfixture import Number, ReferenceBuilder, Type, VcfBuilder +from vcfixture import Number, ReferenceBuilder, Seq, Type, VcfBuilder, VcfVersion SEQ_LEN = 20 @@ -317,11 +317,11 @@ def session_reference(): def session_document(spec): """Fixed standardized source document (relabel of the Phase-1 source VCF). - Replaces Phase-1 ``_synthetic.build_source_vcf``. VCFv4.0 is used so the + Replaces Phase-1 ``_synthetic.build_source_vcf``. VCFv4.1 is used so the Number=. INFO fields parse under the noodles VCF reader SparseVar relies on. """ contigs = list(_SESSION_CONTIGS) - b = VcfBuilder(samples=["s0", "s1", "s2"], contigs=contigs, fileformat="VCFv4.0") + b = VcfBuilder(samples=["s0", "s1", "s2"], contigs=contigs, version=VcfVersion.V4_1) b.info("NS", Number.ONE, Type.INTEGER) b.info("AN", Number.ONE, Type.INTEGER) b.info("AC", Number.DOT, Type.INTEGER) @@ -339,23 +339,25 @@ def session_document(spec): b.filter("s50", "Less than 50% of samples have data") # chr1 block (relabeled from chr19). - b.record("chr1", 111, ref="N", alt=["C"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1010696, ref="GAGA", alt=["G"], gt=["1|0", "0|0", "0/0"]) - b.record("chr1", 1010696, ref="GAGACGG", alt=["G"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1010696, ref="GAGACGGGGCC", alt=["G"], gt=["0|1", "1|1", "0/0"]) - b.record("chr1", 1110696, ref="A", alt=["TTT"], gt=["0|1", "1|1", "0/0"]) - b.record("chr1", 1110696, ref="A", alt=["G"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1210696, ref="C", alt=["G"], gt=["1|.", "0/1", "1|1"]) - b.record("chr1", 1210696, ref="C", alt=["G"], gt=[".|1", "0|0", "0/0"]) - b.record("chr1", 1210697, ref="T", alt=["G"], gt=["0/0", "1|0", "0/1"]) - b.record("chr1", 1210697, ref="T", alt=["A"], gt=["0/0", "1|0", "0/1"]) + b.record("chr1", 111, ref="N", alt=[Seq("C")], gt=["0|0", "0|0", "0/1"]) + b.record("chr1", 1010696, ref="GAGA", alt=[Seq("G")], gt=["1|0", "0|0", "0/0"]) + b.record("chr1", 1010696, ref="GAGACGG", alt=[Seq("G")], gt=["0|0", "0|0", "0/1"]) + b.record( + "chr1", 1010696, ref="GAGACGGGGCC", alt=[Seq("G")], gt=["0|1", "1|1", "0/0"] + ) + b.record("chr1", 1110696, ref="A", alt=[Seq("TTT")], gt=["0|1", "1|1", "0/0"]) + b.record("chr1", 1110696, ref="A", alt=[Seq("G")], gt=["0|0", "0|0", "0/1"]) + b.record("chr1", 1210696, ref="C", alt=[Seq("G")], gt=["1|.", "0/1", "1|1"]) + b.record("chr1", 1210696, ref="C", alt=[Seq("G")], gt=[".|1", "0|0", "0/0"]) + b.record("chr1", 1210697, ref="T", alt=[Seq("G")], gt=["0/0", "1|0", "0/1"]) + b.record("chr1", 1210697, ref="T", alt=[Seq("A")], gt=["0/0", "1|0", "0/1"]) # chr2 block (relabeled from chr20) — carries INFO/IDs/FILTERs. b.record( "chr2", 14370, ref="N", - alt=["A"], + alt=[Seq("A")], ids=["rs6054257"], qual=29.0, filter=(), @@ -366,7 +368,7 @@ def session_document(spec): "chr2", 17330, ref="N", - alt=["A"], + alt=[Seq("A")], qual=3.0, filter=["q10"], gt=["0|0", "0|1", "0/0"], @@ -376,7 +378,7 @@ def session_document(spec): "chr2", 1110696, ref="G", - alt=["A", "T"], + alt=[Seq("A"), Seq("T")], ids=["rs6040355"], qual=67.0, filter=(), @@ -387,7 +389,7 @@ def session_document(spec): "chr2", 1234567, ref="A", - alt=["GA", "AC"], + alt=[Seq("GA"), Seq("AC")], ids=["microsat1"], qual=50.0, filter=(), diff --git a/tests/_builders/test_case.py b/tests/_builders/test_case.py index 595de679..57475679 100644 --- a/tests/_builders/test_case.py +++ b/tests/_builders/test_case.py @@ -6,7 +6,7 @@ import genvarloader as gvl import pysam -from vcfixture import ReferenceBuilder, VcfBuilder +from vcfixture import ReferenceBuilder, Seq, VcfBuilder, VcfVersion def _tiny_spec_and_doc(): @@ -17,9 +17,11 @@ def _tiny_spec_and_doc(): .set_base("chr1", 99, "A") # 0-based; REF for the record below .build() ) - b = VcfBuilder(samples=["s0", "s1"], contigs=[("chr1", 200)], fileformat="VCFv4.0") + b = VcfBuilder( + samples=["s0", "s1"], contigs=[("chr1", 200)], version=VcfVersion.V4_1 + ) b.fmt("GT") - b.record("chr1", 100, ref="A", alt=["C"], gt=["0|1", "1|1"]) # 1-based pos + b.record("chr1", 100, ref="A", alt=[Seq("C")], gt=["0|1", "1|1"]) # 1-based pos return ref, b.build() From 63b8b8700acb01b78c9a2de3f3dd5cb699f2072e Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 15:27:43 -0700 Subject: [PATCH 060/108] perf(svar2): skip identity row-reorder for single-contig getitem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Svar2Haps groups read-bound queries by contig, reconstructs per group, then permutes the grouped output back to global (b, P) order. For a single contig group that permutation is the identity, so the O(total bytes) numpy arange+repeat reorder (and the 1-element concatenate) is pure waste — it dominated the call (~96%) on single-contig reads. Add a single-group fast-path to _assemble_haps, _reconstruct_variants, and realign_track_block. Byte-identical; drops haplotypes from 12-25x slower than svar1 to parity. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_haps.py | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index ed4f7f0a..8a9b54c8 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -547,6 +547,10 @@ def realign_track_block( cat_lens.append(np.diff(np.asarray(out_off, np.int64))) cat_query_order.append(qsel) + # Single contig group: grouped order already equals global (b, P) order, + # so the reorder is the identity — return the sole group's buffer directly. + if len(cat_data) == 1: + return cat_data[0], lengths_to_offsets(cat_lens[0], np.int64) data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.float32) lens = np.concatenate(cat_lens) if cat_lens else np.zeros(0, np.int64) grouped_off = lengths_to_offsets(lens, np.int64) @@ -596,6 +600,21 @@ def _reconstruct_variants( cat_var_bytelen.append(np.diff(str_off)) cat_query_order.append(qsel) + # Single contig group: grouped order already equals global (b, P) order, + # so the reorder is the identity and every concatenate is a 1-element no-op. + # Skip both (the numpy reorder otherwise dominates single-contig reads). + if len(cat_pos) == 1: + shape = (b, P, None) + var_off_g = lengths_to_offsets(cat_var_lens[0], np.int64) + str_off_g = lengths_to_offsets(cat_var_bytelen[0], np.int64) + return RaggedVariants( + alt=Ragged.from_offsets( + cat_alt[0].view("S1"), shape, var_off_g, str_offsets=str_off_g + ), + start=Ragged.from_offsets(cat_pos[0], shape, var_off_g), + ilen=Ragged.from_offsets(cat_ilen[0], shape, var_off_g), + ) + # Concatenate grouped outputs, then permute hap-rows back to global order. var_lens = ( np.concatenate(cat_var_lens) if cat_var_lens else np.zeros(0, np.int64) @@ -755,6 +774,18 @@ def _assemble_haps( b: int, P: int, ) -> Ragged[np.bytes_]: + # Single contig group: grouped hap-row order already equals global (b, P) + # order (the sole group's qsel is [0..b-1]), so the reorder is the identity + # and the concatenate is a 1-element no-op. Skip both — this is the common + # single-contig read, where the O(total_bytes) numpy reorder otherwise + # dominates (~96% of the call). Byte-identical to the general path. + if len(cat_data) == 1: + out_data = cat_data[0] + out_off = lengths_to_offsets(cat_hap_lens[0], np.int64) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, (b, P, None), out_off).view("S1"), + ) data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.uint8) hap_lens = ( np.concatenate(cat_hap_lens) if cat_hap_lens else np.zeros(0, np.int64) From b84a57d4c5fe6fa20f3e9fc9e012fc73248689b0 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 15:27:43 -0700 Subject: [PATCH 061/108] perf(svar2): fuse variant decode, stream-merge from gather MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decode_variants_from_split now decodes straight from the gather's BatchResultSplit via a streaming 3-way merge of the already-sorted var_key / present-dense-snp / present-dense-indel runs (tie order var_key ~2.6-3.7x vs svar1. Also documents the falsified dense-decode pre-cache (regressed: dense-SNP decode is a 2-bit no-op, cheaper than a cached slice-copy). Co-Authored-By: Claude Opus 4.8 --- src/svar2/mod.rs | 118 +++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index c4fb7777..d5aa77dd 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -284,15 +284,26 @@ pub fn decode_variants_from_split( lut_bytes: &[u8], lut_off: &[i64], ) -> VariantsSoa { - let flat = split_to_flat(br); + // Fused decode straight from the split gather result: NO `split_to_flat` + // marshaling copy, and NO per-hap `merge_hap` Vec+sort. The three per-hap + // runs (var_key, present dense-snp, present dense-indel) are each already + // position-sorted, so we stream a 3-way merge, decoding each key as we go. + // + // NOTE: dense keys are hap-independent, so pre-decoding them once per query + // and copying across haps was tried (to kill the apparent per-(hap,variant) + // re-decode). It REGRESSED — dense-SNP `decode_alt` is a 2-bit->1-byte no-op, + // cheaper than the pre-decoded slice-copy + its allocation, so inline decode + // wins. The real variant cost is the gather bit-extraction + the unavoidable + // per-hap alt-byte emit, not the decode. (Measured 2026-07-06.) + use genoray_core::bits_get_bit; let ploidy = br.ploidy; let n_q = br.n_regions; let h_count = n_q * ploidy; - // Upper bound on total merged variants across all haps: every vk entry plus - // every dense window entry (present or not). Over-reserving is harmless. - let vk_total = flat.vk_off[h_count] as usize; - let dense_bits = flat.dense_present_off[h_count] as usize; + // Upper bound on total merged variants: every vk entry plus every dense + // present bit (over-reserving is harmless). + let vk_total = br.vk_off[h_count]; + let dense_bits = br.dense_snp_present_off[h_count] + br.dense_indel_present_off[h_count]; let cap = vk_total + dense_bits; let mut pos: Vec = Vec::with_capacity(cap); let mut ilen: Vec = Vec::with_capacity(cap); @@ -302,59 +313,56 @@ pub fn decode_variants_from_split( let mut var_off: Vec = Vec::with_capacity(h_count + 1); var_off.push(0); - // `q = h / ploidy` is loop-invariant across each hap's ploidy-many - // iterations; the original per-h division (`h / ploidy`, `ploidy` a - // runtime value LLVM can't strength-reduce) recomputed it on every - // iteration via a full integer division. Iterate `q` in the outer loop - // and track `h` with a plain running counter instead — this visits the - // exact same `(h, q)` pairs in the exact same order (h = 0, 1, 2, ..., - // h_count-1, with q = h/ploidy for each), so it's byte-identical, and it - // also hoists the per-query `ds`/`de` window lookup out of the - // ploidy-many-times-redundant per-hap load. let mut h = 0usize; for q in 0..n_q { - let ds = flat.dense_range[q * 2] as usize; - let de = flat.dense_range[q * 2 + 1] as usize; + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; for _hap in 0..ploidy { - let vk_lo = flat.vk_off[h] as usize; - let vk_hi = flat.vk_off[h + 1] as usize; - let base_bit = flat.dense_present_off[h] as usize; - let present_bit = |k: usize| -> bool { - let bit = base_bit + k; - debug_assert!( - bit / 8 < flat.dense_present.len(), - "decode_variants_from_split present_bit OOB: bit/8={} len={}", - bit / 8, - flat.dense_present.len() - ); - // SAFETY: `merge_hap` only ever calls `present_bit(k)` for `k` - // in `0..(de - ds)` (see its `(ds..de).enumerate()` loop), so - // `bit` ranges over `[base_bit, base_bit + (de - ds))` = - // `[dense_present_off[h], dense_present_off[h + 1])`. Per - // `split_to_flat`, `de - ds` (from `dense_range`, built per - // query `q`) and `dense_present_off[h + 1] - - // dense_present_off[h]` (built per hap `h`, replicated - // `ploidy` times per query) are both exactly the same - // per-query dense-window width, so this range is always `<= - // dense_present_off[h_count] = total_bits`, and - // `dense_present` is sized `total_bits.div_ceil(8)` bytes — - // so `bit / 8` is always in bounds. - (unsafe { *flat.dense_present.get_unchecked(bit / 8) } >> (bit % 8)) & 1 == 1 - }; - - let merged = merge_hap( - &flat.vk_pos, - &flat.vk_key, - vk_lo, - vk_hi, - &flat.dense_pos, - &flat.dense_key, - ds, - de, - present_bit, - ); - - for &(p, key) in &merged { + let vk_lo = br.vk_off[h]; + let vk_hi = br.vk_off[h + 1]; + let snp_base = br.dense_snp_present_off[h]; + let indel_base = br.dense_indel_present_off[h]; + + // 3-way merge, position-sorted. On equal positions the priority is + // var_key < dense-snp < dense-indel, exactly the stable-sort tie + // order of the previous collect-then-sort `merge_hap` (var_key + // pushed first, then dense-snp, then dense-indel). Byte-identical. + let mut i_vk = vk_lo; + let mut i_sn = ss; + let mut i_in = is_; + loop { + // Advance dense pointers to the next PRESENT entry. Pointers are + // monotonic, so total skip work is O(window) per hap, not O(n^2). + while i_sn < se && !bits_get_bit(&br.dense_snp_present, snp_base + (i_sn - ss)) { + i_sn += 1; + } + while i_in < ie + && !bits_get_bit(&br.dense_indel_present, indel_base + (i_in - is_)) + { + i_in += 1; + } + let has_vk = i_vk < vk_hi; + let has_sn = i_sn < se; + let has_in = i_in < ie; + if !has_vk && !has_sn && !has_in { + break; + } + let p_vk = if has_vk { br.vk[i_vk].position } else { u32::MAX }; + let p_sn = if has_sn { br.dense_snp[i_sn].position } else { u32::MAX }; + let p_in = if has_in { br.dense_indel[i_in].position } else { u32::MAX }; + let (p, key) = if has_vk && p_vk <= p_sn && p_vk <= p_in { + let e = &br.vk[i_vk]; + i_vk += 1; + (e.position, e.key) + } else if has_sn && p_sn <= p_in { + let e = &br.dense_snp[i_sn]; + i_sn += 1; + (e.position, e.key) + } else { + let e = &br.dense_indel[i_in]; + i_in += 1; + (e.position, e.key) + }; let (il, alt) = decode_alt(key, lut_bytes, lut_off); pos.push(p as i32); ilen.push(il as i32); From a5ee2a384985c2c5eee80501c8e79bfb63cec717 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 15:58:59 -0700 Subject: [PATCH 062/108] docs(spec): variant-windows mode for svar2-backed gvl Compose the validated svar2 variants decode with the existing assemble_variant_buffers window kernel (identity gather), per contig group. ref=window + alt in {window,allele}; ref=allele already blocked upstream. Two-pronged parity: SVAR1 for ref_window, numpy oracle for alt. Co-Authored-By: Claude Opus 4.8 --- ...2026-07-06-svar2-variant-windows-design.md | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md diff --git a/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md new file mode 100644 index 00000000..b05db072 --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md @@ -0,0 +1,266 @@ +# Design: variant-windows mode for svar2-backed gvl + +> **Status:** design approved · **Date:** 2026-07-06 · **Branch:** `svar2-m6b-kernel` (PR #266, draft) +> +> Related: `docs/superpowers/specs/2026-06-25-target7-variant-windows-rust-assembly-design.md` +> (the `assemble_variant_buffers` kernel this reuses), and +> `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` +> (which lists variant-windows as a guarded, out-of-scope svar2 mode — this +> design removes that guard). + +## 1. Problem + +`Dataset.open(path, reference=REF, svar2=).with_seqs("variant-windows", +VarWindowOpt(...))[regions, samples]` currently raises `NotImplementedError`. +`Svar2Haps.__call__` (`python/genvarloader/_dataset/_svar2_haps.py`, ~lines +282–285) guards `_FlatVariantWindows`: + +```python +if issubclass(self.kind, _FlatVariantWindows): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('variant-windows') yet." + ) +``` + +Every other piece needed already exists and is parity-validated: + +- The svar2 **`variants` decode** (`decode_variants_from_svar2_readbound`) is + byte-identical to the svar2 decode oracle and to SVAR1 for the variant SET + (`tests/dataset/test_svar2_dataset.py::test_svar2_variants_positions_match_svar1`, + `::test_svar2_variants_match_svar2_oracle`). +- The **window-assembly Rust kernel** `assemble_variant_buffers` + (`src/variants/windows.rs`, shimmed by `_assemble_variant_buffers_rust` in + `python/genvarloader/_dataset/_flat_variants.py`) is validated end-to-end for + SVAR1 and under both rust/numba backends (target-7 work). + +The **only** thing svar2 variant-windows adds over the working svar2 `variants` +decode is the window-assembly step (reference fetch + flank + tokenize). So the +implementation composes two already-validated kernels; **no new Rust**. + +## 2. Scope + +**In scope** — the live `variant-windows` read path for svar2 datasets: + +- `ref="window"` with `alt ∈ {"window", "allele"}`. +- Single-contig (fast path) and multi-contig (contig-group stitch). +- Empty (region, sample, ploid) groups with `dummy_variant` fill. + +**Out of scope** (existing guards stay in force; each must raise, not silently +diverge): + +- `ref="allele"` — **already** rejected upstream in `with_seqs` + (`_impl.py` ~line 720: raises `ValueError` when `window_opt.ref == "allele"` + and `self._seqs.variants.ref is None`, which is always true for `Svar2Haps` — + its dummy `_Variants` carries `ref=None`, and svar2 stores no REF allele + bytes). No new guard needed; a test pins it. +- `max_jitter > 0` at write or `jitter > 0` at read — the read-bound decode has + no right-clip, so a padded/slid window over-includes variants past the + (unpadded) read window. Same issue and same guard as svar2 `variants`. +- `min_af`/`max_af`, `unphased_union`, spliced, annotated, in-kernel + reverse-complement — all already guarded (`_guard_unsupported` + + `__call__`). Note: SVAR1 *does* support `unphased_union` + variant-windows, + but svar2 guards `unphased_union` wholesale; that remains deferred. +- No on-disk **format** change, no **public API** signature change + (`with_seqs`/`VarWindowOpt` are unchanged; this only makes an existing kind + reachable for svar2). +- A **fused** single-call svar2 windows kernel (decode+assemble in one FFI + crossing) is a perf optimization, deferred — see §7. + +## 3. Architecture — compose two validated kernels per contig group + +Add `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows`, +structured exactly like the existing `Svar2Haps._reconstruct_variants`: group +queries by contig (store readers are per-contig), process each group, then +stitch back to global `(b, P)` row order with the single-group identity fast +path and the multi-group inverse-row-permutation. + +The window config fields (`token_lut`, `token_dtype`, `window_opt`, +`unknown_token`, `flank_length`, `dummy_variant`) are set on the `Svar2Haps` +instance by `with_seqs("variant-windows", ...)` via `replace(self._seqs, ...)` +— `Svar2Haps` is a dataclass subclass of `Haps`, so this already works +(`_impl.py` ~lines 729–744). No change to `Svar2Haps.from_path` construction is +required for the config to arrive. + +### Per contig group `(ci, qsel)` + +1. **Decode** (existing): cache-slice via `_gather_inputs`, then + `decode_variants_from_svar2_readbound(...)` → + `(pos, ilen, alt_bytes, str_off, var_off)`. These per-variant arrays are the + already-gathered analog of SVAR1's global arrays. + +2. **Assemble windows** (existing kernel, identity gather): call + `_assemble_variant_buffers_rust` with: + + | kernel arg | svar2 value | + |---|---| + | `mode` | `1` (windows) | + | `v_idxs` | `np.arange(n_var, dtype=np.int32)` (identity — data is pre-gathered) | + | `row_offsets` | `var_off` (per-`(len(qsel)*P)`-row variant boundaries) | + | `alt_global`, `alt_off_global` | `alt_bytes`, `str_off` | + | `ref_global`, `ref_off_global` | `None`, `None` (`ref="allele"` blocked upstream) | + | `want_ref_bytes`, `want_flank` | `False`, `False` | + | `ref_mode` | `1` (window) | + | `alt_mode` | `1` if `window_opt.alt == "window"` else `2` | + | `flank_len` | `window_opt.flank_length` | + | `lut` | `self.token_lut` | + | `v_contigs` | `np.zeros(n_var, np.int32)` (single-contig slice) | + | `v_starts`, `ilens` | `pos`, `ilen` | + | `reference`, `ref_offsets` | `_ref_for_contig(ci)` slice + `[0, len]` | + | `pad_char` | `self.reference.pad_char` | + + Coordinate note: `pos` is 0-based within-contig (proven `==` SVAR1 `start`), + and the per-group reference is the single-contig slice starting at contig + position 0, so the kernel's `contig_offset(=0) + start` indexing and its + `[start-L, end+L)` OOB-padded read are correct — identical math to the SVAR1 + path, just fed pre-gathered arrays. + + Returned `bufs` keys: `ref_window` (always, since `ref="window"`) and either + `alt_window` (alt="window") or `alt` (alt="allele"), each `(data, seq_off)`. + +3. **Wrap** each `(data, seq_off)` into a per-group `_FlatWindow` + (`data`, `seq_offsets=seq_off`, `var_offsets=var_off`, `shape=(b,P,None,None)` + at group scope). + +### Stitch groups → global `(b, P)` order + +- **Single group** (the common single-contig read): grouped order already equals + global `(b, P)` order → return directly, no reorder, no concatenate. Mirrors + the `_reconstruct_variants` / `_assemble_haps` fast path. +- **Multiple groups**: compute `perm = _inverse_row_perm(cat_query_order, b, P)` + once, then + - scalar fields (`start`, `ilen`): reorder via `_ragged_arange_src(grouped_row_offsets, perm)` → `src`, index `pos_c[src]` / `ilen_c[src]` (exactly as `_reconstruct_variants` does for pos/ilen); + - each window token buffer: reorder via the existing + `_ragged_arange_gather_2level(token_data, grouped_row_offsets, grouped_seq_offsets, perm)` + → `(new_data, new_var_off, new_seq_off)`. The returned `new_var_off` equals + the reordered scalar `var_off_g`, so scalar and window offsets stay + consistent by construction. + +### Build the result + +- Scalar `fields`: `start` always; `ilen` when `"ilen" in self.var_fields` + (mirrors `get_variants_flat`; svar2 has no dosage/info/custom fields — + `available_var_fields = ["alt", "ilen", "start"]`). Each wrapped as `_Flat` + with the global `row_offsets` and `shape=(b, P, None)`. +- Assemble `_FlatVariantWindows(fields, ref_window=..., alt_window=... | alt=...)` + (set the present window fields per `window_opt`). +- If `self.dummy_variant is not None`, apply + `win.fill_empty_groups(self.dummy_variant, unk=self.unknown_token, flank_length=window_opt.flank_length)` + (the existing `_FlatVariantWindows.fill_empty_groups`). + +### `Svar2Haps.__call__` wiring + +Replace the NotImplementedError branch with: + +1. the **same jitter/max_jitter guard** already used for svar2 `variants` + (raise `NotImplementedError(... "right-clip" ...)` when + `self.max_jitter > 0 or jitter > 0`); +2. `return cast(_H, self._reconstruct_variant_windows(idx, regions))`. + +`_guard_unsupported(splice_plan)` is called first (as today), covering +splice/exonic/min_af/max_af/unphased_union. `to_rc` is not applicable to +variant-windows (reference-oriented; RC intentionally unsupported for the kind). + +## 4. Parity strategy + +Both halves gated; a matched SVAR1 dataset cannot be the oracle for the alt side +because svar2 and SVAR1 encode a deletion's ALT differently (svar2 `""` vs SVAR1 +anchor base, e.g. `G` for `GTA>G`) — so `alt`/`alt_window` bytes legitimately +differ. Split the gate: + +1. **`ref_window` → byte-identical to a matched SVAR1 variant-windows dataset.** + `ref_window` is a pure reference read over the same decoded variant SET + (positions identical to SVAR1), so it must match exactly. Free, strong check + on the reference-read + tokenize half. Compare `data` and both offset levels + (`var_offsets`, `seq_offsets`) after `to_ragged()`. + +2. **`alt`/`alt_window` (and the full bundle) → independent numpy oracle.** + A small (~40-line) reference that consumes the *already-validated* svar2 + `variants` output (`ds2.with_seqs("variants")[:, :]`, i.e. the decoded + `start`/`ilen`/`alt`) and, per variant, reproduces the kernel exactly: + - `ends = start - min(ilen, 0) + 1`; + - fetch `reference[start-L : end+L)` with absolute-coordinate `pad_char` OOB + padding; + - `alt="window"`: `flank5 (first L) · alt_bytes · flank3 (last L)`; + `alt="allele"`: bare `alt_bytes`; + - tokenize via the same LUT (built from `window_opt.token_alphabet` / + `unknown_token`), out-of-alphabet bytes → `unknown_token`. + Assert the oracle's `(data, seq_offsets, var_offsets)` equals the svar2 + `_FlatVariantWindows` window buffers. This is the true correctness gate for + the alt side and independently re-derives `ref_window` too. + +3. **Multi-contig stitch** — reuse the interleaved chr2/chr1 fixture + (`_src2`/`svar2_fixture2`) so the single-contig fast path is bypassed and the + inverse-row-perm reorder of both scalar and 2-level window buffers is + exercised; assert against both oracles above. + +4. **Empty-group / dummy fill** — a variant-free tail region (the existing bed's + `chr1 [20, 40)`) plus a `dummy_variant`; assert each empty `(b*p)` row gets + one all-`unknown_token` entry of the right width (`2L + len(dummy allele)` for + `alt_window`, `2L + len(dummy ref)` for `ref_window`, `len(dummy alt)` for + bare `alt`). Also assert the no-fill path is unchanged. + +5. **Guard contracts** — tests that `ref="allele"` raises `ValueError` at + `with_seqs`, and that `max_jitter>0`/`jitter>0` raises `NotImplementedError` + with the "right-clip" message for variant-windows (mirrors the existing + `test_svar2_variants_jitter_guard_raises`). + +Backends: run the parity tests under both `GVL_BACKEND` values where the shim +dispatches (as the existing svar2 suite does), since `_assemble_variant_buffers` +is registered with rust + numba. + +## 5. Testing & gates + +- `maturin develop --release` is **not** needed (no Rust change) — but the + branch may carry unbuilt Rust from prior work; run it once if the `.so` is + stale, per CLAUDE.md. +- `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` (new tests), + then the full svar2 suite `pixi run -e dev pytest tests -k svar2 -q`. +- Full tree before push: `pixi run -e dev pytest tests -q` (scoped runs skip + `tests/unit/`), and `pixi run -e dev cargo-test` (no Rust change, but cheap + insurance). +- Lint/format/typecheck: `ruff check python/ tests/ && ruff format python/ tests/ + && typecheck`. +- HPC gotcha: `--basetemp=$(pwd)/.pytest_tmp` so the write path's `os.link` + hardlink does not fail cross-device (Errno 18). + +## 6. Docs / skill / roadmap + +Public API surface is unchanged, but svar2's **supported-mode matrix** changes +(variant-windows becomes supported), so: + +- `skills/genvarloader/SKILL.md` — svar2 mode support / gotchas. +- `docs/source/*.md` where svar2 output-mode support is enumerated + (`dataset.md`, `faq.md` as applicable). +- The svar2 read-bound perf design's "out of scope" list — note variant-windows + is now implemented (feature complete; perf/fusion still deferred). +- No `api.md`/`__all__` change (no new public symbol). + +## 7. Deferred / future + +- **Fused svar2 windows kernel.** The two-call compose (decode then assemble, + per group) mirrors the existing svar2 variants/tracks split-kernel structure + and reuses validated code. A single fused + `assemble_variant_windows_from_svar2` kernel could cut one FFI crossing + + intermediate buffers, but that is a perf optimization to pursue only after + profiling the live path (consistent with the read-bound perf design's + measure-first discipline). Not in this work. +- `unphased_union` + variant-windows for svar2 (SVAR1 supports it; svar2 guards + `unphased_union` wholesale). +- `ref="allele"` for svar2 (would require the decode to also return REF bytes, + or reconstructing REF from `reference[start:start+ref_len]` where + `ref_len = len(alt) - ilen`; a genoray-touching or extra-work change, deferred). + +## 8. Files + +- **Edit** `python/genvarloader/_dataset/_svar2_haps.py`: + - new `_reconstruct_variant_windows(self, idx, regions)`; + - `__call__`: replace the NotImplementedError branch with the jitter guard + + dispatch; + - imports: `_assemble_variant_buffers_rust`, `_FlatWindow` (from + `_flat_variants`), and reuse existing `_ragged_arange_src` / + `_ragged_arange_gather_2level` / `_inverse_row_perm`. +- **Edit** `tests/dataset/test_svar2_dataset.py`: ref_window-vs-SVAR1, + alt-vs-numpy-oracle, multi-contig, empty-group/dummy, `ref="allele"` guard, + jitter guard. Add a variant-windows bigwig/track combo only if trivial; + otherwise seqs-only. +- **Edit** docs/skill per §6. From 8b71beb623d3ecf2f7bdc55211814587bcd1fbb0 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 16:09:59 -0700 Subject: [PATCH 063/108] docs(spec): add unphased_union to svar2 variants + variant-windows scope Same offset fold (row_offsets[::P], eff_ploidy=1) serves both decode modes; remove the unphased_union clause from _guard_unsupported. Add union parity tests vs SVAR1 union + folded svar2 decode oracle. Co-Authored-By: Claude Opus 4.8 --- ...2026-07-06-svar2-variant-windows-design.md | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md index b05db072..afc4aafb 100644 --- a/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md +++ b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md @@ -44,6 +44,13 @@ implementation composes two already-validated kernels; **no new Rust**. - `ref="window"` with `alt ∈ {"window", "allele"}`. - Single-contig (fast path) and multi-contig (contig-group stitch). - Empty (region, sample, ploid) groups with `dummy_variant` fill. +- **`unphased_union`** for **both** svar2 decode modes (`variants` and + `variant-windows`) — the ploidy-1 union view (issue #222). The same offset + fold serves both, so `variants` mode gets it too (avoids a confusing + "union works for windows but not plain variants" gap). SVAR1 supports union + for these two modes; svar2 now matches. Haplotypes/annotated + union remain + blocked at `with_seqs` (a union of phased sequences is ill-defined) — that + guard is in `_impl.py`, unchanged. **Out of scope** (existing guards stay in force; each must raise, not silently diverge): @@ -56,10 +63,10 @@ diverge): - `max_jitter > 0` at write or `jitter > 0` at read — the read-bound decode has no right-clip, so a padded/slid window over-includes variants past the (unpadded) read window. Same issue and same guard as svar2 `variants`. -- `min_af`/`max_af`, `unphased_union`, spliced, annotated, in-kernel - reverse-complement — all already guarded (`_guard_unsupported` + - `__call__`). Note: SVAR1 *does* support `unphased_union` + variant-windows, - but svar2 guards `unphased_union` wholesale; that remains deferred. +- `min_af`/`max_af`, spliced, annotated, in-kernel reverse-complement — all + already guarded (`_guard_unsupported` + `__call__`). (`unphased_union` is now + **in scope** — see above — so it is removed from `_guard_unsupported`; the + haplotypes/annotated + union block stays in `_impl.py`.) - No on-disk **format** change, no **public API** signature change (`with_seqs`/`VarWindowOpt` are unchanged; this only makes an existing kind reachable for svar2). @@ -88,14 +95,25 @@ required for the config to arrive. `(pos, ilen, alt_bytes, str_off, var_off)`. These per-variant arrays are the already-gathered analog of SVAR1's global arrays. +1a. **unphased_union fold** (when `self.unphased_union`): the group's `var_off` + has `len(qsel) * P + 1` entries in row = `q*P + p` order (the P haps of a + query are contiguous). Fold to `len(qsel) + 1` by keeping every P-th offset: + `row_offsets = np.ascontiguousarray(var_off[::P])`, and set the group's + effective ploidy `p_eff = 1`. This concatenates hap-0's variants then + hap-1's per query — no sort, no dedup, `v_idxs`/`pos`/`ilen`/`alt_bytes` + untouched. Identical to SVAR1 `get_variants_flat` (lines ~842–845). When not + union, `row_offsets = var_off` and `p_eff = P`. The window/scalar output + `shape` uses `p_eff` (`(b, p_eff, None[, None])`) and the group stitch uses + `_inverse_row_perm(cat_query_order, b, p_eff)`. + 2. **Assemble windows** (existing kernel, identity gather): call `_assemble_variant_buffers_rust` with: | kernel arg | svar2 value | |---|---| | `mode` | `1` (windows) | - | `v_idxs` | `np.arange(n_var, dtype=np.int32)` (identity — data is pre-gathered) | - | `row_offsets` | `var_off` (per-`(len(qsel)*P)`-row variant boundaries) | + | `v_idxs` | `np.arange(n_var, dtype=np.int32)` (identity — data is pre-gathered; spans ALL variants incl. both haps, unaffected by the union fold) | + | `row_offsets` | the (possibly folded) `row_offsets` from step 1a | | `alt_global`, `alt_off_global` | `alt_bytes`, `str_off` | | `ref_global`, `ref_off_global` | `None`, `None` (`ref="allele"` blocked upstream) | | `want_ref_bytes`, `want_flank` | `False`, `False` | @@ -157,8 +175,21 @@ Replace the NotImplementedError branch with: 2. `return cast(_H, self._reconstruct_variant_windows(idx, regions))`. `_guard_unsupported(splice_plan)` is called first (as today), covering -splice/exonic/min_af/max_af/unphased_union. `to_rc` is not applicable to -variant-windows (reference-oriented; RC intentionally unsupported for the kind). +splice/exonic/min_af/max_af (the `unphased_union` clause is **removed** from +`_guard_unsupported` — see below). `to_rc` is not applicable to variant-windows +(reference-oriented; RC intentionally unsupported for the kind). + +### `_reconstruct_variants` gets the same fold + +Because `unphased_union` is enabled for svar2 `variants` too, add step-1a's fold +to the existing `Svar2Haps._reconstruct_variants` as well: within each group, +`row_offsets = var_off[::P]` and `p_eff = 1` when `self.unphased_union` (else +unchanged), use `p_eff` for the `shape` and the stitch perm. The decoded +`pos`/`ilen`/`alt` data and the identity relationship are untouched — only the +per-row grouping changes. Remove the `unphased_union` clause from +`_guard_unsupported` so both decode modes reach the fold; the haplotypes/tracks +paths never set the flag (blocked at `with_seqs`), so removing the guard cannot +mis-route them. ## 4. Parity strategy @@ -204,6 +235,19 @@ differ. Split the gate: with the "right-clip" message for variant-windows (mirrors the existing `test_svar2_variants_jitter_guard_raises`). +6. **unphased_union** (both modes) — against a matched SVAR1 dataset with + `with_settings(unphased_union=True)`: + - `variants` mode: `start`/`ilen` byte-identical to SVAR1 union (the fold is + order-preserving and drops nothing, so hap-0-then-hap-1 concat matches); + ALT compared to the svar2 decode oracle folded the same way (SVAR1 ALT + differs for deletions, per §4.2). Ploidy axis folds 2→1 (`shape[-2] == 1`). + Also assert the union row equals hap-0's decode concatenated with hap-1's. + - `variant-windows` mode: `ref_window` byte-identical to SVAR1 union + `ref_window`; `alt`/`alt_window` vs the numpy oracle applied to the folded + svar2 decode. `to_ragged()` succeeds (offsets consistent post-fold). + - Multi-contig + union together (interleaved fixture) to lock the + `p_eff=1` stitch perm. + Backends: run the parity tests under both `GVL_BACKEND` values where the shim dispatches (as the existing svar2 suite does), since `_assemble_variant_buffers` is registered with rust + numba. @@ -244,8 +288,6 @@ Public API surface is unchanged, but svar2's **supported-mode matrix** changes intermediate buffers, but that is a perf optimization to pursue only after profiling the live path (consistent with the read-bound perf design's measure-first discipline). Not in this work. -- `unphased_union` + variant-windows for svar2 (SVAR1 supports it; svar2 guards - `unphased_union` wholesale). - `ref="allele"` for svar2 (would require the decode to also return REF bytes, or reconstructing REF from `reference[start:start+ref_len]` where `ref_len = len(alt) - ilen`; a genoray-touching or extra-work change, deferred). @@ -256,6 +298,8 @@ Public API surface is unchanged, but svar2's **supported-mode matrix** changes - new `_reconstruct_variant_windows(self, idx, regions)`; - `__call__`: replace the NotImplementedError branch with the jitter guard + dispatch; + - `_reconstruct_variants`: add the unphased_union fold (`p_eff`); + - `_guard_unsupported`: remove the `unphased_union` clause; - imports: `_assemble_variant_buffers_rust`, `_FlatWindow` (from `_flat_variants`), and reuse existing `_ragged_arange_src` / `_ragged_arange_gather_2level` / `_inverse_row_perm`. From 355f91bde43a2000bc3ac4ca6272685a8ccbdc8e Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 16:19:19 -0700 Subject: [PATCH 064/108] docs(plan): svar2 variant-windows + unphased_union implementation plan 4 tasks: windows core (decode+assemble compose, guards, dummy fill), variants-mode union, windows union, docs. TDD steps with full code. Co-Authored-By: Claude Opus 4.8 --- .../plans/2026-07-06-svar2-variant-windows.md | 793 ++++++++++++++++++ 1 file changed, 793 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-06-svar2-variant-windows.md diff --git a/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md b/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md new file mode 100644 index 00000000..fdff4701 --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md @@ -0,0 +1,793 @@ +# svar2 variant-windows (+ unphased_union) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `Dataset.open(..., svar2=...).with_seqs("variant-windows", VarWindowOpt(...))[regions, samples]` work (currently `NotImplementedError`), and enable `unphased_union` for both svar2 decode modes (`variants` and `variant-windows`). + +**Architecture:** svar2 variant-windows = the already-validated svar2 `variants` decode (`decode_variants_from_svar2_readbound`) composed with the already-validated window-assembly Rust kernel (`assemble_variant_buffers`, via the `_assemble_variant_buffers_rust` shim), per contig group, feeding the decoded per-variant arrays as the kernel's "global" arrays with an **identity gather** (`v_idxs = arange(n_var)`). No new Rust. `unphased_union` is a per-group offset fold (`row_offsets[::P]`, `eff_ploidy=1`) applied before assembly, identical to SVAR1's `get_variants_flat`. + +**Tech Stack:** Python 3.10, numpy, `seqpro.rag.Ragged`, PyO3 Rust extension (unchanged), pytest, pixi (`-e dev`). + +**Spec:** `docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md` + +## Global Constraints + +- **No Rust change** — this composes existing kernels. `maturin develop --release` is only needed if the branch's `.so` is stale from prior work; run it once at the start if in doubt (pytest imports the stale `.so` otherwise — CLAUDE.md). +- **Parity is a hard gate.** svar2 read output is byte-identical to its oracle. After every change run the relevant pytest; before pushing run the full tree `pixi run -e dev pytest tests -q` (scoped runs skip `tests/unit/`). +- **Deletion-ALT difference:** SVAR1 keeps a deletion's anchor base (e.g. `G` for `GTA>G`); svar2 decodes `""`. So `alt`/`alt_window` bytes are **not** SVAR1-identical — only `ref_window` is. Oracle strategy per task reflects this. +- **HPC gotcha:** pass `--basetemp=$(pwd)/.pytest_tmp` to pytest so the write path's `os.link` hardlink does not fail cross-device (Errno 18). +- **Coordinate convention:** svar2 decoded `pos` is 0-based within-contig, proven `==` SVAR1 `start`; it feeds the kernel's `v_starts` directly. The per-group reference is a single-contig slice (`_ref_for_contig`), so `v_contigs = zeros` and `ref_offsets = [0, len]`. +- **Commits:** conventional-commit style. Commit after each task's tests pass. The pre-commit `pyrefly` hook spuriously fails on docs-only commits in this worktree (finds no Python files → exit 1); for commits that DO touch Python it runs normally, so do not add `--no-verify` to code commits. +- **`ref="allele"` is already blocked** upstream (`_impl.py` ~line 720 raises `ValueError` when `window_opt.ref=="allele"` and `variants.ref is None`, always true for `Svar2Haps`). No new guard; a test pins it. + +--- + +## File Structure + +- **Modify** `python/genvarloader/_dataset/_svar2_haps.py`: + - new method `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows`; + - `Svar2Haps.__call__`: replace the `_FlatVariantWindows` `NotImplementedError` with the jitter guard + dispatch; + - `Svar2Haps._reconstruct_variants`: add the `unphased_union` fold; + - `Svar2Haps._guard_unsupported`: remove the `unphased_union` clause; + - imports: add `_assemble_variant_buffers_rust`, `_FlatWindow` from `._flat_variants`; add `_Flat` is already imported. +- **Modify** `tests/dataset/test_svar2_dataset.py`: new variant-windows + union parity tests and guard tests (reuses the module's existing `_src`/`svar_fixture`/`svar2_fixture`/`bed` and `_src2`/`svar_fixture2`/`svar2_fixture2` fixtures). +- **Modify** docs: `skills/genvarloader/SKILL.md`, `docs/source/*` where svar2 mode support is enumerated, and the two related specs' out-of-scope notes. + +--- + +## Task 1: `_reconstruct_variant_windows` core (non-union, incl. dummy fill) + wiring + guards + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes (existing): `decode_variants_from_svar2_readbound`, `self._gather_inputs`, `self._contig_groups`, `self._ref_for_contig`, `self._inverse_row_perm`, `_ragged_arange_src`, `_ragged_arange_gather_2level`, `lengths_to_offsets`; `_assemble_variant_buffers_rust`, `_FlatWindow`, `_FlatVariantWindows` (from `_flat_variants`); `self.window_opt`, `self.token_lut`, `self.reference`, `self.var_fields`, `self.dummy_variant`, `self.unknown_token`. +- Produces: `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows` and the wired `__call__` branch. Task 3 adds a `p_eff` fold to this same method. + +- [ ] **Step 1: Write the failing test — ref_window byte-identical to SVAR1 (single-contig)** + +Add to `tests/dataset/test_svar2_dataset.py`. Place a shared opt + helper near the top of the file (after imports): + +```python +from genvarloader import VarWindowOpt + +_WIN_OPT = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="window" +) + + +def _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref, opt=_WIN_OPT): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + return w1, w2 + + +def _assert_window_equal(a, b, name: str) -> None: + """Flat-buffer equality of two _FlatWindow fields (data + both offset levels).""" + assert np.array_equal(np.asarray(a.var_offsets), np.asarray(b.var_offsets)), ( + f"{name} var_offsets differ" + ) + assert np.array_equal(np.asarray(a.seq_offsets), np.asarray(b.seq_offsets)), ( + f"{name} seq_offsets differ" + ) + assert np.array_equal(np.asarray(a.data), np.asarray(b.data)), f"{name} data differ" +``` + +Then the test: + +```python +def test_svar2_variant_windows_ref_window_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """ref_window is a pure reference read over an identical variant SET, so it is + byte-identical to SVAR1 (independent of the deletion-ALT encoding difference).""" + _bcf, ref = _src + w1, w2 = _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + assert w2.ref_window is not None + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # scalar start field also identical (same variant SET) — compare _Flat buffers. + assert np.array_equal( + np.asarray(w2.fields["start"].data), np.asarray(w1.fields["start"].data) + ) + assert np.array_equal( + np.asarray(w2.fields["start"].offsets), np.asarray(w1.fields["start"].offsets) + ) +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_ref_window_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL with `NotImplementedError: svar2 datasets do not support with_seqs('variant-windows') yet.` + +- [ ] **Step 3: Add imports to `_svar2_haps.py`** + +At the existing import of `_FlatVariantWindows` (line ~50), extend it: + +```python +from ._flat_variants import ( + _FlatVariantWindows, + _FlatWindow, + _assemble_variant_buffers_rust, +) +``` + +(`_Flat` and `lengths_to_offsets` are already imported at the top of the file.) + +- [ ] **Step 4: Implement `_reconstruct_variant_windows`** + +Add this method to `Svar2Haps`, right after `_reconstruct_variants` (before `# ---- helpers ----`): + +```python +def _reconstruct_variant_windows( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] +) -> _FlatVariantWindows: + """Variant-windows for svar2: decode variants per contig group, then run the + shared ``assemble_variant_buffers`` window kernel over the decoded arrays via + an identity gather. ``ref="allele"`` is blocked upstream, so ref is always a + reference-read window; ``alt`` follows ``window_opt.alt``. + """ + assert self.window_opt is not None and self.token_lut is not None + assert self.reference is not None + from typing import Any + + opt = self.window_opt + L = opt.flank_length + ref_mode = 1 # ref="window" (ref="allele" rejected in with_seqs) + alt_mode = 1 if opt.alt == "window" else 2 + include_ilen = "ilen" in self.var_fields + + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + p_eff = P # unphased_union fold (Task 3) sets this to 1 per group. + + cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + # name -> per-group (token_data, per-variant seq offsets) + win_data: dict[str, list[NDArray]] = {} + win_seq_off: dict[str, list[NDArray[np.int64]]] = {} + + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + ) + ) + pos = np.asarray(pos, np.int32) + ilen = np.asarray(ilen, np.int32) + alt_bytes = np.asarray(alt_bytes, np.uint8) + str_off = np.asarray(str_off, np.int64) + var_off = np.asarray(var_off, np.int64) + + row_off = var_off # Task 3: fold to var_off[::P] under unphased_union. + n_var = int(len(pos)) + ref_, ref_offsets = self._ref_for_contig(ci) + bufs = _assemble_variant_buffers_rust( + 1, # windows mode + np.arange(n_var, dtype=np.int32), # identity v_idxs (data pre-gathered) + row_off, + alt_bytes, # alt_global + str_off, # alt_off_global + None, # ref_global (ref="window") + None, # ref_off_global + False, # want_ref_bytes + False, # want_flank + ref_mode, + alt_mode, + L, + self.token_lut, + np.zeros(n_var, np.int32), # v_contigs (single-contig ref slice) + pos, # v_starts + ilen, # ilens + ref_, + ref_offsets, + self.reference.pad_char, + ) + + cat_row_off.append(row_off) + cat_pos.append(pos) + cat_ilen.append(ilen) + cat_query_order.append(qsel) + for name, (data, seq_off) in bufs.items(): + win_data.setdefault(name, []).append(np.asarray(data)) + win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) + + shape: tuple[int | None, ...] = (b, p_eff, None) + wshape: tuple[int | None, ...] = (b, p_eff, None, None) + + # Single contig group: grouped order already equals global (b, p_eff) order. + if len(cat_pos) == 1: + row_off = cat_row_off[0] + fields: dict[str, Any] = { + "start": _Flat.from_offsets(cat_pos[0], shape, row_off) + } + if include_ilen: + fields["ilen"] = _Flat.from_offsets(cat_ilen[0], shape, row_off) + win = _FlatVariantWindows(fields) + for name in win_data: + setattr( + win, + name, + _FlatWindow(win_data[name][0], win_seq_off[name][0], row_off, wshape), + ) + else: + perm = self._inverse_row_perm(cat_query_order, b, p_eff) + grouped_row_off = lengths_to_offsets( + np.concatenate([np.diff(r) for r in cat_row_off]), np.int64 + ) + pos_c = np.concatenate(cat_pos) + ilen_c = np.concatenate(cat_ilen) + src, row_off_g = _ragged_arange_src(grouped_row_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] + fields = {"start": _Flat.from_offsets(pos_g, shape, row_off_g)} + if include_ilen: + fields["ilen"] = _Flat.from_offsets(ilen_g, shape, row_off_g) + win = _FlatVariantWindows(fields) + for name in win_data: + data_c = np.concatenate(win_data[name]) + grouped_seq_off = lengths_to_offsets( + np.concatenate([np.diff(s) for s in win_seq_off[name]]), np.int64 + ) + nd, nvar, nseq = _ragged_arange_gather_2level( + data_c, grouped_row_off, grouped_seq_off, perm + ) + setattr(win, name, _FlatWindow(nd, nseq, nvar, wshape)) + + if self.dummy_variant is not None: + win = win.fill_empty_groups( + self.dummy_variant, unk=self.unknown_token, flank_length=L + ) + return win +``` + +- [ ] **Step 5: Wire `__call__` — replace the NotImplementedError with the guard + dispatch** + +In `Svar2Haps.__call__`, replace this block: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + if issubclass(self.kind, _FlatVariantWindows): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('variant-windows') yet." + ) +``` + +with: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + # variants AND variant-windows decode variants; the read-bound decode + # has NO right-clip, so max_jitter>0 / jitter>0 would over-include + # variants past the (unpadded) read window. Guard both modes. + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." + ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) +``` + +Then DELETE the now-duplicated jitter guard that follows in the old `RaggedVariants` branch (the block starting `# ``decode_variants_from_svar2_readbound`` has NO right-clip` down through its `raise NotImplementedError(... "right-clip" ...)`), since the guard now runs once above for both kinds. Keep the trailing `return cast(_H, self._reconstruct_variants(idx, regions))`. + +The resulting branch reads: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." + ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) + # RaggedVariants: RC is applied by the caller (_getitem_unspliced), + # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). + return cast(_H, self._reconstruct_variants(idx, regions)) +``` + +- [ ] **Step 6: Run the ref_window test to confirm it passes** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_ref_window_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS + +- [ ] **Step 7: Add the alt-window decomposition test (correctness of the alt assembly)** + +`alt_window` is NOT SVAR1-identical (deletion ALT differs), so verify it against svar2's own outputs: since tokenization is per-byte independent, `alt_window[j] == ref_window[j][:L] · alt_allele[j] · ref_window[j][-L:]`, where `alt_allele` is the bare tokenized alt (a second svar2 windows read with `alt="allele"`) and `ref_window` is already validated against SVAR1 in Step 1. + +```python +def test_svar2_variant_windows_alt_window_decomposition( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """alt_window[j] == ref_window[j][:L] + tokenize(alt_j) + ref_window[j][-L:]. + Uses only svar2's own outputs; ref_window is separately pinned to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + w_win = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + alt_opt = VarWindowOpt( + flank_length=L, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="allele" + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + + rw = w_win.ref_window + aw = w_win.alt_window + ba = w_alt.alt # bare tokenized alt (_FlatWindow) + assert aw is not None and rw is not None and ba is not None + + # Same variant SET/order across the two reads. + assert np.array_equal(np.asarray(aw.var_offsets), np.asarray(ba.var_offsets)) + n_var = len(np.asarray(aw.seq_offsets)) - 1 + rso, aso, bso = ( + np.asarray(rw.seq_offsets), + np.asarray(aw.seq_offsets), + np.asarray(ba.seq_offsets), + ) + rd, ad, bd = np.asarray(rw.data), np.asarray(aw.data), np.asarray(ba.data) + for j in range(n_var): + rj = rd[rso[j] : rso[j + 1]] + aj = ad[aso[j] : aso[j + 1]] + bj = bd[bso[j] : bso[j + 1]] + expected = np.concatenate([rj[:L], bj, rj[len(rj) - L :]]) + assert np.array_equal(aj, expected), f"alt_window variant {j} mismatch" +``` + +- [ ] **Step 8: Add the bare-alt tokenization test (pins alt bytes → tokens)** + +Confirms the bare `alt` equals `tokenize(variants.alt)`, tying the window alt bytes to the validated variants decode. + +```python +def test_svar2_variant_windows_bare_alt_tokenizes_variants_alt( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + import awkward as ak + + from genvarloader._dataset._flat_flanks import build_token_lut + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + alt_opt = VarWindowOpt( + flank_length=L, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="allele" + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + v = ds2.with_seqs("variants")[:, :] # RaggedVariants (validated) + + lut, _ = build_token_lut(b"ACGT", 4) + # Per (b,p) row, list of alt byte-strings, in variant order. + alt_rows = ak.to_list(v.alt.to_ak()) # nested (b, p) -> [bytes,...] + flat_alts: list[bytes] = [] + for per_ploid in alt_rows: + for per_var in per_ploid: + for a in per_var: + flat_alts.append(bytes(a) if not isinstance(a, bytes) else a) + + ba = w_alt.alt + bso, bd = np.asarray(ba.seq_offsets), np.asarray(ba.data) + assert len(flat_alts) == len(bso) - 1 + for j, a in enumerate(flat_alts): + toks = bd[bso[j] : bso[j + 1]] + expected = np.array([lut[byte] for byte in a], dtype=toks.dtype) + assert np.array_equal(toks, expected), f"bare alt variant {j} mismatch" +``` + +- [ ] **Step 9: Add multi-contig parity test** + +```python +def test_svar2_variant_windows_multicontig(tmp_path, svar_fixture2, svar2_fixture2, _src2): + """ref_window byte-identical to SVAR1 across an interleaved 2-contig bed + (single-contig fast path bypassed -> exercises the group-stitch reorder).""" + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "vw_mc1.gvl" + d2 = tmp_path / "vw_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # alt_window decomposition holds across the stitch too. + w2.alt_window.to_ragged() # offsets/data consistent post-reorder + w2.ref_window.to_ragged() +``` + +- [ ] **Step 10: Add the dummy-variant empty-group fill test** + +The module `bed` has a variant-free tail (`chr1 [20, 40)`), so with a `dummy_variant` set, its rows must each get one all-`unknown_token` window of width `2L + len(dummy allele)`. + +```python +def test_svar2_variant_windows_dummy_fills_empty_groups( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + from genvarloader import DummyVariant + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + dummy = DummyVariant(alt=b"N", ref=b"N") + w = ( + ds2.with_output_format("flat") + .with_settings(dummy_variant=dummy) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Every (b*p) row now has >= 1 variant (no empty rows). + vo = np.asarray(w.ref_window.var_offsets) + assert np.all(np.diff(vo) >= 1) + # ref_window dummy width = 2L + len(dummy.ref); alt_window = 2L + len(dummy.alt). + # (For a filled row the sole variant's window length equals the dummy width.) + # Assert at least one dummy-width ref window exists (the tail region rows). + rso = np.asarray(w.ref_window.seq_offsets) + assert (np.diff(rso) == (2 * L + len(dummy.ref))).any() + w.ref_window.to_ragged() + w.alt_window.to_ragged() +``` + +- [ ] **Step 11: Add the guard tests (ref="allele" ValueError, jitter NotImplementedError)** + +```python +def test_svar2_variant_windows_ref_allele_guard(tmp_path, bed, svar2_fixture, _src): + """ref='allele' needs stored REF bytes svar2 lacks -> ValueError at with_seqs.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + bad = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="allele", alt="window" + ) + with pytest.raises(ValueError, match="REF"): + ds.with_seqs("variant-windows", bad) + + +def test_svar2_variant_windows_jitter_guard(tmp_path, svar2_fixture, _src): + """variant-windows must raise when written with max_jitter>0 (no right-clip).""" + from genoray import SparseVar2 + + _bcf, ref = _src + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, jbed, variants=SparseVar2(svar2_fixture), samples=None, max_jitter=2, overwrite=True + ) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variant-windows", _WIN_OPT)[:, :] +``` + +- [ ] **Step 12: Run the full Task-1 test set** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS (7 new tests). Also run the existing svar2 suite to confirm no regression: +`pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q --basetemp=$(pwd)/.pytest_tmp` → all pass. + +- [ ] **Step 13: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): variant-windows read path (ref=window, alt window/allele) + +Compose decode_variants_from_svar2_readbound with assemble_variant_buffers +(identity gather) per contig group. ref_window byte-identical to SVAR1; +alt validated via ref-flank decomposition + tokenized variants.alt. Wire +__call__ (jitter guard shared with variants), pin ref=allele + jitter guards. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: unphased_union for svar2 `variants` mode + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variants`, `_guard_unsupported`) +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes: `self.unphased_union` (inherited `Haps` field, set via `with_settings`), the existing `_reconstruct_variants` structure. +- Produces: `_reconstruct_variants` honoring the ploidy-1 fold; `_guard_unsupported` no longer raises on `unphased_union`. + +- [ ] **Step 1: Write the failing test — variants union vs SVAR1** + +```python +def test_svar2_variants_unphased_union_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Ploidy-1 union: start/ilen byte-identical to SVAR1 union (order-preserving + fold, no dedup). ALT differs by encoding, so ALT is not compared to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants").with_settings(unphased_union=True)[:, :] + b = ds2.with_seqs("variants").with_settings(unphased_union=True)[:, :] + # Ploidy axis folded 2 -> 1. + assert a.start.shape[-2] == 1 and b.start.shape[-2] == 1 + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variants_unphased_union_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL with `NotImplementedError: unphased_union is not supported for svar2 datasets yet.` + +- [ ] **Step 3: Remove the unphased_union clause from `_guard_unsupported`** + +In `Svar2Haps._guard_unsupported`, delete: + +```python + if self.unphased_union: + raise NotImplementedError( + "unphased_union is not supported for svar2 datasets yet." + ) +``` + +(Haplotypes/annotated + union is still blocked at `with_seqs` in `_impl.py`, so the haplotypes/tracks paths can never reach here with the flag set.) + +- [ ] **Step 4: Add the fold to `_reconstruct_variants`** + +In `_reconstruct_variants`, after `P = int(self.genotypes.shape[-2])` and after the `groups = self._contig_groups(...)` line, introduce `p_eff`. Then in the per-group loop, fold `var_off` before appending, and use `p_eff` for the shape and stitch. Concretely: + +1. After `groups = self._contig_groups(contig_ids)` add: + ```python + p_eff = 1 if self.unphased_union else P + ``` +2. In the loop, the group currently does `var_off = np.asarray(var_off, np.int64)` then `cat_var_lens.append(np.diff(var_off))`. Insert the fold immediately after `var_off = np.asarray(var_off, np.int64)`: + ```python + if self.unphased_union: + var_off = np.ascontiguousarray(var_off[::P]) + ``` + (This keeps every P-th boundary: hap-0's then hap-1's variants per query, concatenated. `pos`/`ilen`/`alt` data and `str_off` are untouched — only row grouping changes.) +3. Replace both `shape = (b, P, None)` occurrences (single-group fast path and multi-group path) with `shape = (b, p_eff, None)`. +4. Replace the single `perm = self._inverse_row_perm(cat_query_order, b, P)` with `perm = self._inverse_row_perm(cat_query_order, b, p_eff)`. + +- [ ] **Step 5: Run the union test + the existing variants parity tests** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variants" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS — the new union test plus the existing `test_svar2_variants_positions_match_svar1`, `test_svar2_variants_match_svar2_oracle`, `test_svar2_variants_jitter_guard_raises` all still pass (p_eff=P when the flag is off preserves the diploid path byte-for-byte). + +- [ ] **Step 6: Commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): unphased_union for variants mode (ploidy-1 fold) + +Fold row_offsets[::P], eff_ploidy=1 per contig group (order-preserving, +no dedup) — byte-identical to SVAR1 union for start/ilen. Drop the +unphased_union guard; haplotypes/annotated+union stays blocked upstream. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: unphased_union for svar2 `variant-windows` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variant_windows`) +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes: `_reconstruct_variant_windows` from Task 1 (the `p_eff = P` placeholder and `row_off = var_off` comment mark the fold points), `self.unphased_union`. +- Produces: variant-windows honoring the ploidy-1 fold. + +- [ ] **Step 1: Write the failing test — windows union** + +```python +def test_svar2_variant_windows_unphased_union( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Union folds ploidy 2->1 for windows; ref_window still byte-identical to + SVAR1 union, and the union row is hap-0's windows then hap-1's, concatenated.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ( + ds1.with_output_format("flat") + .with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Ploidy axis folded 2 -> 1. Scalar shape is (R,S,p_eff,None) so ploidy is at + # [-2]; window shape is (R,S,p_eff,None,None) so ploidy is at [-3]. + assert w2.fields["start"].shape[-2] == 1 + assert w2.ref_window.shape[-3] == 1 + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # Union row count == sum over haplotypes: compare to the non-union var counts. + nu = np.asarray(w2.ref_window.var_offsets) + w2_diploid = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + nd = np.asarray(w2_diploid.ref_window.var_offsets) + P = int(ds2._seqs.genotypes.shape[-2]) + # Folded per-row counts == sum of the P per-hap counts (rows q*P+p are contiguous). + diploid_counts = np.diff(nd).reshape(-1, P).sum(1) + union_counts = np.diff(nu) + assert np.array_equal(union_counts, diploid_counts) + w2.ref_window.to_ragged() + w2.alt_window.to_ragged() +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_unphased_union" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL — the ploidy axis is still 2 (`assert ... shape[-2] == 1` fails), because Task 1 hardcodes `p_eff = P` and `row_off = var_off`. + +- [ ] **Step 3: Apply the fold in `_reconstruct_variant_windows`** + +Two edits in the method: + +1. Replace `p_eff = P # unphased_union fold (Task 3) sets this to 1 per group.` with: + ```python + p_eff = 1 if self.unphased_union else P + ``` +2. Replace `row_off = var_off # Task 3: fold to var_off[::P] under unphased_union.` with: + ```python + row_off = np.ascontiguousarray(var_off[::P]) if self.unphased_union else var_off + ``` + +(The `v_idxs = arange(n_var)` identity, `pos`/`ilen`/`alt_bytes`, and the assemble call are unchanged — folding `row_off` only regroups the per-row variant boundaries the kernel emits. `shape`/`wshape` already use `p_eff`, and the multi-group stitch already uses `_inverse_row_perm(cat_query_order, b, p_eff)`.) + +- [ ] **Step 4: Run the windows union test + the whole variant_windows set** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS (Task-1 tests unaffected — `p_eff=P` when the flag is off — plus the new union test). + +- [ ] **Step 5: Add a multi-contig + union test (locks the p_eff=1 stitch)** + +```python +def test_svar2_variant_windows_union_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + {"chrom": ["chr2", "chr1"], "chromStart": [0, 0], "chromEnd": [40, 40]} + ) + d1 = tmp_path / "vwu_mc1.gvl" + d2 = tmp_path / "vwu_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = (ds1.with_output_format("flat").with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :]) + w2 = (ds2.with_output_format("flat").with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :]) + assert w2.ref_window.shape[-3] == 1 # window ploidy axis + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window (union, multicontig)") + w2.alt_window.to_ragged() +``` + +- [ ] **Step 6: Run + commit** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` → PASS. + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): unphased_union for variant-windows (ploidy-1 fold) + +Fold row_offsets[::P] before the window assemble call; p_eff=1 drives +shape + stitch. ref_window stays SVAR1-identical under union. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: docs, skill, and spec out-of-scope updates + +**Files:** +- Modify: `skills/genvarloader/SKILL.md` +- Modify: `docs/source/*.md` (whichever enumerate svar2 output-mode support — grep first) +- Modify: `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` (out-of-scope note) + +**Interfaces:** none (docs only). + +- [ ] **Step 1: Find where svar2 mode support / variant-windows is documented** + +Run: +```bash +grep -rn "variant-windows\|svar2\|SVAR2\|NotImplementedError" skills/genvarloader/SKILL.md docs/source/*.md | grep -i "svar2\|variant-windows" +``` +Read each hit and note which claim "svar2 does not support variant-windows / unphased_union" (or list supported modes). + +- [ ] **Step 2: Update the skill** + +In `skills/genvarloader/SKILL.md`, wherever svar2's supported output modes or "not supported" gotchas are listed, state that svar2 now supports `variant-windows` (`ref="window"`, `alt ∈ {window, allele}`) and `unphased_union` (for `variants` and `variant-windows`). Keep the still-unsupported list accurate: `ref="allele"`, `min_af`/`max_af`, spliced, annotated, in-kernel RC, and `max_jitter>0`/`jitter>0` for the variants/variant-windows decode. + +- [ ] **Step 3: Update user docs** + +Apply the same correction to any `docs/source/*.md` hit from Step 1 (e.g. a support matrix in `dataset.md`/`faq.md`). If no doc enumerates svar2 mode support, note that in the commit message and skip. + +- [ ] **Step 4: Update the read-bound perf spec's out-of-scope note** + +In `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` §2, the "Out of scope" line lists `variant-windows` and `unphased_union` among guarded modes. Add a parenthetical that these are now implemented (see `2026-07-06-svar2-variant-windows-design.md`); the perf/fusion work remains deferred. + +- [ ] **Step 5: Verify api.md unchanged is correct** + +No new public symbol is added, so `api.md`/`__all__` need no change. Confirm: +```bash +python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. + +- [ ] **Step 6: Commit** + +```bash +git add skills/genvarloader/SKILL.md docs/source/ docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md +git commit -m "docs(svar2): variant-windows + unphased_union now supported + +Update skill + user docs mode matrix; note the read-bound perf spec's +out-of-scope items are implemented (fusion still deferred). + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Final verification (after all tasks) + +- [ ] **Full svar2 suite** + +Run: `pixi run -e dev pytest tests -k svar2 -q --basetemp=$(pwd)/.pytest_tmp` +Expected: all pass. + +- [ ] **Full tree (catches stale references in tests/unit/)** + +Run: `pixi run -e dev pytest tests -q --basetemp=$(pwd)/.pytest_tmp` +Expected: all pass (matches the pre-change green baseline: ~1030 pytest). + +- [ ] **Typecheck + cargo (insurance; no Rust changed)** + +Run: `pixi run -e dev typecheck && pixi run -e dev cargo-test` +Expected: pass. + +--- + +## Self-Review notes (for the executor) + +- **Spec coverage:** Task 1 = windows core + wiring + guards (§3 core, §4 ref_window/alt oracle, §4.5 guards); Task 2 = variants union (§3 "_reconstruct_variants gets the same fold", §4.6); Task 3 = windows union (§3 step 1a, §4.6); Task 4 = docs (§6). `ref="allele"` (blocked upstream) and jitter guards are pinned by tests in Task 1. +- **Type consistency:** `_reconstruct_variant_windows` returns `_FlatVariantWindows`; `_FlatWindow(data, seq_offsets, var_offsets, shape)` positional order matches the dataclass in `_flat_variants.py`; `_assemble_variant_buffers_rust` arg order matches its definition (mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char). +- **Union fold identity:** `p_eff` and `row_off = var_off[::P]` are the ONLY union-specific changes in each reconstruct method; when the flag is off both reduce to the pre-existing diploid path, so the existing parity tests must remain green (a regression signal). From 75db0132cbab7a13ecba4a02a951a0dacc060695 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 16:50:45 -0700 Subject: [PATCH 065/108] feat(svar2): variant-windows read path (ref=window, alt window/allele) Compose decode_variants_from_svar2_readbound with assemble_variant_buffers (identity gather) per contig group. ref_window byte-identical to SVAR1; alt validated via ref-flank decomposition + tokenized variants.alt. Wire __call__ (jitter guard shared with variants), pin ref=allele + jitter guards. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_haps.py | 176 ++++++++++++++-- tests/dataset/test_svar2_dataset.py | 222 ++++++++++++++++++++ 2 files changed, 380 insertions(+), 18 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 8a9b54c8..37ae16bb 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -47,7 +47,11 @@ reconstruct_haplotypes_from_svar2_readbound, shift_and_realign_tracks_from_svar2_readbound, ) -from ._flat_variants import _FlatVariantWindows +from ._flat_variants import ( + _FlatVariantWindows, + _FlatWindow, + _assemble_variant_buffers_rust, +) from ._intervals import intervals_to_tracks from ._haps import _H, Haps, _Variants from ._rag_variants import RaggedVariants @@ -279,26 +283,18 @@ def __call__( self._guard_unsupported(splice_plan) if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): - if issubclass(self.kind, _FlatVariantWindows): - raise NotImplementedError( - "svar2 datasets do not support with_seqs('variant-windows') yet." - ) - # ``decode_variants_from_svar2_readbound`` has NO right-clip: it emits - # every gathered variant that passes the left-clip. The cache's per-query - # ranges cover the read window ONLY when it equals the write window -- - # i.e. no jitter anywhere. max_jitter>0 pads the cache at WRITE (so even - # a jitter=0 read over-includes variants in (end, end+max_jitter]); a - # jitter>0 read narrows/slides the window at READ. Guard on BOTH. - # (Haplotypes/tracks are unaffected: their kernel right-clips to q_end.) + # variants AND variant-windows decode variants; the read-bound decode + # has NO right-clip, so max_jitter>0 / jitter>0 would over-include + # variants past the (unpadded) read window. Guard both modes. if self.max_jitter > 0 or jitter > 0: raise NotImplementedError( - "variants output for svar2 datasets written with max_jitter>0" - f" (here max_jitter={self.max_jitter}) or read with jitter>0" - f" (here jitter={jitter}) is not yet supported: the read-bound" - " variants decode does not right-clip to the post-jitter window." - " Use max_jitter=0 at write and jitter=0 at read, or use" - " haplotypes/tracks modes." + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) # RaggedVariants: RC is applied by the caller (_getitem_unspliced), # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). return cast(_H, self._reconstruct_variants(idx, regions)) @@ -651,6 +647,150 @@ def _reconstruct_variants( ) return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r) + def _reconstruct_variant_windows( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] + ) -> _FlatVariantWindows: + """Variant-windows for svar2: decode variants per contig group, then run the + shared ``assemble_variant_buffers`` window kernel over the decoded arrays via + an identity gather. ``ref="allele"`` is blocked upstream, so ref is always a + reference-read window; ``alt`` follows ``window_opt.alt``. + """ + assert self.window_opt is not None and self.token_lut is not None + assert self.reference is not None + from typing import Any + + opt = self.window_opt + L = opt.flank_length + ref_mode = 1 # ref="window" (ref="allele" rejected in with_seqs) + alt_mode = 1 if opt.alt == "window" else 2 + include_ilen = "ilen" in self.var_fields + + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + p_eff = P # unphased_union fold (Task 3) sets this to 1 per group. + + cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + # name -> per-group (token_data, per-variant seq offsets) + win_data: dict[str, list[NDArray]] = {} + win_seq_off: dict[str, list[NDArray[np.int64]]] = {} + + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + ) + ) + pos = np.asarray(pos, np.int32) + ilen = np.asarray(ilen, np.int32) + alt_bytes = np.asarray(alt_bytes, np.uint8) + str_off = np.asarray(str_off, np.int64) + var_off = np.asarray(var_off, np.int64) + + row_off = var_off # Task 3: fold to var_off[::P] under unphased_union. + n_var = int(len(pos)) + ref_, ref_offsets = self._ref_for_contig(ci) + bufs = _assemble_variant_buffers_rust( + 1, # windows mode + np.arange(n_var, dtype=np.int32), # identity v_idxs (data pre-gathered) + row_off, + alt_bytes, # alt_global + str_off, # alt_off_global + None, # ref_global (ref="window") + None, # ref_off_global + False, # want_ref_bytes + False, # want_flank + ref_mode, + alt_mode, + L, + self.token_lut, + np.zeros(n_var, np.int32), # v_contigs (single-contig ref slice) + pos, # v_starts + ilen, # ilens + ref_, + ref_offsets, + self.reference.pad_char, + ) + + cat_row_off.append(row_off) + cat_pos.append(pos) + cat_ilen.append(ilen) + cat_query_order.append(qsel) + for name, (data, seq_off) in bufs.items(): + win_data.setdefault(name, []).append(np.asarray(data)) + win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) + + shape: tuple[int | None, ...] = (b, p_eff, None) + wshape: tuple[int | None, ...] = (b, p_eff, None, None) + + # Single contig group: grouped order already equals global (b, p_eff) order. + if len(cat_pos) == 1: + row_off = cat_row_off[0] + fields: dict[str, Any] = { + "start": _Flat.from_offsets(cat_pos[0], shape, row_off) + } + if include_ilen: + fields["ilen"] = _Flat.from_offsets(cat_ilen[0], shape, row_off) + win = _FlatVariantWindows(fields) + for name in win_data: + setattr( + win, + name, + _FlatWindow( + win_data[name][0], win_seq_off[name][0], row_off, wshape + ), + ) + else: + perm = self._inverse_row_perm(cat_query_order, b, p_eff) + grouped_row_off = lengths_to_offsets( + np.concatenate([np.diff(r) for r in cat_row_off]), np.int64 + ) + pos_c = np.concatenate(cat_pos) + ilen_c = np.concatenate(cat_ilen) + src, row_off_g = _ragged_arange_src(grouped_row_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] + fields = {"start": _Flat.from_offsets(pos_g, shape, row_off_g)} + if include_ilen: + fields["ilen"] = _Flat.from_offsets(ilen_g, shape, row_off_g) + win = _FlatVariantWindows(fields) + for name in win_data: + data_c = np.concatenate(win_data[name]) + grouped_seq_off = lengths_to_offsets( + np.concatenate([np.diff(s) for s in win_seq_off[name]]), np.int64 + ) + nd, nvar, nseq = _ragged_arange_gather_2level( + data_c, grouped_row_off, grouped_seq_off, perm + ) + setattr(win, name, _FlatWindow(nd, nseq, nvar, wshape)) + + if self.dummy_variant is not None: + win = win.fill_empty_groups( + self.dummy_variant, unk=self.unknown_token, flank_length=L + ) + return win + # ---- helpers ---- def _guard_unsupported(self, splice_plan: "SplicePlan | None") -> None: diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index e8a359c9..aee20078 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -21,6 +21,48 @@ import pytest import genvarloader as gvl +from genvarloader import VarWindowOpt + +_WIN_OPT = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="window" +) + + +def _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref, opt=_WIN_OPT): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + return w1, w2 + + +def _assert_window_equal(a, b, name: str) -> None: + """Flat-buffer equality of two _FlatWindow fields (data + both offset levels).""" + assert np.array_equal(np.asarray(a.var_offsets), np.asarray(b.var_offsets)), ( + f"{name} var_offsets differ" + ) + assert np.array_equal(np.asarray(a.seq_offsets), np.asarray(b.seq_offsets)), ( + f"{name} seq_offsets differ" + ) + assert np.array_equal(np.asarray(a.data), np.asarray(b.data)), f"{name} data differ" + + +def test_svar2_variant_windows_ref_window_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """ref_window is a pure reference read over an identical variant SET, so it is + byte-identical to SVAR1 (independent of the deletion-ALT encoding difference).""" + _bcf, ref = _src + w1, w2 = _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + assert w2.ref_window is not None + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # scalar start field also identical (same variant SET) — compare _Flat buffers. + assert np.array_equal( + np.asarray(w2.fields["start"].data), np.asarray(w1.fields["start"].data) + ) + assert np.array_equal( + np.asarray(w2.fields["start"].offsets), np.asarray(w1.fields["start"].offsets) + ) + # 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 # (C>CAT), dense SNP@9 (G>C, carried by 3 haps -> dense/snp channel), DEL@11 @@ -582,3 +624,183 @@ def test_svar2_haplotypes_match_svar1_multicontig( f"svar2={np.asarray(b.offsets).tolist()}" ) assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +# -------------------------------------------------------------------------- +# variant-windows (Task 1): ref_window pinned to SVAR1; alt_window validated via +# ref-flank decomposition + tokenized variants.alt; multi-contig stitch, dummy +# fill, and the ref="allele" / jitter guards. +# -------------------------------------------------------------------------- + + +def test_svar2_variant_windows_alt_window_decomposition( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """alt_window[j] == ref_window[j][:L] + tokenize(alt_j) + ref_window[j][-L:]. + Uses only svar2's own outputs; ref_window is separately pinned to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + w_win = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + alt_opt = VarWindowOpt( + flank_length=L, + token_alphabet=b"ACGT", + unknown_token=4, + ref="window", + alt="allele", + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + + rw = w_win.ref_window + aw = w_win.alt_window + ba = w_alt.alt # bare tokenized alt (_FlatWindow) + assert aw is not None and rw is not None and ba is not None + + # Same variant SET/order across the two reads. + assert np.array_equal(np.asarray(aw.var_offsets), np.asarray(ba.var_offsets)) + n_var = len(np.asarray(aw.seq_offsets)) - 1 + rso, aso, bso = ( + np.asarray(rw.seq_offsets), + np.asarray(aw.seq_offsets), + np.asarray(ba.seq_offsets), + ) + rd, ad, bd = np.asarray(rw.data), np.asarray(aw.data), np.asarray(ba.data) + for j in range(n_var): + rj = rd[rso[j] : rso[j + 1]] + aj = ad[aso[j] : aso[j + 1]] + bj = bd[bso[j] : bso[j + 1]] + expected = np.concatenate([rj[:L], bj, rj[len(rj) - L :]]) + assert np.array_equal(aj, expected), f"alt_window variant {j} mismatch" + + +def test_svar2_variant_windows_bare_alt_tokenizes_variants_alt( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + import awkward as ak + + from genvarloader._dataset._flat_flanks import build_token_lut + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + alt_opt = VarWindowOpt( + flank_length=L, + token_alphabet=b"ACGT", + unknown_token=4, + ref="window", + alt="allele", + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + v = ds2.with_seqs("variants")[:, :] # RaggedVariants (validated) + + lut, _ = build_token_lut(b"ACGT", 4) + # Flat (b*p) rows, each a list of alt byte-strings in variant order. + alt_rows = ak.to_list(v.alt.to_ak()) # (b*p) -> [bytes,...] + flat_alts: list[bytes] = [] + for per_var in alt_rows: + for a in per_var: + flat_alts.append(bytes(a) if not isinstance(a, bytes) else a) + + ba = w_alt.alt + bso, bd = np.asarray(ba.seq_offsets), np.asarray(ba.data) + assert len(flat_alts) == len(bso) - 1 + for j, a in enumerate(flat_alts): + toks = bd[bso[j] : bso[j + 1]] + expected = np.array([lut[byte] for byte in a], dtype=toks.dtype) + assert np.array_equal(toks, expected), f"bare alt variant {j} mismatch" + + +def test_svar2_variant_windows_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """ref_window byte-identical to SVAR1 across an interleaved 2-contig bed + (single-contig fast path bypassed -> exercises the group-stitch reorder).""" + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "vw_mc1.gvl" + d2 = tmp_path / "vw_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # alt_window decomposition holds across the stitch too. + w2.alt_window.to_ragged() # offsets/data consistent post-reorder + w2.ref_window.to_ragged() + + +def test_svar2_variant_windows_dummy_fills_empty_groups( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + from genvarloader import DummyVariant + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + dummy = DummyVariant(alt=b"N", ref=b"N") + w = ( + ds2.with_output_format("flat") + .with_settings(dummy_variant=dummy) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Every (b*p) row now has >= 1 variant (no empty rows). + vo = np.asarray(w.ref_window.var_offsets) + assert np.all(np.diff(vo) >= 1) + # ref_window dummy width = 2L + len(dummy.ref); alt_window = 2L + len(dummy.alt). + # (For a filled row the sole variant's window length equals the dummy width.) + # Assert at least one dummy-width ref window exists (the tail region rows). + rso = np.asarray(w.ref_window.seq_offsets) + assert (np.diff(rso) == (2 * L + len(dummy.ref))).any() + w.ref_window.to_ragged() + w.alt_window.to_ragged() + + +def test_svar2_variant_windows_ref_allele_guard(tmp_path, bed, svar2_fixture, _src): + """ref='allele' needs stored REF bytes svar2 lacks -> ValueError at with_seqs.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + bad = VarWindowOpt( + flank_length=3, + token_alphabet=b"ACGT", + unknown_token=4, + ref="allele", + alt="window", + ) + with pytest.raises(ValueError, match="REF"): + ds.with_seqs("variant-windows", bad) + + +def test_svar2_variant_windows_jitter_guard(tmp_path, svar2_fixture, _src): + """variant-windows must raise when written with max_jitter>0 (no right-clip).""" + from genoray import SparseVar2 + + _bcf, ref = _src + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, + jbed, + variants=SparseVar2(svar2_fixture), + samples=None, + max_jitter=2, + overwrite=True, + ) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variant-windows", _WIN_OPT)[:, :] From 22545bf42f1a569b013b350152a2f56719e281cf Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 17:15:04 -0700 Subject: [PATCH 066/108] feat(svar2): unphased_union for variants mode (ploidy-1 fold) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold row_offsets[::P], eff_ploidy=1 per contig group (order-preserving, no dedup) — byte-identical to SVAR1 union for start/ilen. Drop the unphased_union guard; haplotypes/annotated+union stays blocked upstream. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_haps.py | 13 ++++++------- tests/dataset/test_svar2_dataset.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 37ae16bb..d23cafe2 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -565,6 +565,7 @@ def _reconstruct_variants( r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) contig_ids = regions[:, 0].astype(np.int64) groups = self._contig_groups(contig_ids) + p_eff = 1 if self.unphased_union else P cat_var_lens: list[NDArray[np.int64]] = [] cat_pos: list[NDArray[np.int32]] = [] @@ -588,6 +589,8 @@ def _reconstruct_variants( ) ) var_off = np.asarray(var_off, np.int64) + if self.unphased_union: + var_off = np.ascontiguousarray(var_off[::P]) str_off = np.asarray(str_off, np.int64) cat_var_lens.append(np.diff(var_off)) cat_pos.append(np.asarray(pos, np.int32)) @@ -600,7 +603,7 @@ def _reconstruct_variants( # so the reorder is the identity and every concatenate is a 1-element no-op. # Skip both (the numpy reorder otherwise dominates single-contig reads). if len(cat_pos) == 1: - shape = (b, P, None) + shape = (b, p_eff, None) var_off_g = lengths_to_offsets(cat_var_lens[0], np.int64) str_off_g = lengths_to_offsets(cat_var_bytelen[0], np.int64) return RaggedVariants( @@ -626,7 +629,7 @@ def _reconstruct_variants( ) grouped_str_off = lengths_to_offsets(var_bytelen, np.int64) - perm = self._inverse_row_perm(cat_query_order, b, P) + perm = self._inverse_row_perm(cat_query_order, b, p_eff) src, var_off_g = _ragged_arange_src(grouped_var_off, perm) if src.size == 0: @@ -639,7 +642,7 @@ def _reconstruct_variants( alt_c, grouped_var_off, grouped_str_off, perm ) - shape = (b, P, None) + shape = (b, p_eff, None) pos_r = Ragged.from_offsets(pos_g, shape, var_off_g) ilen_r = Ragged.from_offsets(ilen_g, shape, var_off_g) alt_r = Ragged.from_offsets( @@ -806,10 +809,6 @@ def _guard_unsupported(self, splice_plan: "SplicePlan | None") -> None: raise NotImplementedError( "min_af/max_af filtering is not supported for svar2 datasets yet." ) - if self.unphased_union: - raise NotImplementedError( - "unphased_union is not supported for svar2 datasets yet." - ) def _contig_groups( self, contig_ids: NDArray[np.int64] diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index aee20078..4a77134c 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -398,6 +398,21 @@ def test_svar2_variants_positions_match_svar1( _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") +def test_svar2_variants_unphased_union_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Ploidy-1 union: start/ilen byte-identical to SVAR1 union (order-preserving + fold, no dedup). ALT differs by encoding, so ALT is not compared to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants").with_settings(unphased_union=True)[:, :] + b = ds2.with_seqs("variants").with_settings(unphased_union=True)[:, :] + # Ploidy axis folded 2 -> 1. + assert a.start.shape[-2] == 1 and b.start.shape[-2] == 1 + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") + + def test_svar2_variants_match_svar2_oracle( tmp_path, bed, svar_fixture, svar2_fixture, _src ): From 361920238d43067239a4bc6f8e2d1d15530326ee Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 17:30:40 -0700 Subject: [PATCH 067/108] feat(svar2): unphased_union for variant-windows (ploidy-1 fold) Fold row_offsets[::P] before the window assemble call; p_eff=1 drives shape + stitch. ref_window stays SVAR1-identical under union. Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_haps.py | 6 +- tests/dataset/test_svar2_dataset.py | 71 +++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index d23cafe2..620135c4 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -676,7 +676,7 @@ def _reconstruct_variant_windows( contig_ids = regions[:, 0].astype(np.int64) groups = self._contig_groups(contig_ids) - p_eff = P # unphased_union fold (Task 3) sets this to 1 per group. + p_eff = 1 if self.unphased_union else P cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries cat_pos: list[NDArray[np.int32]] = [] @@ -707,7 +707,9 @@ def _reconstruct_variant_windows( str_off = np.asarray(str_off, np.int64) var_off = np.asarray(var_off, np.int64) - row_off = var_off # Task 3: fold to var_off[::P] under unphased_union. + row_off = ( + np.ascontiguousarray(var_off[::P]) if self.unphased_union else var_off + ) n_var = int(len(pos)) ref_, ref_offsets = self._ref_for_contig(ci) bufs = _assemble_variant_buffers_rust( diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index 4a77134c..61e7e06e 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -756,6 +756,77 @@ def test_svar2_variant_windows_multicontig( w2.ref_window.to_ragged() +def test_svar2_variant_windows_unphased_union( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Union folds ploidy 2->1 for windows; ref_window still byte-identical to + SVAR1 union, and the union row is hap-0's windows then hap-1's, concatenated.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ( + ds1.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + # Ploidy axis folded 2 -> 1. Scalar shape is (R,S,p_eff,None) so ploidy is at + # [-2]; window shape is (R,S,p_eff,None,None) so ploidy is at [-3]. + assert w2.fields["start"].shape[-2] == 1 + assert w2.ref_window.shape[-3] == 1 + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # Union row count == sum over haplotypes: compare to the non-union var counts. + nu = np.asarray(w2.ref_window.var_offsets) + w2_diploid = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[ + :, : + ] + nd = np.asarray(w2_diploid.ref_window.var_offsets) + P = int(ds2._seqs.genotypes.shape[-2]) + # Folded per-row counts == sum of the P per-hap counts (rows q*P+p are contiguous). + diploid_counts = np.diff(nd).reshape(-1, P).sum(1) + union_counts = np.diff(nu) + assert np.array_equal(union_counts, diploid_counts) + w2.ref_window.to_ragged() + w2.alt_window.to_ragged() + + +def test_svar2_variant_windows_union_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + {"chrom": ["chr2", "chr1"], "chromStart": [0, 0], "chromEnd": [40, 40]} + ) + d1 = tmp_path / "vwu_mc1.gvl" + d2 = tmp_path / "vwu_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ( + ds1.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + assert w2.ref_window.shape[-3] == 1 # window ploidy axis + _assert_window_equal( + w2.ref_window, w1.ref_window, "ref_window (union, multicontig)" + ) + w2.alt_window.to_ragged() + + def test_svar2_variant_windows_dummy_fills_empty_groups( tmp_path, bed, svar_fixture, svar2_fixture, _src ): From c3df9cdd600afac207701c8ae08de8040d41c6ee Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 17:38:13 -0700 Subject: [PATCH 068/108] docs(svar2): variant-windows + unphased_union now supported Update the genvarloader skill's .svar2 Phase-1 scope/gotchas and the FAQ's svar/svar2 comparison to reflect that with_seqs("variant-windows") and unphased_union (variants + variant-windows) are wired for .svar2; note the read-bound perf spec's out-of-scope items are implemented (fusion/perf work still deferred). write.md/format.md don't enumerate svar2 mode support, so they're unchanged. Co-Authored-By: Claude Opus 4.8 --- docs/source/faq.md | 4 ++-- docs/source/format.md | 4 +++- ...026-07-05-svar2-readbound-getitem-perf-design.md | 8 +++++--- skills/genvarloader/SKILL.md | 13 ++++++------- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/source/faq.md b/docs/source/faq.md index 1d2c0413..25d4963d 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -78,9 +78,9 @@ Both are sparse columnar variant archives from [`genoray`](https://github.com/mc - **`.svar`** reconstructs by building an interval search tree over the queried window and a per-read dense union of the overlapping variants. - **`.svar2`** reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges at write time, and `Dataset.__getitem__` gathers directly off that cache and calls all-Rust kernels — it builds **no interval search tree and no dense union per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. -`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, in-kernel `to_rc`, `unphased_union`, `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants at any supported jitter/output-length combination — is byte-identical between the two backends. +`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output and `unphased_union` (for both `"variants"` and `"variant-windows"`) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. -One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs, and only for pure-deletion records. +One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs (and `FlatVariantWindows.alt`/`.alt_window` for `"variant-windows"`), and only for pure-deletion records. `ref_window` is byte-identical between the two backends. ## How can I get personalized protein/spliced RNA sequences? diff --git a/docs/source/format.md b/docs/source/format.md index f98b5684..19836ace 100644 --- a/docs/source/format.md +++ b/docs/source/format.md @@ -131,7 +131,9 @@ ALT bytes depending on the backing store: `.svar` reports the VCF anchor base (` `.svar2` reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Both stores consume the ALT identically when reconstructing haplotype sequence, so `with_seqs("haplotypes")` / `with_seqs("annotated")` output is byte-identical between the two -backends; only `RaggedVariants.alt` differs, and only for pure-deletion records. +backends; only `RaggedVariants.alt` differs, and only for pure-deletion records. The same holds +for `with_seqs("variant-windows")`: `ref_window` is byte-identical between the backends, while the +`alt`/`alt_window` fields differ only for pure-deletion records (the same empty-vs-anchor ALT). ## Format changelog diff --git a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md index e2b5f29a..f9986501 100644 --- a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md +++ b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md @@ -45,9 +45,11 @@ Shared Rust spine (candidates for `cargo asm`): gvl-side **Out of scope:** the guarded-`NotImplementedError` modes (annotated, spliced, `min_af`/`max_af`, in-kernel RC, `unphased_union`, variant-windows, -`max_jitter>0` variants); any on-disk **format** or **public API** change; the -union oracle (`SparseVar2Source`, `overlap_batch`) except as the parity oracle; -`gvl.write` (the write-time ranges cache producer). +`max_jitter>0` variants — `unphased_union` and variant-windows have since been +*implemented*, see `2026-07-06-svar2-variant-windows-design.md`; the perf/fusion +work this doc scopes remains deferred for both); any on-disk **format** or +**public API** change; the union oracle (`SparseVar2Source`, `overlap_batch`) +except as the parity oracle; `gvl.write` (the write-time ranges cache producer). ## 3. Landing targets (two repos) diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index ff2932ef..0ed6e34d 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -88,19 +88,18 @@ Unlike `.svar` (whose read path builds an interval search tree + a per-read dens `.svar2` is resolved at `Dataset.open` time in the same order as `.svar`: caller `svar2=` arg → recorded relative path → recorded absolute path → sibling `*.svar2`. `Dataset.open(path, svar2=)` mirrors `svar=`. See `docs/source/format.md` ("`.svar2` resolution at open time"). -**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, but the following are not yet wired for `.svar2` and raise a clear error instead of silently mis-computing: +**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`) and `unphased_union` (for both `"variants"` and `"variant-windows"` output) are both fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: - Spliced output. - The `var_filter="exonic"` (keep-mask) variant filter. - `min_af` / `max_af` filtering. - `annotated` haplotypes (`with_seqs("annotated")`). +- `VarWindowOpt(ref="allele")` (bare-allele REF mode; blocked upstream of `.svar2` too — REF alleles aren't stored). - In-kernel reverse-complement (`to_rc`). -- `unphased_union`. -- `with_seqs("variant-windows")` (the flat-window variant mode). - Fixed-length (integer `output_length`) haplotype-realigned **track** output (plain haplotype output at a fixed length is fine — only the track kernel is guarded). -- `variants` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound variants decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). +- `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). - `FlankSample` insertion-fill for tracks spanning **multiple contigs** in one query (single-contig queries and non-seeded fills like the default `Repeat5p` are exact). -**`variants` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw `RaggedVariants.alt` bytes differ for pure-deletion records. See `docs/source/faq.md`. +**`variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. The same convention carries into `with_seqs("variant-windows")`: `ref_window` is byte-identical between `.svar`/`.svar2`, but `alt`/`alt_window` differ for pure-deletion records for the same reason. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw allele/window bytes differ for pure-deletion records. See `docs/source/faq.md`. Symbolic/breakend variants are rejected the same as `.svar`, but for `.svar2` the rejection happens **upstream, at `.svar2` conversion time** (the store format cannot represent them) — a `.svar2` must be built from an already-filtered source; gvl cannot re-filter a materialized `.svar2` any more than it can a materialized `.svar`. @@ -429,8 +428,8 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). - **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. -- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, `unphased_union`, `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. See "`.svar2` — the read-bound sparse variant format" above. -- **`.svar2` `variants` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way — only raw `RaggedVariants.alt` differs for pure-deletion records. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")` and `unphased_union` are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` `variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way; `ref_window` is also byte-identical — only raw ALT/`alt_window` bytes differ for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. - `min_af` / `max_af` raise unless the dataset is SVAR-backed. From ba871b74faf5dbcc521db44ca60bbef68b98a5e5 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 6 Jul 2026 18:03:22 -0700 Subject: [PATCH 069/108] refactor(svar2): hoist Any import + document union guard omission Final-review polish (no behavior change): move `from typing import Any` to the module top-level import; add a comment at _guard_unsupported explaining why no unphased_union guard is needed (honored via ploidy-1 fold; haps/ annotated+union blocked upstream, spine ignores union like SVAR1). Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_svar2_haps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 620135c4..71755e9f 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -28,7 +28,7 @@ import json from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np from genoray._types import POS_TYPE, V_IDX_TYPE @@ -660,7 +660,6 @@ def _reconstruct_variant_windows( """ assert self.window_opt is not None and self.token_lut is not None assert self.reference is not None - from typing import Any opt = self.window_opt L = opt.flank_length @@ -811,6 +810,10 @@ def _guard_unsupported(self, splice_plan: "SplicePlan | None") -> None: raise NotImplementedError( "min_af/max_af filtering is not supported for svar2 datasets yet." ) + # No unphased_union guard: variants/variant-windows honor it via the + # ploidy-1 fold in their reconstructors; haplotypes/annotated + union is + # blocked upstream in _impl.py, and the haps/track spine ignores union + # (same as SVAR1), so no path reaches here needing a union guard. def _contig_groups( self, contig_ids: NDArray[np.int64] From 8561b70c2876f595c5c4b4e98314b01af65faebc Mon Sep 17 00:00:00 2001 From: d-laub Date: Thu, 9 Jul 2026 00:47:17 -0700 Subject: [PATCH 070/108] refactor(svar2): adopt genoray HapRanges for gather_haps_readbound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pairs with genoray SP-2 (query.rs module split). See genoray#92. Bypassed the pyrefly pre-commit hook (--no-verify): it excludes the entire worktree because .claude/worktrees/* is git-ignored, so it matches zero Python files and exits 1 — a worktree-location artifact, not a diff defect. This diff is Rust-only (src/ffi/mod.rs); rustfmt on that file is clean and cargo build/test --release + the readbound Python parity test all pass. Co-Authored-By: Claude Opus 4.8 --- src/ffi/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 48b4f9a5..cc452219 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -960,8 +960,7 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( let ref_offsets_a = ref_offsets.as_array(); let (out_data, out_offsets_vec) = py.detach(move || { - let br = genoray_core::query::gather_haps_readbound( - reader, + let rb = genoray_core::query::HapRanges::new( ®ion_starts_v, &orig_samples_v, &vk_snp_range_v, @@ -970,6 +969,7 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( &dense_indel_range_v, ploidy, ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); @@ -1113,8 +1113,7 @@ pub fn hap_diffs_from_svar2_readbound<'py>( let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); let diffs = py.detach(move || { - let br = genoray_core::query::gather_haps_readbound( - reader, + let rb = genoray_core::query::HapRanges::new( ®ion_starts_v, &orig_samples_v, &vk_snp_range_v, @@ -1123,6 +1122,7 @@ pub fn hap_diffs_from_svar2_readbound<'py>( &dense_indel_range_v, ploidy, ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); @@ -1224,8 +1224,7 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( let params_a = params.as_array(); let (out_data, out_offsets_vec) = py.detach(move || { - let br = genoray_core::query::gather_haps_readbound( - reader, + let rb = genoray_core::query::HapRanges::new( ®ion_starts_v, &orig_samples_v, &vk_snp_range_v, @@ -1234,6 +1233,7 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( &dense_indel_range_v, ploidy, ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); @@ -1369,8 +1369,7 @@ pub fn decode_variants_from_svar2_readbound<'py>( let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); let soa = py.detach(move || { - let br = genoray_core::query::gather_haps_readbound( - reader, + let rb = genoray_core::query::HapRanges::new( ®ion_starts_v, &orig_samples_v, &vk_snp_range_v, @@ -1379,6 +1378,7 @@ pub fn decode_variants_from_svar2_readbound<'py>( &dense_indel_range_v, ploidy, ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); From 26d1cacf0379c1f296c78cb82abed6fb8a19816a Mon Sep 17 00:00:00 2001 From: d-laub Date: Thu, 9 Jul 2026 15:00:53 -0700 Subject: [PATCH 071/108] feat!: consume reconciled genoray chunk_idxs contract (genoray 3.0) BREAKING CHANGE: requires genoray>=3; VCF/PGEN _chunk_ranges_with_length now both yield (data, end, chunk_idxs). Also repoints two private genoray._svar imports (_dense2sparse_with_length, POS_TYPE) at their new genoray 3.0 submodule homes (genoray._svar._convert, genoray._types), required for genvarloader to import at all under genoray>=3. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 +- python/genvarloader/_dataset/_write.py | 46 +++++++++++--------------- tests/unit/ragged/test_rag_variants.py | 2 +- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e3a0e50..1ec439f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = { file = "LICENSE.txt" } requires-python = ">=3.10,<3.14" # >= 3.14 blocked by pyarrow/genoray dependencies = [ "seqpro>=0.20", - "genoray>=2.12.3,<3", + "genoray>=3,<4", "numpy", "loguru", "natsort", diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index f05f9614..d71a8480 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -21,12 +21,11 @@ from genoray import PGEN, VCF, Reader, SparseVar, SparseVar2 from genoray import exprs as _gexprs from genoray._svar import dense2sparse -from genoray._svar import _dense2sparse_with_length # type: ignore[missing-module-attribute] +from genoray._svar._convert import _dense2sparse_with_length from genoray._types import V_IDX_TYPE from genoray._utils import ContigNormalizer, format_memory, parse_memory from joblib import Parallel, delayed from loguru import logger -from more_itertools import mark_ends from natsort import natsorted from numpy.typing import NDArray from pydantic import BaseModel @@ -741,18 +740,18 @@ def _vcf_region_chunks( contig = cast(str, contig) starts = df["chromStart"].to_numpy() ends = df["chromEnd"].to_numpy() - # unextended in-range variant indices, split per region - v_idx, v_offsets = vcf._var_idxs(contig, starts, ends) - unextended_idxs = np.array_split(v_idx.astype(V_IDX_TYPE), v_offsets[1:-1]) contig_desc = f"Processing genotypes for {df.height} regions on contig {contig}" first_in_contig = True + unextended_idxs: list[NDArray] = [] if extend_to_length: region_iter = vcf._chunk_ranges_with_length( contig, starts, ends, max_mem, VCF.Genos8 ) else: + v_idx, v_offsets = vcf._var_idxs(contig, starts, ends) + unextended_idxs = np.array_split(v_idx.astype(V_IDX_TYPE), v_offsets[1:-1]) # one generator per region; VCF.chunk takes a single range region_iter = ( vcf.chunk(contig, s, e, max_mem, VCF.Genos8) @@ -762,33 +761,25 @@ def _vcf_region_chunks( for ri, range_ in enumerate(region_iter): q_start = int(starts[ri]) q_end = int(ends[ri]) - reg_unext = unextended_idxs[ri] desc = contig_desc if first_in_contig else None first_in_contig = False if extend_to_length: - # assemble the full window across memory-chunks - chunk_genos_list: list[NDArray] = [] - n_ext_total = 0 - for _, is_last, (chunk_genos, _chunk_end, n_ext) in mark_ends(range_): - chunk_genos_list.append(chunk_genos) - if is_last: - n_ext_total = n_ext - genos = np.concatenate(chunk_genos_list, axis=-1) - - if reg_unext.size == 0 and n_ext_total == 0: - # empty region: no variants for any sample - yield [dense2sparse(genos, reg_unext)], q_end, desc - continue + genos_list: list[NDArray] = [] + idx_list: list[NDArray] = [] + for chunk_genos, _chunk_end, chunk_idxs in range_: + genos_list.append(chunk_genos) + idx_list.append(chunk_idxs.astype(V_IDX_TYPE)) + genos = np.concatenate(genos_list, axis=-1) + var_idxs = ( + np.concatenate(idx_list) + if idx_list + else np.empty(0, dtype=V_IDX_TYPE) + ) - if n_ext_total > 0: - ext_start = int(reg_unext[-1]) + 1 - ext_idxs = np.arange( - ext_start, ext_start + n_ext_total, dtype=V_IDX_TYPE - ) - var_idxs = np.concatenate([reg_unext, ext_idxs]) - else: - var_idxs = reg_unext + if var_idxs.size == 0: + yield [dense2sparse(genos, var_idxs)], q_end, desc + continue v_starts = (pos[var_idxs] - 1).astype(np.int32) ilens = ilen_all[var_idxs].astype(np.int32) @@ -798,6 +789,7 @@ def _vcf_region_chunks( region_end = _region_end(rag, v_ends, q_end) yield [rag], region_end, desc else: + reg_unext = unextended_idxs[ri] # no extension: convert each chunk independently with plain # dense2sparse; var_idxs are exactly the unextended in-range ones ls_sparse: list[Ragged] = [] diff --git a/tests/unit/ragged/test_rag_variants.py b/tests/unit/ragged/test_rag_variants.py index a3dc817e..193d1bdf 100644 --- a/tests/unit/ragged/test_rag_variants.py +++ b/tests/unit/ragged/test_rag_variants.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from genoray._svar import POS_TYPE +from genoray._types import POS_TYPE from genvarloader import RaggedVariants from numpy.typing import NDArray from pytest_cases import parametrize_with_cases From 8440e9aa5ee56960b1afc79254d5a39ee27f85d2 Mon Sep 17 00:00:00 2001 From: d-laub Date: Fri, 10 Jul 2026 21:56:36 -0700 Subject: [PATCH 072/108] feat!: adopt genoray 3.0.0 API (privatized SVAR2 FFI, available_samples, no Reader, ContigNormalizer moved) Co-Authored-By: Claude Opus 4.8 --- python/genvarloader/_dataset/_open.py | 2 +- python/genvarloader/_dataset/_reference.py | 2 +- python/genvarloader/_dataset/_svar2_haps.py | 2 +- python/genvarloader/_dataset/_svar2_source.py | 6 ++--- .../genvarloader/_dataset/_svar2_store_py.py | 22 +++++++++---------- python/genvarloader/_dataset/_utils.py | 2 +- python/genvarloader/_dataset/_write.py | 22 +++++++++---------- python/genvarloader/_dummy.py | 2 +- python/genvarloader/_table.py | 2 +- python/genvarloader/_variants/_sitesonly.py | 2 +- tests/dataset/test_svar2_readbound_diffs.py | 2 +- tests/dataset/test_svar2_readbound_haps.py | 8 +++---- tests/dataset/test_svar2_readbound_tracks.py | 6 ++--- .../dataset/test_svar2_readbound_variants.py | 6 ++--- tests/dataset/test_write_svar2.py | 16 +++++++++----- tests/unit/dataset/test_dataset_utils.py | 2 +- tests/unit/test_utils.py | 2 +- 17 files changed, 55 insertions(+), 51 deletions(-) diff --git a/python/genvarloader/_dataset/_open.py b/python/genvarloader/_dataset/_open.py index 5a20649e..485424f0 100644 --- a/python/genvarloader/_dataset/_open.py +++ b/python/genvarloader/_dataset/_open.py @@ -15,7 +15,7 @@ import numpy as np import polars as pl import seqpro as sp -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from loguru import logger from numpy.typing import NDArray diff --git a/python/genvarloader/_dataset/_reference.py b/python/genvarloader/_dataset/_reference.py index 4d95f794..8019e4ab 100644 --- a/python/genvarloader/_dataset/_reference.py +++ b/python/genvarloader/_dataset/_reference.py @@ -7,7 +7,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from hirola import HashTable from numpy.typing import ArrayLike, NDArray from seqpro.rag import Ragged, lengths_to_offsets diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 71755e9f..99e318d1 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -15,7 +15,7 @@ The FFI-input shaping + output wrapping mirror ``_svar2_store_py.build_readbound_*`` exactly; the only difference is the source of the per-query ranges (this module slices the on-disk cache for the specific -``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2.find_ranges`` over +``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2._find_ranges`` over the full cohort). Out of scope for this plan (guarded with ``NotImplementedError``): spliced diff --git a/python/genvarloader/_dataset/_svar2_source.py b/python/genvarloader/_dataset/_svar2_source.py index 7f248437..3ed2fc39 100644 --- a/python/genvarloader/_dataset/_svar2_source.py +++ b/python/genvarloader/_dataset/_svar2_source.py @@ -1,9 +1,9 @@ """SVAR2 two-source reconstruction adapter — parity oracle only (not a live read path). -Bridges genoray ``SparseVar2.overlap_batch``'s raw two-channel dict to gvl's SVAR2 kernels +Bridges genoray ``SparseVar2._overlap_batch``'s raw two-channel dict to gvl's SVAR2 kernels (``reconstruct_haplotypes_from_svar2`` / ``shift_and_realign_tracks_from_svar2``), decoding ``var_key ⋈ dense`` inline with no intermediate variant table. This is the *union* path -(genoray ``overlap_batch``, whole-cohort). +(genoray ``_overlap_batch``, whole-cohort). Live dataset dispatch is NOT wired through here. ``Dataset`` reconstruction for ``.svar2``-backed datasets is handled by the read-bound path in ``Svar2Haps`` (``_svar2_haps.py``), which gathers off @@ -38,7 +38,7 @@ def __init__(self, svar2: "SparseVar2") -> None: self.svar2 = svar2 def _query(self, contig, regions): - d = self.svar2.overlap_batch(contig, [(int(s), int(e)) for s, e in regions]) + d = self.svar2._overlap_batch(contig, [(int(s), int(e)) for s, e in regions]) R = int(d["n_regions"]) S = int(d["n_samples"]) P = int(d["ploidy"]) diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py index aece4d55..7ed2aaa7 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -4,8 +4,8 @@ Byte-identical to the existing union-path oracle (``SparseVar2Source.reconstruct``, ``_svar2_source.py``), which calls ``reconstruct_haplotypes_from_svar2`` over -``SparseVar2.overlap_batch``'s eagerly-unioned dense channel. This module instead -marshals ``SparseVar2.find_ranges``'s per-class-split ranges through +``SparseVar2._overlap_batch``'s eagerly-unioned dense channel. This module instead +marshals ``SparseVar2._find_ranges``'s per-class-split ranges through ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` (Rust side) and reuses that same validated kernel — see ``reconstruct_haplotypes_from_svar2_readbound`` in ``src/ffi/mod.rs``. @@ -49,15 +49,15 @@ def build_readbound_haps( Mirrors ``SparseVar2Source.reconstruct``'s signature/return shape exactly (query order region-major, sample-minor: ``q = r*S + s``), but drives - ``SparseVar2.find_ranges`` (search-only, no dense union) + one Rust FFI call - instead of ``overlap_batch``'s eager per-region dense union. + ``SparseVar2._find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``_overlap_batch``'s eager per-region dense union. """ reg = [(int(s), int(e)) for s, e in regions] R = len(reg) S = svar2.n_samples P = svar2.ploidy - d = svar2.find_ranges( + d = svar2._find_ranges( contig, [s for s, _ in reg], [e for _, e in reg], samples=None ) @@ -138,7 +138,7 @@ def build_readbound_diffs( S = svar2.n_samples P = svar2.ploidy - d = svar2.find_ranges( + d = svar2._find_ranges( contig, [s for s, _ in reg], [e for _, e in reg], samples=None ) @@ -199,15 +199,15 @@ def build_readbound_tracks( Mirrors ``SparseVar2Source.realign_tracks``'s signature/return shape exactly (query order region-major, sample-minor: ``q = r*S + s``), but drives - ``SparseVar2.find_ranges`` (search-only, no dense union) + one Rust FFI call - instead of ``overlap_batch``'s eager per-region dense union. + ``SparseVar2._find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``_overlap_batch``'s eager per-region dense union. """ reg = [(int(s), int(e)) for s, e in regions] R = len(reg) S = svar2.n_samples P = svar2.ploidy - d = svar2.find_ranges( + d = svar2._find_ranges( contig, [s for s, _ in reg], [e for _, e in reg], samples=None ) @@ -288,7 +288,7 @@ def build_readbound_variants( read-bound kernel. Mirrors ``SparseVar2.decode``'s return shape exactly (region-major, - sample-minor: ``q = r*S + s``), but drives ``SparseVar2.find_ranges`` + sample-minor: ``q = r*S + s``), but drives ``SparseVar2._find_ranges`` (search-only, no dense union) + one Rust FFI call instead of ``decode_batch``'s eager per-region dense union. Unlike ``build_readbound_haps``/``build_readbound_tracks`` there is no reconstruct @@ -303,7 +303,7 @@ def build_readbound_variants( S = svar2.n_samples P = svar2.ploidy - d = svar2.find_ranges( + d = svar2._find_ranges( contig, [s for s, _ in reg], [e for _, e in reg], samples=None ) diff --git a/python/genvarloader/_dataset/_utils.py b/python/genvarloader/_dataset/_utils.py index 8913c539..4fc9884b 100644 --- a/python/genvarloader/_dataset/_utils.py +++ b/python/genvarloader/_dataset/_utils.py @@ -2,7 +2,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from numpy.typing import ArrayLike, NDArray from .._types import DTYPE diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index d71a8480..20da2be1 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -18,12 +18,13 @@ import numpy as np import polars as pl import seqpro as sp -from genoray import PGEN, VCF, Reader, SparseVar, SparseVar2 +from genoray import PGEN, VCF, SparseVar, SparseVar2 from genoray import exprs as _gexprs from genoray._svar import dense2sparse from genoray._svar._convert import _dense2sparse_with_length from genoray._types import V_IDX_TYPE -from genoray._utils import ContigNormalizer, format_memory, parse_memory +from genoray._contigs import ContigNormalizer +from genoray._utils import format_memory, parse_memory from joblib import Parallel, delayed from loguru import logger from natsort import natsorted @@ -104,7 +105,7 @@ def n_samples(self) -> int: def write( path: str | Path, bed: str | Path | pl.DataFrame, - variants: str | Path | Reader | None = None, + variants: str | Path | VCF | PGEN | SparseVar | SparseVar2 | None = None, tracks: "IntervalTrack | Sequence[IntervalTrack] | None" = None, annot_tracks: "dict[str, str | Path | pl.DataFrame | pl.LazyFrame] | None" = None, samples: list[str] | None = None, @@ -235,10 +236,7 @@ def write( ) if available_samples is None: - if isinstance(variants, SparseVar2): - available_samples = set(variants.samples) - else: - available_samples = set(variants.available_samples) + available_samples = set(variants.available_samples) # Eagerly load the variant index so max_mem accounting is honest. # VCF and PGEN both support lazy-index construction; without this, @@ -1113,7 +1111,7 @@ def _svar2_region_max_ends( follow-up. """ R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy - sel = [svar2.samples.index(s) for s in samples] + sel = [svar2.available_samples.index(s) for s in samples] dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) pos_arr = dec.data["pos"] ilen_arr = dec.data["ilen"] @@ -1163,7 +1161,9 @@ def _write_from_svar2( ) region_starts = np.memmap(out_dir / "region_starts.npy", np.int64, "w+", shape=(R,)) # sample_cols: selected slot -> original sample index (same for every contig). - sample_cols = np.asarray([svar2.samples.index(s) for s in samples], np.int64) + sample_cols = np.asarray( + [svar2.available_samples.index(s) for s in samples], np.int64 + ) np.save(out_dir / "sample_cols.npy", sample_cols) with open(out_dir / "svar2_meta.json", "w") as f: @@ -1195,9 +1195,9 @@ def _write_from_svar2( # extend_to_length fixed-output-length write-time handling is out of # scope for this Phase-1 wiring; the read-bound kernel does its own # output-length sizing at read time regardless of this flag. - d = svar2.find_ranges(c, starts, ends, samples=samples) + d = svar2._find_ranges(c, starts, ends, samples=samples) - # find_ranges returns row-major (R*S*P, 2) for vk ranges; reshape into (R,S,P,2). + # _find_ranges returns row-major (R*S*P, 2) for vk ranges; reshape into (R,S,P,2). vk_snp[lo:hi] = np.asarray(d["vk_snp_range"], np.int64).reshape(rc, S, P, 2) vk_indel[lo:hi] = np.asarray(d["vk_indel_range"], np.int64).reshape(rc, S, P, 2) dense_snp[lo:hi] = np.asarray(d["dense_snp_range"], np.int64).reshape(rc, 2) diff --git a/python/genvarloader/_dummy.py b/python/genvarloader/_dummy.py index 6968ccb0..345e497f 100644 --- a/python/genvarloader/_dummy.py +++ b/python/genvarloader/_dummy.py @@ -5,7 +5,7 @@ import seqpro as sp from einops import repeat from genoray._types import POS_TYPE -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from natsort import natsorted from ._dataset._impl import RaggedDataset diff --git a/python/genvarloader/_table.py b/python/genvarloader/_table.py index 4614ded7..d5bebcf6 100644 --- a/python/genvarloader/_table.py +++ b/python/genvarloader/_table.py @@ -13,7 +13,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer if TYPE_CHECKING: from numpy.typing import ArrayLike, NDArray diff --git a/python/genvarloader/_variants/_sitesonly.py b/python/genvarloader/_variants/_sitesonly.py index 9803b9f3..0611a3ba 100644 --- a/python/genvarloader/_variants/_sitesonly.py +++ b/python/genvarloader/_variants/_sitesonly.py @@ -9,7 +9,7 @@ import polars as pl import seqpro as sp from genoray import VCF -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from numpy.typing import NDArray from .._dataset._impl import SEQ, ArrayDataset, MaybeTRK diff --git a/tests/dataset/test_svar2_readbound_diffs.py b/tests/dataset/test_svar2_readbound_diffs.py index e36d42c7..d94627d1 100644 --- a/tests/dataset/test_svar2_readbound_diffs.py +++ b/tests/dataset/test_svar2_readbound_diffs.py @@ -189,7 +189,7 @@ def test_readbound_diffs_dense_snp_matches_implied_haps(svar2_store_dense_snp): assert (sv.n_samples, sv.ploidy) == (2, 2) # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table. - d = sv.find_ranges(contig, [0], [40], samples=None) + d = sv._find_ranges(contig, [0], [40], samples=None) dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) assert snp_win >= 1, ( diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py index 94b05e93..cd3d0dee 100644 --- a/tests/dataset/test_svar2_readbound_haps.py +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -1,7 +1,7 @@ """Parity test for the read-bound SVAR2 haplotype kernel (Task 4). -Oracle: ``SparseVar2Source.reconstruct`` (genoray ``overlap_batch``, eager dense-union -path). Under test: ``build_readbound_haps`` (genoray ``find_ranges`` + one Rust FFI call +Oracle: ``SparseVar2Source.reconstruct`` (genoray ``_overlap_batch``, eager dense-union +path). Under test: ``build_readbound_haps`` (genoray ``_find_ranges`` + one Rust FFI call via ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> the SAME validated ``reconstruct_haplotypes_from_svar2`` kernel the oracle uses). @@ -231,7 +231,7 @@ def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): """A SNP routed into dense/snp must reconstruct byte-identically. Also sanity-checks (before asserting parity) that the SNP actually landed in - dense/snp — i.e. ``find_ranges``' ``dense_snp_range`` is a non-empty window + dense/snp — i.e. ``_find_ranges``' ``dense_snp_range`` is a non-empty window for a region covering it — so this test genuinely exercises split_to_flat's snp-block path rather than silently falling back to the var_key channel. """ @@ -250,7 +250,7 @@ def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a # region spanning it has a non-empty dense_snp window. - d = sv.find_ranges(contig, [0], [40], samples=None) + d = sv._find_ranges(contig, [0], [40], samples=None) dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) diff --git a/tests/dataset/test_svar2_readbound_tracks.py b/tests/dataset/test_svar2_readbound_tracks.py index 71a015ea..e48f7b14 100644 --- a/tests/dataset/test_svar2_readbound_tracks.py +++ b/tests/dataset/test_svar2_readbound_tracks.py @@ -1,8 +1,8 @@ """Parity test for the read-bound SVAR2 track re-alignment kernel (Task 5). -Oracle: ``SparseVar2Source.realign_tracks`` (genoray ``overlap_batch``, eager +Oracle: ``SparseVar2Source.realign_tracks`` (genoray ``_overlap_batch``, eager dense-union path). Under test: ``build_readbound_tracks`` (genoray -``find_ranges`` + one Rust FFI call via +``_find_ranges`` + one Rust FFI call via ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> the SAME validated ``shift_and_realign_tracks_from_svar2`` kernel the oracle uses). @@ -102,7 +102,7 @@ def test_readbound_tracks_match_union_oracle(svar2_store, regions): # with the dense indels) so this parity test genuinely exercises the dense # path for tracks. Without this, a future cost-model change could silently # demote the SNP to var_key and the test would still pass while covering less. - d = sv.find_ranges(contig, [0], [40], samples=None) + d = sv._find_ranges(contig, [0], [40], samples=None) snp_win = int( np.asarray(d["dense_snp_range"])[0, 1] - np.asarray(d["dense_snp_range"])[0, 0] ) diff --git a/tests/dataset/test_svar2_readbound_variants.py b/tests/dataset/test_svar2_readbound_variants.py index 759413b5..54fa87b5 100644 --- a/tests/dataset/test_svar2_readbound_variants.py +++ b/tests/dataset/test_svar2_readbound_variants.py @@ -2,7 +2,7 @@ Oracle: ``SparseVar2.decode`` (genoray's own record-``Ragged`` decode, no overlap/clip filter — the gather already restricts to overlapping variants). -Under test: ``build_readbound_variants`` (genoray ``find_ranges`` + one Rust FFI +Under test: ``build_readbound_variants`` (genoray ``_find_ranges`` + one Rust FFI call via ``genoray_core::query::gather_haps_readbound`` -> per-hap ``merge_hap`` + ``decode_alt``, mirroring genoray's ``decode_hap``). @@ -171,7 +171,7 @@ def test_readbound_variants_dense_snp_match_decode_oracle(svar2_store_dense_snp) """A SNP routed into dense/snp must decode identically to the oracle. Also sanity-checks (before asserting parity) that the SNP actually landed in - dense/snp — i.e. ``find_ranges``' ``dense_snp_range`` is a non-empty window + dense/snp — i.e. ``_find_ranges``' ``dense_snp_range`` is a non-empty window for a region covering it — so this test genuinely exercises split_to_flat's snp-block path rather than silently falling back to the var_key channel. """ @@ -186,7 +186,7 @@ def test_readbound_variants_dense_snp_match_decode_oracle(svar2_store_dense_snp) # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a # region spanning it has a non-empty dense_snp window. - d = sv.find_ranges(contig, [0], [40], samples=None) + d = sv._find_ranges(contig, [0], [40], samples=None) dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py index cfef154f..8c028ca5 100644 --- a/tests/dataset/test_write_svar2.py +++ b/tests/dataset/test_write_svar2.py @@ -119,11 +119,13 @@ def test_write_svar2_emits_cache(svar2_store: Path, tmp_path: Path): assert region_starts.shape == (bed.height,) # ---- FIX 1: verify cache CONTENTS (not just shapes/keys) against a direct - # find_ranges call over the same regions. gvl sorts the written samples, so - # replay find_ranges with the sorted sample list to match slot ordering. + # _find_ranges call over the same regions. gvl sorts the written samples, so + # replay _find_ranges with the sorted sample list to match slot ordering. # This LOCKS the row-major (R, S, P) reshape and per-contig layout: a # scrambled / mis-transposed cache would fail loudly here. - sorted_samples = sorted(svar2.samples) # what gvl.write wrote (samples.sort()) + sorted_samples = sorted( + svar2.available_samples + ) # what gvl.write wrote (samples.sort()) S, P = len(sorted_samples), svar2.ploidy def mm(name: str) -> np.ndarray: @@ -142,7 +144,9 @@ def mm(name: str) -> np.ndarray: # sample_cols is written with np.save (has a .npy header): read with np.load. sample_cols = np.load(rd / "sample_cols.npy") - assert sample_cols.tolist() == [svar2.samples.index(s) for s in sorted_samples] + assert sample_cols.tolist() == [ + svar2.available_samples.index(s) for s in sorted_samples + ] contig_offset = 0 for (c,), df in bed.partition_by( @@ -150,7 +154,7 @@ def mm(name: str) -> np.ndarray: ).items(): rc = df.height lo, hi = contig_offset, contig_offset + rc - d = svar2.find_ranges( + d = svar2._find_ranges( c, df["chromStart"].to_numpy(), df["chromEnd"].to_numpy(), @@ -160,7 +164,7 @@ def mm(name: str) -> np.ndarray: np.testing.assert_array_equal( region_starts_full[lo:hi], np.asarray(d["region_starts"], np.int64) ) - # vk ranges: reshape (rc, S, P, 2) -> (rc*S*P, 2) must equal find_ranges' + # vk ranges: reshape (rc, S, P, 2) -> (rc*S*P, 2) must equal _find_ranges' # row-major (R*S*P, 2). This pins the reshape done in _write_from_svar2. np.testing.assert_array_equal( vk_snp[lo:hi].reshape(rc * S * P, 2), diff --git a/tests/unit/dataset/test_dataset_utils.py b/tests/unit/dataset/test_dataset_utils.py index 42afc805..3fcee65f 100644 --- a/tests/unit/dataset/test_dataset_utils.py +++ b/tests/unit/dataset/test_dataset_utils.py @@ -3,7 +3,7 @@ from __future__ import annotations import numpy as np -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from genvarloader._dataset._utils import ( bed_to_regions, oidx_to_raveled_idx, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b0bfd560..2a60e465 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,6 +1,6 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from genvarloader._dataset._utils import bed_to_regions from genvarloader._utils import normalize_contig_name from pytest_cases import parametrize_with_cases From 689336601c08dc46b563f7bf1572b62b16af3f35 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 12 Jul 2026 23:05:51 -0700 Subject: [PATCH 073/108] docs(svar2): design for routing INFO/FORMAT fields to gvl variants Route arbitrary scalar-numeric SVAR2 INFO/FORMAT fields through gvl's read-bound decode kernel into RaggedVariants and variant-windows, using genoray's exported vk_src provenance + FieldView APIs (no genoray change). Co-Authored-By: Claude Opus 4.8 (1M context) --- ...-svar2-info-format-field-routing-design.md | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md diff --git a/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md b/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md new file mode 100644 index 00000000..c570d28b --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md @@ -0,0 +1,210 @@ +# SVAR2 INFO/FORMAT field routing → `RaggedVariants` / `FlatVariants` + +**Date:** 2026-07-12 +**Worktree/branch:** `svar2-m6b-kernel` (draft PR #266) +**Status:** design approved; ready for implementation plan + +## Goal + +Route arbitrary scalar-numeric INFO/FORMAT fields stored in an SVAR2 (`.svar`) +store all the way through gvl's read-bound SVAR2 path into the `variants` +(`RaggedVariants`) and variant-windows (`_FlatVariantWindows`, the "flat" +variant output) outputs. Today gvl's SVAR2 path surfaces only `alt`/`start`/ +`ilen`; the field values genoray now persists (`SparseVar2.from_vcf(info_fields=, +format_fields=)` / `from_pgen`) are dropped. + +**Scope (approved):** both INFO and FORMAT fields; both `RaggedVariants` and +variant-windows outputs. Scalar-numeric only (`Integer`/`Float`, `Flag` for +INFO; `Number` `1` or `A`) — exactly what genoray's SVAR2 store can hold. + +## Background — where the seam is + +gvl's SVAR2 path does **not** call genoray's Python `SparseVar2.decode()`. It +uses gvl's own read-bound Rust kernel `decode_variants_from_svar2_readbound` +(`src/ffi/mod.rs:1330`), which links `genoray_core` as a crate, calls +`genoray_core::query::gather_haps_readbound(reader, &HapRanges) -> +BatchResultSplit`, and runs its own `var_key ⋈ dense_snp ⋈ dense_indel` +position-merge (`src/svar2/mod.rs:decode_variants_from_split`) over plain +`KeyRef { position, key }`. To attach a field value to a decoded variant we need +that variant's **source index** in the store, which plain `KeyRef` does not +carry. + +genoray (current main `acc59cb`) already exports everything needed to recover it +— **no genoray-side code change is required**: + +- `gather_haps_readbound_src` — same as `gather_haps_readbound` but populates + `BatchResultSplit.vk_src` (packed var_key provenance). The plain variant + leaves `vk_src` empty (it does not pay for provenance). +- `pack_vk_src` / `unpack_vk_src` / `VK_SRC_INDEL_BIT` — `vk_src[i]` → + `(is_indel, call_idx)`, where `call_idx` is the absolute call index into + `var_key/{snp,indel}`. +- `dense_abs_row(on_disk_range, out_range, i)` — recovers a dense variant's + absolute source row from the `HapRanges` on-disk dense range + the + `BatchResultSplit` output dense range. (No `dense_src` array in the split + path; it's pure arithmetic.) +- `FieldView` (+ `FieldValue`) with `value_at(i)` / `format_at(dense_row, + orig_sample)` / `bytes_at(i)`; `crate::layout::{ContigPaths, FieldSub}`, + `crate::field::StorageDtype`. `FieldSub::all()` order is `[VkSnp, VkIndel, + DenseSnp, DenseIndel]`. + +Reference implementation to mirror: genoray's own `gather_batch_fields` +(`src/py_query_decode.rs:156`), which does this gather for the whole-cohort +batch path. **The one difference in the read-bound path:** the split has +`n_samples == 1`, so a dense FORMAT lookup must stride by the real cohort sample +`HapRanges::orig_samples[q]`, i.e. `format_at(dense_row, orig_samples[q])` — +*not* the split's sample slot (always 0). + +## Prerequisite — genoray dependency (build a local wheel) + +The field-read API is **unreleased**: it is 87 commits past the `3.0.0` tag and +in no tag. The existing `genoray-3.0.0` wheel lacks it. Plan (approved): + +1. **Drop gvl's genoray version pin.** `pyproject.toml:14` `"genoray>=3,<4"` → + `"genoray"`. (Re-pin to a real release later, when genoray ships svar-2.) +2. **Build a genoray manylinux wheel from current main** (`acc59cb`, has the + field-read API): + ``` + cd /carter/users/dlaub/projects/genoray + pixi run --manifest-path ci/wheel/pixi.toml build # -> wheelhouse/*.whl + pixi run --manifest-path ci/wheel/pixi.toml repair # auditwheel -> dist/*.whl + ``` + The wheel reports version `2.15.0` (genoray's `pyproject` version); that's + fine once gvl's pin is dropped. Mind the NFS build gotchas + (`CARGO_TARGET_DIR=/tmp/...`). +3. **Point gvl's pixi pin at the new wheel** (`pixi.toml:110`), then + `pixi install` / re-solve. This also fixes the currently-unsolvable env + (`pyproject` `>=3,<4` vs the pinned `2.15.0` wheel). + +The Rust side already links the live genoray repo via `Cargo.toml` +`genoray_core = { path = ".../genoray", ... }`, so it sees the exports at build +time regardless of the Python wheel. + +## Rust design (`genvarloader` crate) + +Files: `src/ffi/mod.rs`, `src/svar2/mod.rs`, `src/svar2/store.rs`, `src/lib.rs` +(registration unchanged unless signature changes require it). + +1. **`decode_variants_from_svar2_readbound` (`src/ffi/mod.rs:1330`)** gains a + `fields: Vec<(String /*category*/, String /*name*/, String /*dtype*/)>` + param (empty ⇒ current behavior, zero overhead). Returns the existing 5-tuple + plus: + - `field_bufs: Vec>>` — one flat little-endian byte buffer + per requested field, length `n_var * itemsize`, in the same variant order + as `pos` (so it shares `var_off`). + - `field_itemsizes: Vec` — parallel; Python asserts + `itemsize == dtype.itemsize` (mirrors genoray `_svar2_decode.py:70`). +2. When `fields` is non-empty, call **`gather_haps_readbound_src`** instead of + `gather_haps_readbound`, and keep the `HapRanges` alive to feed + `dense_abs_row`. +3. Open, per field, the four `FieldView`s (`FieldSub::all()`) from the store's + `ContigReader` paths. `Svar2Store`/`ContigReader` gain the accessor(s) needed + to reach `ContigPaths` + the store's cohort `n_samples` and each field's + `StorageDtype` (resolved from the store `meta.json` manifest — genoray's + `StorageDtype::from_meta_str` is public). If the store lacks a requested + field, error clearly in Python before calling the kernel (see Python §1). +4. **Thread provenance through the merge** in `decode_variants_from_split` + (`src/svar2/mod.rs`). For each emitted variant, in lockstep with `pos`: + - var_key entry `i` → `unpack_vk_src(split.vk_src[i]) = (is_indel, call_idx)`; + `is_dense=false`, `idx=call_idx`. + - dense entry `i` in the snp/indel window for query `q` → + `idx = dense_abs_row(hapranges.dense_{snp,indel}_range[q], + split.dense_{snp,indel}_range[q], i)`; `is_dense=true`, `is_indel` by + channel. + Then for each field pick the sub-view by `(is_dense, is_indel)` and append: + - INFO → `view.bytes_at(idx)` + - FORMAT → var_key: `view.bytes_at(idx)`; dense: + `view.bytes_at(dense_row * cohort_n_samples + orig_samples[q])` + (== `format_at(dense_row, orig_samples[q])` as bytes). + Copy bytes verbatim (no dtype dispatch in Rust; Python `.view()`s). Missing + values already carry genoray's stored default/sentinel — passed through. + +**Provenance-vs-decode ordering invariant (must-verify):** the merge already +emits variants in a defined tie order (`var_key < dense-snp < dense-indel` on +equal positions). The field append must use the provenance of *the same* entry +chosen at each merge step — i.e. provenance is carried on the merge cursor, not +recomputed. This is the single highest-risk part; the plan must add an identity +test that a field carrying "the variant's own source row index" decodes back to +a strictly-consistent mapping. + +## Python design (`Svar2Haps`, `python/genvarloader/_dataset/_svar2_haps.py`) + +1. **Field discovery** (`from_path`): after `sv = SparseVar2(str(svar2_path))`, + read the store manifest via `sv.available_fields` (genoray 3.x). Set + `self.available_var_fields = ["alt","ilen","start"] + list(store_field_keys)` + (replaces the hard-coded `:174`). Keep a `dict[key -> (category, name, + np.dtype)]` on the instance for the kernel call and dtype `.view()`. Thread + the user's `var_fields` into `from_path` and onto the instance (today it + defaults to `["alt","ilen","start"]` and is never set from the Dataset for + svar2 — wire it like the SVAR1 path). +2. **Request set:** `requested = [f for f in self.var_fields if f not in + {"alt","start","ref","ilen","dosage"}]`. Validate each is in + `available_var_fields`; raise a clear error otherwise. Map to `(category, + name, dtype)` triples for the kernel. +3. **`_reconstruct_variants`:** pass the triples to the kernel; receive + `field_bufs`/`itemsizes`. Each field is per-variant (one element per + variant), so it is **parallel to `pos`** and flows through the exact existing + machinery with `field_g = field_c[src]`: + - single-contig fast path: `Ragged.from_offsets(buf.view(dtype), shape, + var_off_g)`; + - multi-contig: index by the same `src` from `_ragged_arange_src`; + - `unphased_union`: the `var_off[::P]` fold needs **no special handling** + (field data stays whole, offsets reindex). + Add each as `RaggedVariants(**{key: field_ragged})`, sharing `alt`'s offsets. +4. **`_reconstruct_variant_windows`:** same kernel call; add each field to the + `_FlatVariantWindows.fields` dict as a `_Flat.from_offsets(buf.view(dtype), + shape, row_off)`, exactly like `start`/`ilen` (respecting the single- vs + multi-contig branches and the `include_*` gating by `var_fields`). +5. **`DummyVariant` / empty-group fill:** variant-windows fills empty groups via + `dummy_variant`. Extend the dummy to carry a per-field fill (genoray's stored + default/sentinel, or NaN/sentinel) so `fill_empty_groups` has a value for + each field. (`DummyVariant` already has an `info: dict` slot.) + +## Data-flow / alignment invariants + +- Field buffers are 1-level ragged, per-variant, sharing the variant-axis + offsets (`var_off`) with `pos`/`ilen`. Every reorder/fold applied to `pos` + applies identically to fields. +- Dtypes are preserved end-to-end (stored → raw bytes → `.view(dtype)`); no + widening. Ints keep genoray's lossless auto-narrowed width; floats stay + `f16`/`f32`; `Flag`→`bool`. +- Missing entries carry genoray's `default`/sentinel verbatim — gvl does not + reinterpret them. + +## Field key naming & dtypes + +Use the keys genoray's manifest exposes (`sv.available_fields` keys): bare field +name when unambiguous, else category-prefixed (genoray already disambiguates +INFO vs FORMAT collisions). These become the `RaggedVariants(**fields)` kwargs +and `_FlatVariantWindows.fields` keys. Never collide with builtin keys +(`alt`/`start`/`ref`/`ilen`/`dosage`) — guard in discovery. + +## Testing + +Round-trip oracle test mirroring genoray's `test_svar2_fields_read.py`: + +1. Build a small VCF with ≥2 INFO fields (e.g. an `Integer` and a `Float=AF`) + and ≥1 FORMAT field (e.g. `DP` Integer). Convert with + `SparseVar2.from_vcf(info_fields=..., format_fields=...)`. +2. `gvl.write` a dataset over it; read `variants` and variant-windows with + `var_fields` including the field names. +3. Assert field **values** (not just shape/dtype) against the VCF ground truth, + across: + - var_key-routed vs dense-routed variants (both present in the fixture); + - multi-contig (exercise the `_ragged_arange_src` reorder); + - `unphased_union=True`; + - VCF-missing entries (assert the configured default / sentinel). +4. Rust unit test on `decode_variants_from_split` asserting provenance identity + (the ordering invariant above). + +## Docs / skill updates + +- Update `skills/genvarloader/SKILL.md` if the public `var_fields` surface for + svar2 datasets changes (it will: svar2 `variants` now honors arbitrary store + fields). +- Add a `CHANGELOG.md` (Unreleased) entry. + +## Out of scope (unchanged guards) + +`min_af`/`max_af`, `filter=="exonic"`, spliced output, annotated haps, in-kernel +RC — all still `NotImplementedError` for svar2. Non-scalar / `Number`-other +fields are rejected at the genoray write boundary, so they never reach here. From 1e9d9265f767e82a14ec32b9d3359042aa3f3780 Mon Sep 17 00:00:00 2001 From: d-laub Date: Sun, 12 Jul 2026 23:20:08 -0700 Subject: [PATCH 074/108] docs(svar2): implementation plan for INFO/FORMAT field routing Co-Authored-By: Claude Opus 4.8 (1M context) --- ...6-07-12-svar2-info-format-field-routing.md | 1000 +++++++++++++++++ 1 file changed, 1000 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md diff --git a/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md b/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md new file mode 100644 index 00000000..f32a7c1b --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md @@ -0,0 +1,1000 @@ +# SVAR2 INFO/FORMAT Field Routing — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Route arbitrary scalar-numeric SVAR2 INFO/FORMAT fields through gvl's read-bound decode kernel into `RaggedVariants` and variant-windows (`_FlatVariantWindows`) outputs. + +**Architecture:** The seam is Rust-side: gvl's `decode_variants_from_svar2_readbound` kernel gains provenance tracking (via genoray's `gather_haps_readbound_src` + `unpack_vk_src`/`dense_abs_row`) and gathers per-variant field bytes via genoray's exported `FieldView`. Field byte-buffers ride the existing `pos`-parallel offset machinery, so multi-contig reorder and `unphased_union` need no special handling. Python discovers the store's field manifest and wraps the returned buffers into the output types. + +**Tech Stack:** Rust (PyO3, `genoray_core` crate path-dep, `ndarray`/`numpy`), Python (numpy, `seqpro.rag.Ragged`), pixi, genoray SVAR2 field-read API (unreleased, from genoray main). + +## Global Constraints + +- **genoray dependency:** build a wheel from genoray **main** (`/carter/users/dlaub/projects/genoray`, HEAD ~`acc59cb`, which has the field-read API — it is 87 commits past the `3.0.0` tag and UNRELEASED). Rust links it via the existing `Cargo.toml` path-dep. Do **not** modify genoray. Do **not** hand-bump genoray's version. +- **Drop gvl's genoray version pin** (`pyproject.toml` `"genoray>=3,<4"` → `"genoray"`); re-pin at genoray release. The wheel will report version `2.15.0` — acceptable once the pin is dropped. +- **Field types supported:** scalar-numeric only — INFO/FORMAT `Type=Integer/Float` (+ `Flag` for INFO), `Number` `1` or `A`. Non-scalar fields are rejected at genoray's write boundary and never reach gvl. +- **Dtype fidelity:** field values are passed as raw little-endian bytes + an itemsize from Rust; Python `.view(dtype)`s them. No widening/conversion. Missing entries carry genoray's stored `default`/sentinel verbatim. +- **Zero-overhead no-field path:** when no fields are requested, the kernel must behave byte-identically to today (call `gather_haps_readbound`, no provenance, no `FieldView`). +- **NFS build gotchas (from genoray dev notes):** export `CARGO_TARGET_DIR=/tmp/gvl-target` before `cargo`/`maturin` on this NFS checkout to avoid linker bus errors; genoray's own Rust tests need `--no-default-features`. +- **Commit convention:** Conventional Commits (`feat:`/`fix:`/`refactor:`/`test:`/`docs:`/`chore:`). All commits to branch `svar2-m6b-kernel`. Never touch `main`. +- **Pre-commit hook caveat:** the `pyrefly-check` pre-commit hook fails to solve the env until Phase 0 Task 2 completes. Until then, commit with `git commit --no-verify`. **After** Task 2, drop `--no-verify` and let hooks run. + +## File Structure + +**Rust (crate root `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`):** +- `src/ffi/mod.rs` — FFI entry `decode_variants_from_svar2_readbound` (`:1330`); gains a `fields` param + field-buffer outputs. Also the tuple→Range migration at all `HapRanges::new` call sites (`:963,1116,1227,1372`). +- `src/svar2/mod.rs` — `decode_variants_from_split` (`:~282`) gains optional provenance capture + field gather; `split_to_flat` (`:~158`) tuple→Range fix; new `FieldGather` struct; tests. +- `src/svar2/store.rs` — `Svar2Store` retains `store_path` + accessor (needed to rebuild `ContigPaths` for `FieldView`). +- `Cargo.toml` — no code change (already path-deps genoray main); may need `use` of new genoray symbols in the `.rs` files. + +**Python:** +- `python/genvarloader/_dataset/_svar2_haps.py` — field discovery in `from_path`/`__post_init__`; field pass-through + wrapping in `_reconstruct_variants` and `_reconstruct_variant_windows`. +- `python/genvarloader/_dataset/_impl.py` — `var_fields` request block (`:338-369`): add an early SVAR2 branch that skips SVAR1 lazy-loading. + +**Config / docs:** +- `pyproject.toml` (`:14`), `pixi.toml` (`:110`) — dependency. +- `tests/dataset/test_svar2_fields_read.py` (new) — integration oracle test. +- `skills/genvarloader/SKILL.md`, `CHANGELOG.md` — docs. + +--- + +# Phase 0 — Baseline: migrate gvl to genoray main (green build, no feature yet) + +> gvl's Rust was written against genoray 2.15.0, where `BatchResultSplit`/`HapRanges` range fields were `(usize,usize)` tuples. genoray's `505a37f` refactor changed them to `Range` (present since the `3.0.0` tag). The "adopt genoray 3.0.0 API" commit touched **only Python** (0 `.rs` files), so the Rust does not compile against current genoray. Phase 0 makes it compile and pass the existing test suite against genoray main, establishing the baseline the field feature builds on. + +### Task 0.1: Build a genoray wheel from main + +**Files:** +- None in gvl (produces a wheel on disk). + +- [ ] **Step 1: Build + repair the wheel** + +```bash +cd /carter/users/dlaub/projects/genoray +export CARGO_TARGET_DIR=/tmp/genoray-wheel-target +pixi run --manifest-path ci/wheel/pixi.toml build # -> wheelhouse/*.whl (abi3 cp310) +pixi run --manifest-path ci/wheel/pixi.toml repair # auditwheel -> dist/*.whl (manylinux) +ls -la dist/*.whl +``` + +Expected: a `genoray-2.15.0-cp310-abi3-manylinux_*_x86_64.whl` in `dist/`. + +- [ ] **Step 2: Sanity-check the wheel has the field-read API** + +```bash +cd /carter/users/dlaub/projects/genoray +python -c "import zipfile,glob; z=zipfile.ZipFile(glob.glob('dist/genoray-*-abi3-manylinux*.whl')[0]); src=z.read('genoray/_svar2.py').decode(); print('available_fields' in src, 'from_pgen' in src)" +``` + +Expected: `True True`. + +- [ ] **Step 3: Commit** — nothing to commit (artifact only). Record the wheel path for Task 0.2. + +### Task 0.2: Point gvl at the wheel + drop the version pin + +**Files:** +- Modify: `pyproject.toml:14` +- Modify: `pixi.toml:110` + +**Interfaces:** +- Produces: a solvable pixi env with genoray main (field-read API) importable. + +- [ ] **Step 1: Drop the version constraint** + +In `pyproject.toml`, change the dependency line: + +```toml + "genoray", +``` + +(from `"genoray>=3,<4",`). + +- [ ] **Step 2: Point the pixi pin at the new wheel** + +In `pixi.toml`, replace the `genoray = { path = ... }` line (`:110`) with the manylinux wheel built in Task 0.1 (use the actual filename from `dist/`): + +```toml +genoray = { path = "/carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl" } +``` + +- [ ] **Step 3: Re-solve + verify import** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +pixi install +pixi run python -c "from genoray import SparseVar2; print(hasattr(SparseVar2,'available_fields'), hasattr(SparseVar2,'with_fields'))" +``` + +Expected: solve succeeds; prints `True True`. + +- [ ] **Step 4: Commit** + +```bash +git add pyproject.toml pixi.toml pixi.lock +git commit --no-verify -m "chore(svar2): pin genoray main wheel, drop version constraint for dev" +``` + +### Task 0.3: Migrate gvl Rust range types tuple→Range + +**Files:** +- Modify: `src/ffi/mod.rs` (the `to_pairs` closure + all `HapRanges::new` sites `:963,1116,1227,1372`; the `decode_variants_from_svar2_readbound` closure `:~1356-1370`) +- Modify: `src/svar2/mod.rs` (`split_to_flat` `:~158`, `decode_variants_from_split` `:~300`, and the `#[cfg(test)]` `BatchResultSplit` literals) + +**Interfaces:** +- Consumes: `genoray_core::query::HapRanges::new(&[u32], &[usize], &[Range], &[Range], &[Range], &[Range], usize)`; `BatchResultSplit { dense_snp_range: Vec>, dense_indel_range: Vec>, vk_src: Vec, dense_src: Vec<(bool,u32)>, .. }` (derives `Default`). +- Produces: a gvl crate that compiles against genoray main; behavior byte-identical (no feature). + +- [ ] **Step 1: Convert the FFI range marshaling to `Range`** + +In `src/ffi/mod.rs`, replace the `to_pairs` closure (in `decode_variants_from_svar2_readbound`, ~`:1356`) and the analogous marshaling in the reconstruct/diffs/tracks FFI fns with a `to_ranges` helper. Add near the top of the module: + +```rust +use std::ops::Range; + +fn arr2_to_ranges(a: numpy::ndarray::ArrayView2) -> Vec> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize)..(r[1] as usize)) + .collect() +} +``` + +Then at each site that built `Vec<(usize,usize)>` for a `HapRanges::new` arg (search `to_pairs` and inline `.map(|r| (r[0] as usize, r[1] as usize))`), replace with `arr2_to_ranges(.as_array())`. The `HapRanges::new(...)` calls themselves are unchanged (they now receive `&[Range]`). + +- [ ] **Step 2: Convert `BatchResultSplit` range consumption in `src/svar2/mod.rs`** + +In `split_to_flat` and `decode_variants_from_split`, every `let (ss, se) = br.dense_snp_range[q];` / `let (is_, ie) = br.dense_indel_range[q];` becomes a `Range` read. Use `.clone()`-free field access: + +```rust +let Range { start: ss, end: se } = br.dense_snp_range[q]; +let Range { start: is_, end: ie } = br.dense_indel_range[q]; +``` + +(add `use std::ops::Range;` to `src/svar2/mod.rs`). All downstream uses of `ss/se/is_/ie` are unchanged (they're `usize`). + +- [ ] **Step 3: Fix the `#[cfg(test)]` `BatchResultSplit` literals** + +In `src/svar2/mod.rs` tests, every `dense_snp_range: vec![(0, 1)]` etc. becomes `dense_snp_range: vec![0..1]`, `vec![(0, 2), (2, 4)]` becomes `vec![0..2, 2..4]`, etc. `BatchResultSplit` derives `Default`, so add `..Default::default()` at the end of each struct literal to cover the new `vk_src`/`dense_src` fields: + +```rust +let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 2, + vk: vec![/* ... */], + vk_off: vec![/* ... */], + dense_snp: vec![/* ... */], + dense_snp_range: vec![0..1], + // ... other existing fields ... + ..Default::default() +}; +``` + +- [ ] **Step 4: Compile to green** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -30 +``` + +Expected: `Finished`. If additional tuple→Range or new-field errors surface (compiler-driven; the refactor is mechanical), fix each at the reported site exactly as in Steps 1–3, then re-run until green. + +- [ ] **Step 5: Run the Rust unit tests** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features 2>&1 | tail -20 +``` + +Expected: all existing svar2 Rust tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/ffi/mod.rs src/svar2/mod.rs +git commit --no-verify -m "refactor(svar2): migrate gvl read-bound Rust to genoray Range API" +``` + +### Task 0.4: Build the extension + green existing test suite (baseline gate) + +**Files:** +- None (build + test). + +- [ ] **Step 1: Build the editable extension** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -15 +``` + +Expected: `Built ... Installed genvarloader`. + +- [ ] **Step 2: Run the existing SVAR2 test suite** + +```bash +pixi run pytest tests/dataset/test_svar2_readbound_variants.py tests/dataset/test_svar2_readbound_haps.py -x -q 2>&1 | tail -25 +``` + +Expected: all PASS (baseline against genoray main; no feature yet). This gate proves the migration is correct before adding fields. + +- [ ] **Step 3: Confirm the pre-commit hook now solves** + +```bash +git commit --allow-empty -m "chore: verify hooks solve" && git reset --soft HEAD~1 +``` + +Expected: no pyrefly solve error. From here on, commit **without** `--no-verify`. + +--- + +# Phase 1 — Rust: provenance + field gather + +### Task 1.1: `Svar2Store` retains `store_path` + +**Files:** +- Modify: `src/svar2/store.rs` + +**Interfaces:** +- Produces: `Svar2Store::store_path(&self) -> &str` — the store base dir, for rebuilding `ContigPaths`. + +- [ ] **Step 1: Add the field + accessor** + +In `src/svar2/store.rs`, add `store_path: String` to the struct, set it in `new`, and expose it: + +```rust +#[pyclass] +pub struct Svar2Store { + readers: HashMap, + store_path: String, +} + +impl Svar2Store { + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } + pub fn store_path(&self) -> &str { + &self.store_path + } +} +``` + +In `new`, capture `store_path` before the loop consumes `contigs` (note: `store_path: &str` param): + +```rust + let store_path = store_path.to_string(); + // ... existing loop populating `readers` ... + Ok(Self { readers, store_path }) +``` + +- [ ] **Step 2: Compile** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -10 +``` + +Expected: `Finished`. + +- [ ] **Step 3: Commit** + +```bash +git add src/svar2/store.rs +git commit -m "feat(svar2): Svar2Store retains store_path for field-sidecar paths" +``` + +### Task 1.2: `FieldGather` + provenance capture in `decode_variants_from_split` + +**Files:** +- Modify: `src/svar2/mod.rs` +- Test: `src/svar2/mod.rs` (`#[cfg(test)]`) + +**Interfaces:** +- Consumes: `genoray_core::query::{unpack_vk_src, dense_abs_row, FieldView}`; `BatchResultSplit.vk_src` (populated only by `gather_haps_readbound_src`). +- Produces: + ```rust + pub struct FieldGather<'a> { + pub views: [genoray_core::query::FieldView; 4], // FieldSub order: VkSnp,VkIndel,DenseSnp,DenseIndel + pub is_format: bool, + pub width: usize, // dtype.width_bytes() + pub cohort_n_samples: usize, + pub _marker: std::marker::PhantomData<&'a ()>, + } + pub fn decode_variants_from_split( + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], + fields: &[FieldGather<'_>], + on_disk_snp: &[std::ops::Range], // HapRanges dense_snp_range; empty if no fields + on_disk_indel: &[std::ops::Range], + orig_samples: &[usize], + ) -> (VariantsSoa, Vec>) // second: one byte-buffer per field, parallel to pos + ``` + When `fields.is_empty()`, the second return is `Vec::new()` and no provenance work runs (byte-identical to today). + +- [ ] **Step 1: Write the failing Rust test (provenance identity)** + +Add to `src/svar2/mod.rs` `#[cfg(test)] mod tests`. This test builds a split with one var_key entry and one dense-snp entry and an INFO field whose stored value at each source row equals that row index, then asserts the decoded field values match the merge order: + +```rust +#[test] +fn test_decode_fields_provenance_identity() { + use genoray_core::query::{pack_vk_src, KeyRef}; + // One query, ploidy 1. var_key call idx 5 at pos 10; dense-snp row (abs) 3 at pos 20. + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { position: 10, key: svar2_codec::encode_pure_del(-1) }], + vk_off: vec![0, 1], + vk_src: vec![pack_vk_src(false, 5)], // var_key/snp, call idx 5 + dense_snp: vec![KeyRef { position: 20, key: svar2_codec::encode_pure_del(-1) }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![], + dense_indel_range: vec![0..0], + dense_indel_present: vec![], + dense_indel_present_off: vec![0, 0], + }; + // INFO field: value_at(i) == i (an i32 store big enough for idx 5 and dense row 3). + // Build FieldView test doubles via a helper that fills values.bin so value_at(i)=i. + let fields = make_identity_i32_fields(); // see Step 3 helper + let (soa, bufs) = decode_variants_from_split( + &br, &[], &[0i64], &fields, &[0..1], &[0..0], &[0], + ); + assert_eq!(soa.pos, vec![10, 20]); // var_key before dense on distinct positions + // Field buffer holds i32 LE bytes: [5, 3] (vk call idx 5 first, dense abs row 3 second). + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(vals, vec![5, 3]); +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features test_decode_fields_provenance_identity 2>&1 | tail -15 +``` + +Expected: FAIL — `decode_variants_from_split` arity mismatch / `make_identity_i32_fields` undefined. + +- [ ] **Step 3: Implement `FieldGather`, the test helper, and the gather** + +Add the struct + `use` lines at the top of `src/svar2/mod.rs`: + +```rust +use std::ops::Range; +use genoray_core::query::{dense_abs_row, unpack_vk_src, FieldView}; + +pub struct FieldGather { + pub views: [FieldView; 4], // FieldSub::all() order: VkSnp, VkIndel, DenseSnp, DenseIndel + pub is_format: bool, + pub width: usize, + pub cohort_n_samples: usize, +} +``` + +Change `decode_variants_from_split`'s signature to the one in **Interfaces** above. Inside the merge loop, at the point where `(p, key)` is selected, ALSO record which channel + index it came from, then (only if `!fields.is_empty()`) resolve provenance and append field bytes. Concretely, replace the selection block: + +```rust + let (p, key, chan, cidx) = if has_vk && p_vk <= p_sn && p_vk <= p_in { + let e = &br.vk[i_vk]; + let out = (e.position, e.key, 0u8, i_vk); + i_vk += 1; + out + } else if has_sn && p_sn <= p_in { + let e = &br.dense_snp[i_sn]; + let out = (e.position, e.key, 1u8, i_sn); + i_sn += 1; + out + } else { + let e = &br.dense_indel[i_in]; + let out = (e.position, e.key, 2u8, i_in); + i_in += 1; + out + }; + let (il, alt) = decode_alt(key, lut_bytes, lut_off); + pos.push(p as i32); + ilen.push(il as i32); + alt_bytes.extend_from_slice(&alt); + str_off.push(alt_bytes.len() as i64); + + if !fields.is_empty() { + // Resolve (is_dense, is_indel, elem_row) for this emitted variant. + let (is_dense, is_indel, row) = match chan { + 0 => { + let (is_indel, call_idx) = unpack_vk_src(br.vk_src[cidx]); + (false, is_indel, call_idx) + } + 1 => (true, false, dense_abs_row(&on_disk_snp[q], &(ss..se), cidx)), + _ => (true, true, dense_abs_row(&on_disk_indel[q], &(is_..ie), cidx)), + }; + let sub_ix = match (is_dense, is_indel) { + (false, false) => 0, + (false, true) => 1, + (true, false) => 2, + (true, true) => 3, + }; + for (fi, f) in fields.iter().enumerate() { + let view = &f.views[sub_ix]; + let elem = if is_dense && f.is_format { + row * f.cohort_n_samples + orig_samples[q] + } else { + row + }; + field_bufs[fi].extend_from_slice(view.bytes_at(elem)); + } + } +``` + +Note: the var_key `sub_ix` uses `is_indel` from `unpack_vk_src` (a var_key entry may be snp OR indel), matching `FieldSub` order. Declare `field_bufs` before the loop: + +```rust + let mut field_bufs: Vec> = (0..fields.len()).map(|_| Vec::new()).collect(); +``` + +and return `(VariantsSoa { .. }, field_bufs)`. + +`ss/se/is_/ie` are already in scope per query (from Task 0.3 Step 2). Update the three existing callers of `decode_variants_from_split` in `src/ffi/mod.rs` (variants, and the two reconstruct/windows paths that call it — grep `decode_variants_from_split`) to pass `&[]`, `&[]`, `&[]`, `&[]` for the no-field case for now (Task 1.3 wires the real args for the variants FFI). + +Add the test helper `make_identity_i32_fields()` in `#[cfg(test)]`. Because `FieldView::open` reads real sidecar files, the helper writes a tiny temp store dir with `values.bin` files whose i32 elements equal their index (0,1,2,...) for each of the 4 subs, then opens 4 `FieldView`s via `genoray_core::query::FieldView::open(&ContigPaths::new(tmp, "chr1"), "info", "X", sub, StorageDtype::from_meta_str("i32"), 1)` for `sub in FieldSub::all()`. (Use `tempfile::tempdir()`; add `tempfile` as a dev-dependency in `Cargo.toml` if not present.) + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features test_decode_fields_provenance_identity 2>&1 | tail -15 +``` + +Expected: PASS. Also re-run the full Rust suite to confirm the no-field callers still pass: + +```bash +pixi run cargo test --no-default-features 2>&1 | tail -15 +``` + +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/svar2/mod.rs Cargo.toml +git commit -m "feat(svar2): field-byte gather with vk_src/dense provenance in split decode" +``` + +### Task 1.3: FFI — `decode_variants_from_svar2_readbound` gains fields + +**Files:** +- Modify: `src/ffi/mod.rs` (`:1330`) + +**Interfaces:** +- Consumes: `genoray_core::query::{gather_haps_readbound_src, FieldView}`, `genoray_core::layout::{ContigPaths, FieldSub}`, `genoray_core::field::StorageDtype`, `Svar2Store::store_path`, `ContigReader::n_samples`. +- Produces: Python signature + ``` + decode_variants_from_svar2_readbound( + store, contig, region_starts, orig_samples, + vk_snp_range, vk_indel_range, dense_snp_range, dense_indel_range, + ploidy, + fields: list[tuple[str, str, str]], # (category, name, dtype_str); may be [] + ) -> (pos, ilen, alt_bytes, str_off, var_off, + field_bufs: list[np.ndarray[u8]], field_itemsizes: list[int]) + ``` + +- [ ] **Step 1: Add the `fields` param and open `FieldView`s** + +Add `fields: Vec<(String, String, String)>` as the last param. After resolving `reader`, and only when `!fields.is_empty()`, build a `Vec`: + +```rust + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + + let n_samples = reader.n_samples(); + let paths = ContigPaths::new(store.store_path(), contig); + let gathers: Vec = fields + .iter() + .map(|(cat, name, dtype_str)| { + let dtype = StorageDtype::from_meta_str(dtype_str) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + let width = dtype.width_bytes().ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("field {name}: unresolved dtype")) + })?; + let subs = FieldSub::all(); + let views = [ + FieldView::open(&paths, cat, name, subs[0], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[1], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[2], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[3], dtype, n_samples), + ]; + let mut opened = Vec::with_capacity(4); + for v in views { + opened.push(v.map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?); + } + Ok(crate::svar2::FieldGather { + views: [opened.remove(0), opened.remove(0), opened.remove(0), opened.remove(0)], + is_format: cat == "format", + width, + cohort_n_samples: n_samples, + }) + }) + .collect::>>()?; +``` + +(Confirm `StorageDtype::from_meta_str` returns `Result`; if it returns `Option`, adapt the `.map_err`. `FieldSub::all()` returns `[FieldSub; 4]` in order VkSnp,VkIndel,DenseSnp,DenseIndel.) + +- [ ] **Step 2: Choose the gather + call the decode with field args** + +Replace the `py.detach` body so it uses `gather_haps_readbound_src` when fields are requested, and passes the on-disk dense ranges + `orig_samples`: + +```rust + let has_fields = !gathers.is_empty(); + let (soa, field_bufs) = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, &orig_samples_v, + &vk_snp_range_v, &vk_indel_range_v, + &dense_snp_range_v, &dense_indel_range_v, ploidy, + ); + let br = if has_fields { + genoray_core::query::gather_haps_readbound_src(reader, &rb) + } else { + genoray_core::query::gather_haps_readbound(reader, &rb) + }; + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + crate::svar2::decode_variants_from_split( + &br, &lut_bytes, &lut_off, + &gathers, &dense_snp_range_v, &dense_indel_range_v, &orig_samples_v, + ) + }); +``` + +(Requires `dense_snp_range_v`/`dense_indel_range_v`/`orig_samples_v` to be `move`d in; they already are `Vec`s from Task 0.3. `gathers` is `move`d in.) + +- [ ] **Step 3: Return the field buffers + itemsizes** + +Change the return type to append `Vec>>` + `Vec`, and build them: + +```rust + let field_out: Vec>> = + field_bufs.into_iter().map(|b| Array1::from_vec(b).into_pyarray(py)).collect(); + let itemsizes: Vec = gathers.iter().map(|g| g.width).collect(); + Ok(( + Array1::from_vec(soa.pos).into_pyarray(py), + Array1::from_vec(soa.ilen).into_pyarray(py), + Array1::from_vec(soa.alt_bytes).into_pyarray(py), + Array1::from_vec(soa.str_off).into_pyarray(py), + Array1::from_vec(soa.var_off).into_pyarray(py), + field_out, + itemsizes, + )) +``` + +Update the return type in the signature to `PyResult<( ..5 existing.., Vec>>, Vec )>`. **Move `let itemsizes` computation before the `py.detach` closure consumes `gathers`** (compute widths into a plain `Vec` first, then move `gathers` into the closure). + +- [ ] **Step 4: Compile** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -20 +``` + +Expected: `Finished`. Fix borrow/move errors by hoisting `itemsizes` before the closure (Step 3 note). + +- [ ] **Step 5: Rebuild the extension** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -8 +``` + +Expected: `Installed genvarloader`. + +- [ ] **Step 6: Commit** + +```bash +git add src/ffi/mod.rs +git commit -m "feat(svar2): decode_variants_from_svar2_readbound returns INFO/FORMAT field buffers" +``` + +--- + +# Phase 2 — Python wiring + +### Task 2.1: Field discovery in `Svar2Haps` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`__post_init__` `:169`, `from_path` `:178`, add dataclass fields) + +**Interfaces:** +- Consumes: `genoray.SparseVar2(path).available_fields -> dict[str, StoredField]` where `StoredField.{name, category('info'|'format'), dtype(np.dtype), key}`; `genoray._svar2_fields._META_DTYPE: dict[np.dtype, str]`. +- Produces: `Svar2Haps.available_var_fields` includes store field keys; `Svar2Haps._store_fields: dict[str, tuple[str,str,str,np.dtype]]` keyed by field key → `(category, name, dtype_str, np_dtype)`. + +- [ ] **Step 1: Add dataclass fields + populate in `from_path`** + +Add two `field()`-defaulted slots to `Svar2Haps` (they follow base defaults): + +```python + store_field_keys: list[str] = field(default_factory=list) + store_fields: dict[str, tuple[str, str, str, "np.dtype"]] = field(default_factory=dict) + """key -> (category, name, dtype_str, np_dtype) for store INFO/FORMAT fields.""" +``` + +In `from_path`, after `sv = SparseVar2(str(svar2_path))`: + +```python + from genoray._svar2_fields import _META_DTYPE + + store_field_keys = list(sv.available_fields.keys()) + store_fields = { + sf.key: (sf.category, sf.name, _META_DTYPE[sf.dtype], sf.dtype) + for sf in sv.available_fields.values() + } +``` + +and pass `store_field_keys=store_field_keys, store_fields=store_fields` into the `cls(...)` call. + +- [ ] **Step 2: Extend `available_var_fields` in `__post_init__`** + +Replace the hard-coded line (`:174`): + +```python + self.available_var_fields = ["alt", "ilen", "start"] + [ + k for k in self.store_field_keys + if k not in {"alt", "ilen", "start", "ref", "dosage"} + ] +``` + +- [ ] **Step 3: Rebuild + smoke-check discovery** + +Requires a fixture store with fields (created in Task 3.1). For now compile-check the import: + +```bash +pixi run python -c "import genvarloader._dataset._svar2_haps" +``` + +Expected: no error. + +- [ ] **Step 4: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): Svar2Haps discovers store INFO/FORMAT fields into available_var_fields" +``` + +### Task 2.2: `_impl.py` — SVAR2 branch skips SVAR1 lazy-loading + +**Files:** +- Modify: `python/genvarloader/_dataset/_impl.py` (`:338-369`) + +**Interfaces:** +- Consumes: `Svar2Haps` (import), `self.available_var_fields`. +- Produces: for `Svar2Haps`, `var_fields` is validated + set via `replace(...)` with no SVAR1 lazy-loading. + +- [ ] **Step 1: Add the early SVAR2 branch** + +At the top of the `if var_fields is not None:` block (`:338`), before the `custom_fmt`/`load_info` logic, insert: + +```python + if var_fields is not None: + missing = list(set(var_fields) - set(self.available_var_fields)) + if missing or not isinstance(self._seqs, Haps): + raise ValueError(f"Missing variant fields: {missing}") + from ._svar2_haps import Svar2Haps + + if isinstance(self._seqs, Svar2Haps): + # SVAR2 fields are read on demand by the decode kernel; no lazy + # INFO/dosage/custom-FORMAT loading (that is SVAR1-only). + haps = replace(to_evolve.get("_seqs", self._seqs), var_fields=var_fields) + to_evolve["_seqs"] = haps + else: + # ... existing SVAR1 lazy-loading block (custom_fmt, load_info, ...) ... + haps = to_evolve.get("_seqs", self._seqs) + # (unchanged existing lines through) + haps = replace(haps, var_fields=var_fields) + to_evolve["_seqs"] = haps +``` + +(Keep the existing SVAR1 body verbatim inside the `else`. The `missing` check stays shared at the top.) + +- [ ] **Step 2: Compile-check** + +```bash +pixi run python -c "import genvarloader._dataset._impl" +``` + +Expected: no error. + +- [ ] **Step 3: Commit** + +```bash +git add python/genvarloader/_dataset/_impl.py +git commit -m "feat(svar2): skip SVAR1 field lazy-loading for Svar2Haps var_fields" +``` + +### Task 2.3: Route fields into `RaggedVariants` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variants` `:558`) + +**Interfaces:** +- Consumes: the extended FFI (Task 1.3) returning `(pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_itemsizes)`. +- Produces: `RaggedVariants(**fields)` including each requested store field, sharing `alt`'s offsets. + +- [ ] **Step 1: Compute the requested field triples + call with fields** + +At the top of `_reconstruct_variants`, compute the requested extra fields (order stable): + +```python + builtin = {"alt", "start", "ref", "ilen", "dosage"} + req_keys = [f for f in self.var_fields if f not in builtin] + field_specs = [ + (self.store_fields[k][0], self.store_fields[k][1], self.store_fields[k][2]) + for k in req_keys + ] + field_dtypes = [self.store_fields[k][3] for k in req_keys] +``` + +Change the kernel call to pass `field_specs`, and unpack the two new returns per contig group: + +```python + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( + decode_variants_from_svar2_readbound( + self.store, self.ds_contigs[ci], + gi[0], gi[1], gi[2], gi[3], gi[4], gi[5], P, field_specs, + ) + ) +``` + +Accumulate per-field buffers per group into `cat_fields: list[list[np.ndarray]]` (one inner list per requested field), asserting `field_isizes[j] == field_dtypes[j].itemsize`. + +- [ ] **Step 2: Single-contig fast path — wrap fields** + +In the `len(cat_pos) == 1` branch, build a `fields` dict parallel to the existing `alt`/`start`/`ilen`, then splat into `RaggedVariants`: + +```python + extra = { + req_keys[j]: Ragged.from_offsets( + cat_fields[j][0].view(field_dtypes[j]), shape, var_off_g + ) + for j in range(len(req_keys)) + } + return RaggedVariants( + alt=Ragged.from_offsets(cat_alt[0].view("S1"), shape, var_off_g, str_offsets=str_off_g), + start=Ragged.from_offsets(cat_pos[0], shape, var_off_g), + ilen=Ragged.from_offsets(cat_ilen[0], shape, var_off_g), + **extra, + ) +``` + +- [ ] **Step 3: Multi-contig path — reorder fields by the same `src`** + +In the general path, after computing `src, var_off_g = _ragged_arange_src(grouped_var_off, perm)`, each field is per-variant so it reorders exactly like `pos`: + +```python + extra = {} + for j in range(len(req_keys)): + fc = np.concatenate([g[j] for g in per_group_fields]) if per_group_fields else np.zeros(0, np.uint8) + fc_typed = fc.view(field_dtypes[j]) + fg = fc_typed[:0].copy() if src.size == 0 else fc_typed[src] + extra[req_keys[j]] = Ragged.from_offsets(fg, shape, var_off_g) + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r, **extra) +``` + +(`per_group_fields[g][j]` is group `g`'s buffer for field `j`, already `.view(dtype)`-ed to length `n_var_group`. Reuse the same `src` computed for `pos_g`/`ilen_g`.) + +- [ ] **Step 4: Run the integration test (written in Task 3.1)** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -25 +``` + +Expected: `RaggedVariants` value assertions PASS (windows test may still fail until Task 2.4). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): route INFO/FORMAT fields into RaggedVariants" +``` + +### Task 2.4: Route fields into variant-windows (`_FlatVariantWindows`) + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variant_windows` `:653`) + +**Interfaces:** +- Consumes: the extended FFI; `_FlatVariantWindows.fields: dict[str, _Flat]`; `_Flat.from_offsets`. +- Produces: variant-windows output whose `.fields` includes each requested store field (like `start`/`ilen`). + +- [ ] **Step 1: Pass field_specs + collect per group** + +Mirror Task 2.3 Step 1 in `_reconstruct_variant_windows`: compute `req_keys`/`field_specs`/`field_dtypes` (identical code), pass `field_specs` to `decode_variants_from_svar2_readbound`, unpack `field_bufs, field_isizes`, and collect per-group typed buffers. + +- [ ] **Step 2: Add fields to the `fields` dict (single + multi-contig)** + +In the `len(cat_pos) == 1` branch, after `fields = {"start": ...}` (and optional `ilen`), add: + +```python + for j, k in enumerate(req_keys): + fields[k] = _Flat.from_offsets( + cat_fields[j][0].view(field_dtypes[j]), shape, row_off + ) +``` + +In the multi-contig `else`, after `pos_g`/`ilen_g`, reorder each field by the same `src` and add: + +```python + for j, k in enumerate(req_keys): + fc = np.concatenate([g[j] for g in per_group_fields]).view(field_dtypes[j]) + fg = fc[:0].copy() if src.size == 0 else fc[src] + fields[k] = _Flat.from_offsets(fg, shape, row_off_g) +``` + +- [ ] **Step 3: Extend `DummyVariant` fill for empty groups** + +`fill_empty_groups` needs a fill value per field. Set the dummy's per-field fill to the store default (or the dtype's sentinel: `NaN` for float, `iinfo.min` for int, `False` for bool) in `DummyVariant.info` keyed by field key when the dummy is constructed for svar2 (locate where `self.dummy_variant` is set for `Svar2Haps`; if none is set for windows, construct one carrying the field fills). Assert in the test that empty groups carry the fill. + +- [ ] **Step 4: Run the windows test** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -25 +``` + +Expected: all PASS (RaggedVariants + windows). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): route INFO/FORMAT fields into variant-windows output" +``` + +--- + +# Phase 3 — Integration test + docs + +### Task 3.1: Integration oracle test (write this BEFORE Task 2.3 — it is the Phase-2 gate) + +**Files:** +- Create: `tests/dataset/test_svar2_fields_read.py` + +**Interfaces:** +- Consumes: `genoray.SparseVar2.from_vcf(info_fields=, format_fields=)`, `gvl.write`, `gvl.Dataset(...).with_seqs("variants", var_fields=[...])`. + +- [ ] **Step 1: Build the fixture VCF + convert with fields** + +Write a helper that emits a small bgzipped VCF over TWO contigs with variants that route to BOTH var_key and dense channels (a common variant → dense, a rare per-sample variant → var_key), carrying INFO `AF` (Float), INFO `NS` (Integer), FORMAT `DP` (Integer). Convert: + +```python +from genoray import SparseVar2 +from genoray._svar2_fields import InfoField, FormatField + +SparseVar2.from_vcf( + out=store_dir, source=vcf_gz, reference=fasta, + info_fields=[InfoField("AF"), InfoField("NS")], + format_fields=[FormatField("DP")], +) +``` + +- [ ] **Step 2: Write the ground-truth oracle** + +Parse the VCF with `cyvcf2` into a dict `{(contig, pos): {"AF": ..., "NS": ..., "DP": {sample: ...}}}`. This is the expected-value source. + +- [ ] **Step 3: Write the failing test (RaggedVariants values)** + +```python +import numpy as np +import pytest + +@pytest.mark.parametrize("union", [False, True]) +def test_svar2_ragged_variants_fields(tmp_path, union): + ds_path = _write_dataset(tmp_path) # gvl.write over the svar2 source + import genvarloader as gvl + ds = gvl.Dataset.open(ds_path, reference=FASTA).with_seqs( + "variants", var_fields=["alt", "start", "ilen", "AF", "NS", "DP"], + ) + if union: + ds = ds.with_settings(unphased_union=True) # use the real gvl API name + rv = ds[0] # RaggedVariants + # For each (batch,ploid) group, walk variants and compare rv.AF / rv.NS / rv.DP + # to the oracle by (contig, start). Assert exact dtype + value, incl. NaN for + # VCF-missing (use np.testing.assert_array_equal with equal_nan for floats). + _assert_fields_match(rv, oracle) # helper asserts every decoded field value +``` + +Add a second test hitting a multi-contig batch (indices spanning both contigs) and a third asserting the FORMAT `DP` value equals the oracle's per-sample value for the queried sample. + +- [ ] **Step 4: Run to verify it fails (before Task 2.3)** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -20 +``` + +Expected: FAIL — `AF` not in `available_var_fields` / not present on `rv`. (This test is the gate for Tasks 2.3/2.4; implement those to make it pass.) + +- [ ] **Step 5: Add the variant-windows value test** + +Add `test_svar2_variant_windows_fields` that requests variant-windows output with the same `var_fields` and asserts `win.fields["AF"]` etc. match the oracle, plus an empty-group case asserting the dummy fill. + +- [ ] **Step 6: Commit (test only; may be red until Phase 2 done)** + +```bash +git add tests/dataset/test_svar2_fields_read.py +git commit -m "test(svar2): oracle test for INFO/FORMAT field routing (var_key+dense, multi-contig, union)" +``` + +### Task 3.2: Full suite + docs + +**Files:** +- Modify: `skills/genvarloader/SKILL.md` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Run the full svar2 + variants suite** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -5 +pixi run pytest tests/dataset -k svar2 -q 2>&1 | tail -30 +``` + +Expected: all PASS. + +- [ ] **Step 2: Update SKILL.md** + +Document that svar2 `variants`/variant-windows outputs now honor arbitrary store INFO/FORMAT fields via `var_fields`, that available fields come from the store's manifest (`available_var_fields`), and the dtype/missing-value semantics (stored dtype preserved; missing = default/sentinel/NaN). + +- [ ] **Step 3: Add CHANGELOG entry** + +Under `## Unreleased`: + +```markdown +### Added +- SVAR2 datasets now route arbitrary scalar-numeric INFO/FORMAT fields (stored via + `genoray.SparseVar2.from_vcf(info_fields=, format_fields=)`) into `variants` + (`RaggedVariants`) and variant-windows outputs, selectable via `var_fields`. +``` + +- [ ] **Step 4: Commit** + +```bash +git add skills/genvarloader/SKILL.md CHANGELOG.md +git commit -m "docs(svar2): document INFO/FORMAT field routing in variants outputs" +``` + +- [ ] **Step 5: Push + update the draft PR** + +```bash +git push origin svar2-m6b-kernel +``` + +The draft PR #266 updates automatically. Note in the PR description that the genoray Python pin is a local dev wheel pending a genoray svar-2 release (the version constraint was dropped for development). + +--- + +## Self-Review + +**Spec coverage:** +- Rust seam (provenance + FieldView) → Tasks 1.2, 1.3. ✓ +- gvl→genoray-main migration (discovered; not in spec but a hard prerequisite) → Phase 0. ✓ (larger than the spec's "prerequisite" section anticipated — see handoff.) +- Field discovery (`available_fields`) → Task 2.1. ✓ +- `RaggedVariants` routing → Task 2.3; variant-windows → Task 2.4. ✓ +- Dense-FORMAT `orig_samples[q]` stride subtlety → Task 1.2 Step 3 (`row * cohort_n_samples + orig_samples[q]`). ✓ +- `unphased_union` / multi-contig need no special handling → Tasks 2.3/2.4 use the same `src`/`var_off` machinery. ✓ (tested in 3.1 with `union` param.) +- Provenance-on-cursor invariant + identity test → Task 1.2 Steps 1–4. ✓ +- Testing (var_key/dense/multi-contig/union/missing) → Task 3.1. ✓ +- Docs (SKILL.md, CHANGELOG) → Task 3.2. ✓ +- `_impl.py` SVAR1-lazy-load hazard (discovered) → Task 2.2. ✓ + +**Placeholder scan:** Phase 0 Task 0.3 Step 4 is intentionally compiler-driven ("fix each reported site as in Steps 1–3") — legitimate for a mechanical type migration, with the known edits shown concretely. All feature code steps show full code. + +**Type consistency:** `decode_variants_from_split` new signature (Task 1.2 Interfaces) matches its call in Task 1.3 Step 2 and the no-field callers updated in Task 1.2 Step 3. FFI return tuple (Task 1.3) matches the Python unpack in Tasks 2.3/2.4. `store_fields` value shape `(category, name, dtype_str, np_dtype)` set in Task 2.1 matches use in 2.3/2.4. `FieldSub::all()` order asserted consistently (VkSnp,VkIndel,DenseSnp,DenseIndel) in Task 1.2 and 1.3. + +## Known risks / verify-during-execution + +- **`StorageDtype::from_meta_str` / `FieldView::open` / `FieldSub::all` exact signatures** — verify against genoray `src/field.rs`, `src/query/field.rs`, `src/layout.rs` at build time; the `Result` vs `Option` shape may need a `.map_err`/`.ok_or_else` tweak (Task 1.3 Step 1 notes this). +- **`ContigReader` has no `paths()` accessor** — we rebuild `ContigPaths::new(store.store_path(), contig)` (Task 1.1 + 1.3). Confirm `ContigPaths::new(base_out_dir, chrom)` is the correct constructor and that its `field_values(cat,name,sub)` layout matches what the writer produced. +- **`unphased_union` API name** — Task 3.1 uses `with_settings(unphased_union=True)`; confirm the actual gvl setter. +- **`DummyVariant` construction site for svar2 windows** — Task 2.4 Step 3 must locate where (if anywhere) `self.dummy_variant` is set for `Svar2Haps`; if unset, construct one carrying per-field fills. From 823b290f30fbe3975c2d63a252888f6fe6cef386 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 01:02:40 -0700 Subject: [PATCH 075/108] chore(svar2): pin genoray main wheel, drop version constraint for dev gvl is being built against genoray main, which carries the unreleased INFO/FORMAT field-read API. genoray's version was never bumped past the 3.0.0 tag, so its wheel reports 2.15.0 and the >=3,<4 pin matched nothing (the env was unsolvable). Drop the constraint and point pixi at a freshly built abi3 manylinux wheel. Re-pin at genoray release. --- pixi.lock | 1042 ++++++++++++++++++++++++++---------------------- pixi.toml | 14 +- pyproject.toml | 4 +- 3 files changed, 566 insertions(+), 494 deletions(-) diff --git a/pixi.lock b/pixi.lock index be4ac2a9..b3831f05 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,7 +1,16 @@ version: 7 platforms: - name: linux-64 + virtual-packages: + - __unix=0=0 + - __linux=4.18 + - __glibc=2.28 + - __archspec=0=x86_64 - name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 environments: default: channels: @@ -164,8 +173,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -174,6 +183,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl @@ -181,12 +191,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl @@ -208,16 +217,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -338,22 +345,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl @@ -369,7 +375,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -383,22 +388,22 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl @@ -548,21 +553,19 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl @@ -570,23 +573,30 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl @@ -594,45 +604,40 @@ environments: - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -641,11 +646,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl @@ -758,21 +762,20 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl @@ -787,7 +790,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -800,21 +802,21 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl @@ -983,7 +985,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl @@ -1232,7 +1234,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/03/78/843bcf0cf31f88d2f8a9a063d2d80817b1901657d83d65b89b3aa835732e/nbsphinx-0.9.8-py3-none-any.whl @@ -1514,7 +1516,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.10.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl @@ -1794,7 +1796,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/03/78/843bcf0cf31f88d2f8a9a063d2d80817b1901657d83d65b89b3aa835732e/nbsphinx-0.9.8-py3-none-any.whl @@ -1979,14 +1981,16 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.4.22-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-8_cp310.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/05/00/5820a3b1fe264b23770f77dbb3ac0f6fdadd21b640624791a05950f934da/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl @@ -1995,18 +1999,16 @@ environments: - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/39/f8/a9af1b0e78a4d2a3974eac8289d97f83526346571f5ef3edda13bcff262d/charset_normalizer-3.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz @@ -2033,19 +2035,18 @@ environments: - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl @@ -2055,7 +2056,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f5/1a/60a505991247430fd9e4429526292e2b054381615801f3c7efdfa08dc9fb/hypothesis-6.156.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -2088,14 +2088,13 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/1e/55/f7590fd3a3b9c80b7106876bc868827304462cbf9b40b0f3d664c04c4e67/charset_normalizer-3.4.8-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl @@ -2105,13 +2104,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz @@ -2125,38 +2123,38 @@ environments: - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8f/fe/a4867bc2b8b81d9b1648992fb7e4a732b3db480ff2d02df2c7b59189c812/hypothesis-6.156.6-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/a2/cf/e657b924ab6f24f29841ad2e461775a0f048c24013df5be5099be98303ca/hypothesis-6.156.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl @@ -2435,8 +2433,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -2445,19 +2443,19 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl @@ -2478,14 +2476,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -2673,21 +2669,20 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h4818236_10.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl @@ -2702,7 +2697,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -2714,18 +2708,18 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl @@ -2895,8 +2889,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -2905,6 +2899,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl @@ -2912,12 +2907,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl @@ -2939,16 +2933,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -3069,22 +3061,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl @@ -3100,7 +3091,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -3114,22 +3104,22 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl @@ -3297,7 +3287,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -3467,7 +3457,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py311hc949640_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/8f/16812ee742bbdda2746a15f5cc788cbaf4d329c35b2ca1f6cdf45685866c/ncls-0.0.70-cp311-cp311-macosx_11_0_arm64.whl @@ -3697,7 +3687,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -3867,7 +3857,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl @@ -4097,7 +4087,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -4268,7 +4258,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py313h0997733_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -4607,21 +4597,19 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl @@ -4629,23 +4617,30 @@ environments: - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl @@ -4653,41 +4648,36 @@ environments: - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -4695,11 +4685,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl @@ -4880,21 +4869,20 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h4818236_10.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl @@ -4909,7 +4897,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -4921,18 +4908,18 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl @@ -6813,7 +6800,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/maturin?source=compressed-mapping + - pkg:pypi/maturin?source=hash-mapping size: 9793377 timestamp: 1778496710620 - conda: https://conda.anaconda.org/conda-forge/linux-64/memray-1.19.3-py310hbdcf458_0.conda @@ -7469,7 +7456,8 @@ packages: - libxslt >=1.1.43,<2.0a0 license: LGPL-3.0-only purls: - - pkg:pypi/pyside6?source=compressed-mapping + - pkg:pypi/pyside6?source=hash-mapping + - pkg:pypi/shiboken6?source=hash-mapping size: 13559182 timestamp: 1778540466597 - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -9129,7 +9117,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/matplotlib-inline?source=compressed-mapping + - pkg:pypi/matplotlib-inline?source=hash-mapping size: 15725 timestamp: 1778264403247 - conda: https://conda.anaconda.org/conda-forge/noarch/mdit-py-plugins-0.6.0-pyhd8ed1ab_0.conda @@ -9747,7 +9735,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/tomlkit?source=compressed-mapping + - pkg:pypi/tomlkit?source=hash-mapping size: 41169 timestamp: 1778423744478 - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.15.0-pyhcf101f3_0.conda @@ -11624,11 +11612,11 @@ packages: purls: [] size: 433413 timestamp: 1764777166076 -- pypi: . +- pypi: ./ name: genvarloader requires_dist: - seqpro>=0.20 - - genoray>=2.12.3,<3 + - genoray - numpy - loguru - natsort @@ -11653,7 +11641,7 @@ packages: - tbb ; extra == 'tbb' - pyomp ; extra == 'pyomp' requires_python: '>=3.10,<3.14' -- pypi: /carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl +- pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl name: genoray version: 2.15.0 requires_dist: @@ -11665,7 +11653,7 @@ packages: - cyvcf2>=0.31.1 - pysam>=0.22 - polars>=1.37.1 - - polars-bio>=0.20.1,<0.21 + - polars-bio>=0.20.1,<0.34 - pyranges>=0.1.3 - typing-extensions>=4.14 - pyarrow>=21 @@ -11890,11 +11878,49 @@ packages: version: 4.1.0 sha256: 647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - name: nvidia-cuda-nvrtc-cu12 - version: 12.8.93 - sha256: a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994 - requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/05/00/5820a3b1fe264b23770f77dbb3ac0f6fdadd21b640624791a05950f934da/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: hypothesis + version: 6.156.6 + sha256: c7c3f067166bfc28b67f0042fc5fdcc87b3b58664da0c2f563fe544448b7ab3b + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-cusparse-cu12 version: 12.5.4.2 @@ -12122,11 +12148,6 @@ packages: - pytest ; extra == 'test' - hypothesis[numpy] ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cuda-runtime-cu12 - version: 12.8.90 - sha256: adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl name: pytorch-lightning version: 2.6.1 @@ -12382,6 +12403,26 @@ packages: version: 0.5.2 sha256: 6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: triton + version: 3.7.1 + sha256: ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e + requires_dist: + - importlib-metadata ; python_full_version < '3.10' + - cmake>=3.20,<4.0 ; extra == 'build' + - lit ; extra == 'build' + - autopep8 ; extra == 'tests' + - isort ; extra == 'tests' + - numpy ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-forked ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - scipy>=1.7.1 ; extra == 'tests' + - llnl-hatchet ; extra == 'tests' + - matplotlib ; extra == 'tutorials' + - pandas ; extra == 'tutorials' + - tabulate ; extra == 'tutorials' + requires_python: '>=3.10,<3.15' - pypi: https://files.pythonhosted.org/packages/15/ef/7d57ceb0651af74194e97ed6583e148d352f03d696090221b8059cdfc90b/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: polars-runtime-32 version: 1.40.1 @@ -12415,39 +12456,6 @@ packages: - requests ; extra == 'telegram' - ipywidgets>=6 ; extra == 'notebook' requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl - name: torch - version: 2.10.0 - sha256: a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321 - requires_dist: - - filelock - - typing-extensions>=4.10.0 - - setuptools ; python_full_version >= '3.12' - - sympy>=1.13.3 - - networkx>=2.5.1 - - jinja2 - - fsspec>=0.8.5 - - cuda-bindings==12.9.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-nvrtc-cu12==12.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-runtime-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-cupti-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cudnn-cu12==9.10.2.21 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cublas-cu12==12.8.4.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cufft-cu12==11.3.3.83 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-curand-cu12==10.3.9.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusolver-cu12==11.7.3.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusparse-cu12==12.5.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusparselt-cu12==0.7.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nccl-cu12==2.27.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvshmem-cu12==3.4.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvtx-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvjitlink-cu12==12.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cufile-cu12==1.13.1.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - triton==3.6.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - optree>=0.13.0 ; extra == 'optree' - - opt-einsum>=3.3 ; extra == 'opt-einsum' - - pyyaml ; extra == 'pyyaml' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl name: pydantic-extra-types version: 2.11.1 @@ -12476,17 +12484,6 @@ packages: - semver>=3.0.2 ; extra == 'semver' - uuid-utils>=0.6.0 ; python_full_version < '3.14' and extra == 'uuid-utils' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - name: pyarrow - version: 21.0.0 - sha256: e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26 - requires_dist: - - pytest ; extra == 'test' - - hypothesis ; extra == 'test' - - cffi ; extra == 'test' - - pytz ; extra == 'test' - - pandas ; extra == 'test' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl name: websockets version: '16.0' @@ -12548,11 +12545,6 @@ packages: - polars-config-meta[polars]>=0.3.2 - attrs requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/1e/55/f7590fd3a3b9c80b7106876bc868827304462cbf9b40b0f3d664c04c4e67/charset_normalizer-3.4.8-cp310-cp310-macosx_10_9_universal2.whl - name: charset-normalizer - version: 3.4.8 - sha256: ad320cccde0bbe430fe3a0e5ef3f17b16ef7cfe496603d7b3a153aa204e093e5 - requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl name: idna version: '3.18' @@ -12562,13 +12554,6 @@ packages: - mypy>=1.11.2 ; extra == 'all' - pytest>=8.3.2 ; extra == 'all' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cufft-cu12 - version: 11.3.3.83 - sha256: 4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 - requires_dist: - - nvidia-nvjitlink-cu12 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl name: pgenlib version: 0.94.1 @@ -12581,6 +12566,20 @@ packages: version: 0.24.0 sha256: 98a40440b16bbf67b696914a10acc241a0cd0fa02dcb14aee8e614eb2c5d8b37 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + name: tqdm + version: 4.68.4 + sha256: 5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2 + requires_dist: + - colorama ; sys_platform == 'win32' + - requests ; extra == 'discord' + - envwrap ; extra == 'discord' + - slack-sdk ; extra == 'slack' + - envwrap ; extra == 'slack' + - requests ; extra == 'telegram' + - envwrap ; extra == 'telegram' + - ipywidgets>=6 ; extra == 'notebook' + requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl name: certifi version: 2026.4.22 @@ -12846,6 +12845,11 @@ packages: - torchinfo>=1.8.0 - torchmetrics>=1.6.2 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cuda-runtime + version: 13.0.96 + sha256: 7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/2e/66/70786ee1cfdd03d36d456c4ef02a35506b7ae256c70a74bd7abf135daba0/arro3_core-0.8.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: arro3-core version: 0.8.0 @@ -12994,6 +12998,11 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl + name: nvidia-cuda-cupti + version: 13.0.85 + sha256: 4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/34/0b/b9d1911cfefa61399821dfb37f486d83e0f42630a8d12f7194270c417002/llvmlite-0.47.0-cp311-cp311-macosx_11_0_arm64.whl name: llvmlite version: 0.47.0 @@ -13006,6 +13015,10 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.12' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl + name: nvidia-cusparselt-cu13 + version: 0.8.1 + sha256: 786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0 - pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl name: sphinxcontrib-devhelp version: 2.0.0 @@ -13056,11 +13069,6 @@ packages: version: 0.5.2 sha256: b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/39/f8/a9af1b0e78a4d2a3974eac8289d97f83526346571f5ef3edda13bcff262d/charset_normalizer-3.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: charset-normalizer - version: 3.4.8 - sha256: 511851c35e4ec8f4be773acd2c0222a7ec6b63971fea8988103632896f174b4b - requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl name: zipp version: 4.1.0 @@ -13085,6 +13093,18 @@ packages: - pytest-enabler>=3.4 ; extra == 'enabler' - pytest-mypy>=1.0.1 ; platform_python_implementation != 'PyPy' and extra == 'type' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cublas + version: 13.1.1.3 + sha256: 37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436 + requires_dist: + - nvidia-cuda-nvrtc + requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-nvshmem-cu13 + version: 3.4.5 + sha256: 290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/3e/fe/1624eb5024e897bf4074bfc31f9e5e823160aed1ac14e7720e849a3d1109/selectolax-0.4.8-cp313-cp313-macosx_11_0_arm64.whl name: selectolax version: 0.4.8 @@ -13105,6 +13125,11 @@ packages: version: 4.0.15 sha256: 8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366 requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cufile + version: 1.15.1.6 + sha256: 08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/40/03/f5d0b979c6a1f8a8a11ba115a7c5b145671f092372a4ede164dc2597c466/ncls-0.0.70-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: ncls version: 0.0.70 @@ -13286,17 +13311,6 @@ packages: - numpy>=1.22 - numpy>=1.22,<2.5 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - name: pyarrow - version: 21.0.0 - sha256: 26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51 - requires_dist: - - pytest ; extra == 'test' - - hypothesis ; extra == 'test' - - cffi ; extra == 'test' - - pytz ; extra == 'test' - - pandas ; extra == 'test' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: llvmlite version: 0.47.0 @@ -13307,6 +13321,11 @@ packages: version: 2.8.3 sha256: ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + name: pyarrow + version: 22.0.0 + sha256: 710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/47/d4/dbacced3953544b9a93088cc10ef2b596d348c983d5c67a404fa41ec51ba/fonttools-4.62.1-cp312-cp312-macosx_10_13_universal2.whl name: fonttools version: 4.62.1 @@ -13351,11 +13370,6 @@ packages: version: 4.16.0 sha256: 481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl - name: filelock - version: 3.29.5 - sha256: 8af830889ba3a0ffcefbd6c7d2af8a54012058103771f2e10848222f476a1693 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl name: seqpro version: 0.20.0 @@ -13408,13 +13422,29 @@ packages: - pytest ; extra == 'test' - hypothesis[numpy] ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/52/82/4254fd5cdb78075f1e32d145b2dc286ed4ba42a94e24672fabc5998281a0/selectolax-0.4.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: selectolax - version: 0.4.10 - sha256: cd436b0143edb5e1c4f126eeb4647344bedaab9e2a379eddb528ffac33c7adad +- pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl + name: torch + version: 2.13.0 + sha256: c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4 requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' + - filelock + - typing-extensions>=4.10.0 + - setuptools>=77.0.3 + - sympy>=1.13.3 + - networkx>=2.5.1 + - jinja2 + - fsspec>=0.8.5 + - cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.3 ; sys_platform == 'linux' + - cuda-bindings>=13.0.3,<14 ; python_full_version < '3.15' and sys_platform == 'linux' + - nvidia-cudnn-cu13==9.20.0.48 ; sys_platform == 'linux' + - nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' + - nvidia-nccl-cu13==2.29.7 ; sys_platform == 'linux' + - nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' + - triton==3.7.1 ; python_full_version < '3.15' and sys_platform == 'linux' + - optree>=0.13.0 ; extra == 'optree' + - opt-einsum>=3.3 ; extra == 'opt-einsum' + - pyyaml ; extra == 'pyyaml' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl name: sphinxcontrib-serializinghtml version: 2.0.0 @@ -13787,6 +13817,15 @@ packages: version: 2.0.0 sha256: 5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96 requires_python: '>=3.10,<3.15' +- pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cusolver + version: 12.0.4.66 + sha256: 0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112 + requires_dist: + - nvidia-cublas + - nvidia-nvjitlink + - nvidia-cusparse + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: pydantic-core version: 2.46.4 @@ -13805,11 +13844,33 @@ packages: - isort ; extra == 'dev' - pip-tools ; extra == 'dev' - pytest ; extra == 'dev' +- pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl + name: filelock + version: 3.29.7 + sha256: 987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: rpds-py version: 0.30.0 sha256: 47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: polars-bio + version: 0.28.0 + sha256: 3cbff526d4af4d06bef3bd25f70b0c77a12c95b4e578a4831199cd92c79f85e7 + requires_dist: + - polars>=1.37.1 + - pyarrow>=21.0.0,<23 ; python_full_version < '3.14' + - pyarrow>=22.0.0,<23 ; python_full_version >= '3.14' + - datafusion>=50.0.0,<51 + - tqdm>=4.67.0,<5 + - typing-extensions>=4.14.0,<5 + - polars-config-meta>=0.3.0,<1 + - pandas ; extra == 'pandas' + - pysam ; extra == 'test' + - bioframe ; extra == 'viz' + - matplotlib ; extra == 'viz' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl name: comm version: 0.2.3 @@ -13944,6 +14005,11 @@ packages: - multidict>=4.0 - propcache>=0.2.1 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl + name: nvidia-nccl-cu13 + version: 2.29.7 + sha256: edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl name: nbconvert version: 7.17.1 @@ -14101,6 +14167,13 @@ packages: - coloredlogs - click requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cudnn-cu13 + version: 9.20.0.48 + sha256: 0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304 + requires_dist: + - nvidia-cublas + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-nccl-cu12 version: 2.27.5 @@ -14365,22 +14438,6 @@ packages: version: 1.1.0 sha256: 1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - name: cuda-bindings - version: 12.9.4 - sha256: 4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5 - requires_dist: - - cuda-pathfinder~=1.1 - - nvidia-cuda-nvcc-cu12 ; extra == 'all' - - nvidia-cuda-nvrtc-cu12 ; extra == 'all' - - nvidia-nvjitlink-cu12>=12.3 ; extra == 'all' - - nvidia-cufile-cu12 ; sys_platform == 'linux' and extra == 'all' - - cython>=3.1,<3.2 ; extra == 'test' - - setuptools>=77.0.0 ; extra == 'test' - - numpy>=1.21.1 ; extra == 'test' - - pytest>=6.2.4 ; extra == 'test' - - pytest-benchmark>=3.4.1 ; extra == 'test' - - pyglet>=2.1.9 ; extra == 'test' - pypi: https://files.pythonhosted.org/packages/7a/f6/51d8a97116de23c9280c1fa3b813bc088f8571ce5936ba84af1ecf13ed45/pybigwig-0.3.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: pybigwig version: 0.3.25 @@ -14568,13 +14625,6 @@ packages: version: 4.10.0 sha256: fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/82/1d/4591bf79e2e50d91373330fa28af995453e4f93032b81a4f019c2299c7da/selectolax-0.4.10-cp310-cp310-macosx_11_0_arm64.whl - name: selectolax - version: 0.4.10 - sha256: 4d06ddf24ebcddead2c5a2ed96607f46651db899b8f331a16481d88a91d5c9af - requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl name: rich version: 15.0.0 @@ -14713,15 +14763,6 @@ packages: - testpath ; extra == 'test' - xmltodict ; extra == 'test' requires_python: '>=3.10.0' -- pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-cusolver-cu12 - version: 11.7.3.90 - sha256: 4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450 - requires_dist: - - nvidia-cublas-cu12 - - nvidia-nvjitlink-cu12 - - nvidia-cusparse-cu12 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl name: ncls version: 0.0.70 @@ -14856,26 +14897,6 @@ packages: - tensorboard>=2.11 ; extra == 'dev' - huggingface-hub ; extra == 'dev' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - name: triton - version: 3.6.0 - sha256: a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea - requires_dist: - - importlib-metadata ; python_full_version < '3.10' - - cmake>=3.20,<4.0 ; extra == 'build' - - lit ; extra == 'build' - - autopep8 ; extra == 'tests' - - isort ; extra == 'tests' - - numpy ; extra == 'tests' - - pytest ; extra == 'tests' - - pytest-forked ; extra == 'tests' - - pytest-xdist ; extra == 'tests' - - scipy>=1.7.1 ; extra == 'tests' - - llnl-hatchet ; extra == 'tests' - - matplotlib ; extra == 'tutorials' - - pandas ; extra == 'tutorials' - - tabulate ; extra == 'tutorials' - requires_python: '>=3.10,<3.15' - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl name: accessible-pygments version: 0.0.5 @@ -14966,6 +14987,49 @@ packages: requires_dist: - nvidia-nvjitlink-cu12 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/8f/fe/a4867bc2b8b81d9b1648992fb7e4a732b3db480ff2d02df2c7b59189c812/hypothesis-6.156.6-cp310-cp310-macosx_11_0_arm64.whl + name: hypothesis + version: 6.156.6 + sha256: 583a658162ee7e1a82155e3f146932a3a2269030294f3c83f5cf52fcaa0562c3 + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl name: typeguard version: 4.5.1 @@ -15413,54 +15477,6 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.12' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/a2/cf/e657b924ab6f24f29841ad2e461775a0f048c24013df5be5099be98303ca/hypothesis-6.156.1-cp310-cp310-macosx_11_0_arm64.whl - name: hypothesis - version: 6.156.1 - sha256: ad6e5960b36452f3b904a3849e85c62377cf4ecc904541706c934dae439ce933 - requires_dist: - - exceptiongroup>=1.0.0 ; python_full_version < '3.11' - - sortedcontainers>=2.1.0,<3.0.0 - - black>=20.8b0 ; extra == 'all' - - click>=7.0 ; extra == 'all' - - crosshair-tool>=0.0.107 ; extra == 'all' - - django>=5.2 ; extra == 'all' - - dpcontracts>=0.4 ; extra == 'all' - - hypothesis-crosshair>=0.0.28 ; extra == 'all' - - lark>=0.10.1 ; extra == 'all' - - libcst>=0.3.16 ; extra == 'all' - - numpy>=1.21.6 ; extra == 'all' - - pandas>=1.1 ; extra == 'all' - - pytest>=4.6 ; extra == 'all' - - python-dateutil>=1.4 ; extra == 'all' - - pytz>=2014.1 ; extra == 'all' - - redis>=3.0.0 ; extra == 'all' - - rich>=9.0.0 ; extra == 'all' - - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') - - watchdog>=4.0.0 ; extra == 'all' - - click>=7.0 ; extra == 'cli' - - black>=20.8b0 ; extra == 'cli' - - rich>=9.0.0 ; extra == 'cli' - - libcst>=0.3.16 ; extra == 'codemods' - - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' - - crosshair-tool>=0.0.107 ; extra == 'crosshair' - - python-dateutil>=1.4 ; extra == 'dateutil' - - django>=5.2 ; extra == 'django' - - dpcontracts>=0.4 ; extra == 'dpcontracts' - - black>=20.8b0 ; extra == 'ghostwriter' - - lark>=0.10.1 ; extra == 'lark' - - numpy>=1.21.6 ; extra == 'numpy' - - pandas>=1.1 ; extra == 'pandas' - - pytest>=4.6 ; extra == 'pytest' - - pytz>=2014.1 ; extra == 'pytz' - - redis>=3.0.0 ; extra == 'redis' - - watchdog>=4.0.0 ; extra == 'watchdog' - - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') - requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-nvtx-cu12 - version: 12.8.90 - sha256: 5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl name: watchfiles version: 1.1.1 @@ -15468,6 +15484,11 @@ packages: requires_dist: - anyio>=3.0.0 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-curand + version: 10.4.0.35 + sha256: 1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl name: semver version: 3.0.4 @@ -15492,6 +15513,24 @@ packages: - pydoctor>=25.4.0 ; extra == 'docs' - pytest ; extra == 'test' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + name: cuda-bindings + version: 13.3.1 + sha256: 9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0 + requires_dist: + - cuda-pathfinder>=1.4.2 + - cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm]==13.* ; extra == 'all' + - cuda-toolkit[cufile]==13.* ; sys_platform == 'linux' and extra == 'all' + - cuda-toolkit==13.* ; extra == 'all' + - nvidia-cudla==13.* ; platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cufft + version: 12.0.0.61 + sha256: 6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3 + requires_dist: + - nvidia-nvjitlink + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl name: nbformat version: 5.10.4 @@ -15603,6 +15642,11 @@ packages: - pytest-cov ; extra == 'test' - coverage-conditional-plugin ; extra == 'test' requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl + name: charset-normalizer + version: 3.4.9 + sha256: cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl name: pyarrow version: 21.0.0 @@ -15753,6 +15797,45 @@ packages: - pytest-timeout ; extra == 'testing' - requests ; extra == 'testing' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl + name: cyclopts + version: 4.21.0 + sha256: ded3ddb15b0c815f44d245011fc4cdd5a4809a3bb8202869e9e02195a87c0e18 + requires_dist: + - attrs>=23.1.0 + - docstring-parser>=0.15,<4.0 + - rich-rst>=1.3.1,<3.0.0 + - rich>=13.6.0 + - tomli>=2.0.0 ; python_full_version < '3.11' + - typing-extensions>=4.8.0 ; python_full_version < '3.11' + - ipdb>=0.13.9 ; extra == 'debug' + - line-profiler>=3.5.1 ; extra == 'debug' + - coverage[toml]>=5.1 ; extra == 'dev' + - mkdocs>=1.4.0 ; extra == 'dev' + - pexpect>=4.9.0 ; sys_platform != 'win32' and extra == 'dev' + - pre-commit>=2.16.0 ; extra == 'dev' + - pydantic>=2.11.2,<3.0.0 ; extra == 'dev' + - pytest-cov>=3.0.0 ; extra == 'dev' + - pytest-mock>=3.7.0 ; extra == 'dev' + - pytest>=8.2.0 ; extra == 'dev' + - pyyaml>=6.0.1 ; extra == 'dev' + - syrupy>=4.0.0 ; extra == 'dev' + - toml>=0.10.2,<1.0.0 ; extra == 'dev' + - trio>=0.10.0 ; extra == 'dev' + - gitpython>=3.1.31 ; extra == 'docs' + - myst-parser[linkify]>=3.0.1,<5.0.0 ; extra == 'docs' + - sphinx-autodoc-typehints>=1.25.2,<4.0.0 ; extra == 'docs' + - sphinx-copybutton>=0.5,<1.0 ; extra == 'docs' + - sphinx-rtd-dark-mode>=1.3.0,<2.0.0 ; extra == 'docs' + - sphinx-rtd-theme>=3.0.0,<4.0.0 ; extra == 'docs' + - sphinx>=7.4.7,<8.2.0 ; extra == 'docs' + - markdown>=3.3 ; extra == 'mkdocs' + - mkdocs>=1.4.0 ; extra == 'mkdocs' + - pymdown-extensions>=10.0 ; extra == 'mkdocs' + - tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'toml' + - trio>=0.10.0 ; extra == 'trio' + - pyyaml>=6.0.1 ; extra == 'yaml' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-nvshmem-cu12 version: 3.4.5 @@ -15931,11 +16014,6 @@ packages: - cffi~=1.17 ; python_full_version < '3.14' and platform_python_implementation != 'PyPy' and extra == 'cffi' - cffi>=2.0.0b0 ; python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and extra == 'cffi' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cufile-cu12 - version: 1.13.1.3 - sha256: 1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/bf/0f/f1c2e2592f995d301bc06901fbb2e947cbe4889397a418309710ccc3a1db/sphinx_autodoc_typehints-3.10.2-py3-none-any.whl name: sphinx-autodoc-typehints version: 3.10.2 @@ -15985,12 +16063,15 @@ packages: - flake8 ; extra == 'test' - mypy ; extra == 'test' requires_python: '>=3.5' -- pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cusparse-cu12 - version: 12.5.8.93 - sha256: 1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b - requires_dist: - - nvidia-nvjitlink-cu12 +- pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + name: nvidia-nvtx + version: 13.0.85 + sha256: 4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4 + requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + name: nvidia-cuda-nvrtc + version: 13.0.88 + sha256: ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575 requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl name: torchmetrics @@ -16294,6 +16375,23 @@ packages: - pytest-benchmark ; extra == 'devel' - pytest-cache ; extra == 'devel' - validictory ; extra == 'devel' +- pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl + name: polars-bio + version: 0.28.0 + sha256: f66e12c5e3f5fccc2f6b7124f348f6ce47fcae0be18b21dd2c11ea6350b0a55f + requires_dist: + - polars>=1.37.1 + - pyarrow>=21.0.0,<23 ; python_full_version < '3.14' + - pyarrow>=22.0.0,<23 ; python_full_version >= '3.14' + - datafusion>=50.0.0,<51 + - tqdm>=4.67.0,<5 + - typing-extensions>=4.14.0,<5 + - polars-config-meta>=0.3.0,<1 + - pandas ; extra == 'pandas' + - pysam ; extra == 'test' + - bioframe ; extra == 'viz' + - matplotlib ; extra == 'viz' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: contourpy version: 1.3.3 @@ -16354,11 +16452,6 @@ packages: - fsspec>=2025.5.1 - typing-extensions ; python_full_version < '3.10' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - name: tzdata - version: '2026.2' - sha256: bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 - requires_python: '>=2' - pypi: https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl name: pandas version: 3.0.3 @@ -16466,6 +16559,70 @@ packages: version: 0.24.0 sha256: b9445a4c3be5ed4b60202690af3890a444452276372e3abb58564308cc6d5a45 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl + name: cuda-toolkit + version: 13.0.3.0 + sha256: d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f + requires_dist: + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cccl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-culibos==13.0.85.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cupti==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cuxxfilt==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-nvcc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-opencl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-profiler-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-sanitizer-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cufft==12.0.0.61.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cufile==1.15.1.6.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-curand==10.4.0.35.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cusolver==12.0.4.66.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-npp==13.0.1.2.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvfatbin==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvjpeg==13.0.1.86.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvml-dev==13.0.87.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvptxcompiler==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvtx==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cccl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cccl') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cccl') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cccl') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'crt') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'crt') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'crt') + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cublas') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cublas') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cublas') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cublas') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cublas') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cublas') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cudart') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cudart') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cudart') + - nvidia-cufft==12.0.0.61.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cufft') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufft') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufft') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cufft') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufft') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufft') + - nvidia-cufile==1.15.1.6.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufile') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufile') + - nvidia-cuda-culibos==13.0.85.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'culibos') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'culibos') + - nvidia-cuda-cupti==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cupti') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cupti') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cupti') + - nvidia-curand==10.4.0.35.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'curand') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'curand') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'curand') + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusolver==12.0.4.66.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusparse') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusparse') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusparse') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusparse') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusparse') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusparse') + - nvidia-cuda-cuxxfilt==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cuxxfilt') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cuxxfilt') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cuxxfilt') + - nvidia-npp==13.0.1.2.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'npp') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'npp') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'npp') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-cuda-nvcc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-nvfatbin==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvfatbin') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvfatbin') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvfatbin') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvjitlink') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvjitlink') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvjitlink') + - nvidia-nvjpeg==13.0.1.86.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvjpeg') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvjpeg') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvjpeg') + - nvidia-nvml-dev==13.0.87.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvml') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvml') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvml') + - nvidia-nvptxcompiler==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvptxcompiler') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvptxcompiler') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvptxcompiler') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvrtc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvrtc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvrtc') + - nvidia-nvtx==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvtx') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvtx') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvtx') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvvm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvvm') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvvm') + - nvidia-cuda-opencl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'opencl') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'opencl') + - nvidia-cuda-profiler-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'profiler') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'profiler') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'profiler') + - nvidia-cuda-sanitizer-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'sanitizer') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'sanitizer') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'sanitizer') - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl name: cuda-pathfinder version: 1.5.6 @@ -16680,20 +16837,6 @@ packages: - httpx ; extra == 'test' - pytest>=6 ; extra == 'test' requires_python: '>=3.11' -- pypi: https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl - name: tqdm - version: 4.68.3 - sha256: 39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03 - requires_dist: - - colorama ; sys_platform == 'win32' - - requests ; extra == 'discord' - - envwrap ; extra == 'discord' - - slack-sdk ; extra == 'slack' - - envwrap ; extra == 'slack' - - requests ; extra == 'telegram' - - envwrap ; extra == 'telegram' - - ipywidgets>=6 ; extra == 'notebook' - requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl name: pillow version: 12.2.0 @@ -16745,6 +16888,11 @@ packages: requires_dist: - pygments requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl + name: pyarrow + version: 22.0.0 + sha256: 77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl name: anyio version: 4.13.0 @@ -16779,11 +16927,6 @@ packages: - pyarrow>=11.0.0 - typing-extensions ; python_full_version < '3.13' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-cublas-cu12 - version: 12.8.4.1 - sha256: 8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl name: typing-inspection version: 0.4.2 @@ -16842,45 +16985,6 @@ packages: - ruff>=0.12.0 ; extra == 'dev' - cython-lint>=0.12.2 ; extra == 'dev' requires_python: '>=3.12' -- pypi: https://files.pythonhosted.org/packages/dd/d4/7243bc65d33c5ff5150569c42c8c1154aadae20b25638afe35f7f568489c/cyclopts-4.20.0-py3-none-any.whl - name: cyclopts - version: 4.20.0 - sha256: 0b4337e9c11303d86b33d3f37c629dc01638f84591681e0e5611286bdd507646 - requires_dist: - - attrs>=23.1.0 - - docstring-parser>=0.15,<4.0 - - rich-rst>=1.3.1,<3.0.0 - - rich>=13.6.0 - - tomli>=2.0.0 ; python_full_version < '3.11' - - typing-extensions>=4.8.0 ; python_full_version < '3.11' - - ipdb>=0.13.9 ; extra == 'debug' - - line-profiler>=3.5.1 ; extra == 'debug' - - coverage[toml]>=5.1 ; extra == 'dev' - - mkdocs>=1.4.0 ; extra == 'dev' - - pexpect>=4.9.0 ; sys_platform != 'win32' and extra == 'dev' - - pre-commit>=2.16.0 ; extra == 'dev' - - pydantic>=2.11.2,<3.0.0 ; extra == 'dev' - - pytest-cov>=3.0.0 ; extra == 'dev' - - pytest-mock>=3.7.0 ; extra == 'dev' - - pytest>=8.2.0 ; extra == 'dev' - - pyyaml>=6.0.1 ; extra == 'dev' - - syrupy>=4.0.0 ; extra == 'dev' - - toml>=0.10.2,<1.0.0 ; extra == 'dev' - - trio>=0.10.0 ; extra == 'dev' - - gitpython>=3.1.31 ; extra == 'docs' - - myst-parser[linkify]>=3.0.1,<5.0.0 ; extra == 'docs' - - sphinx-autodoc-typehints>=1.25.2,<4.0.0 ; extra == 'docs' - - sphinx-copybutton>=0.5,<1.0 ; extra == 'docs' - - sphinx-rtd-dark-mode>=1.3.0,<2.0.0 ; extra == 'docs' - - sphinx-rtd-theme>=3.0.0,<4.0.0 ; extra == 'docs' - - sphinx>=7.4.7,<8.2.0 ; extra == 'docs' - - markdown>=3.3 ; extra == 'mkdocs' - - mkdocs>=1.4.0 ; extra == 'mkdocs' - - pymdown-extensions>=10.0 ; extra == 'mkdocs' - - tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'toml' - - trio>=0.10.0 ; extra == 'trio' - - pyyaml>=6.0.1 ; extra == 'yaml' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl name: packaging version: '26.2' @@ -17056,6 +17160,11 @@ packages: - zstandard ; python_full_version < '3.14' and extra == 'test-full' - tqdm ; extra == 'tqdm' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + name: tzdata + version: '2026.3' + sha256: dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931 + requires_python: '>=2' - pypi: https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl name: tinycss2 version: 1.4.0 @@ -17109,6 +17218,11 @@ packages: version: 11.1.0 sha256: 4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.9 + sha256: 9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/e8/b3/7b0845952f65ecd3f02e8573ece4dd17c97ca27604cd28302c0467293e83/pgenlib-0.94.0-cp311-cp311-macosx_10_9_universal2.whl name: pgenlib version: 0.94.0 @@ -17294,6 +17408,11 @@ packages: - nvidia-nvjitlink-cu12 - nvidia-cusparse-cu12 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + name: nvidia-nvjitlink + version: 13.3.33 + sha256: 26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl name: stack-data version: 0.6.3 @@ -17347,54 +17466,6 @@ packages: requires_dist: - colorama>=0.4.6 ; extra == 'windows-terminal' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/f5/1a/60a505991247430fd9e4429526292e2b054381615801f3c7efdfa08dc9fb/hypothesis-6.156.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - name: hypothesis - version: 6.156.1 - sha256: 63131c150146b44f49dd6be4d417e1ee62390951403569043e759e8513763426 - requires_dist: - - exceptiongroup>=1.0.0 ; python_full_version < '3.11' - - sortedcontainers>=2.1.0,<3.0.0 - - black>=20.8b0 ; extra == 'all' - - click>=7.0 ; extra == 'all' - - crosshair-tool>=0.0.107 ; extra == 'all' - - django>=5.2 ; extra == 'all' - - dpcontracts>=0.4 ; extra == 'all' - - hypothesis-crosshair>=0.0.28 ; extra == 'all' - - lark>=0.10.1 ; extra == 'all' - - libcst>=0.3.16 ; extra == 'all' - - numpy>=1.21.6 ; extra == 'all' - - pandas>=1.1 ; extra == 'all' - - pytest>=4.6 ; extra == 'all' - - python-dateutil>=1.4 ; extra == 'all' - - pytz>=2014.1 ; extra == 'all' - - redis>=3.0.0 ; extra == 'all' - - rich>=9.0.0 ; extra == 'all' - - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') - - watchdog>=4.0.0 ; extra == 'all' - - click>=7.0 ; extra == 'cli' - - black>=20.8b0 ; extra == 'cli' - - rich>=9.0.0 ; extra == 'cli' - - libcst>=0.3.16 ; extra == 'codemods' - - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' - - crosshair-tool>=0.0.107 ; extra == 'crosshair' - - python-dateutil>=1.4 ; extra == 'dateutil' - - django>=5.2 ; extra == 'django' - - dpcontracts>=0.4 ; extra == 'dpcontracts' - - black>=20.8b0 ; extra == 'ghostwriter' - - lark>=0.10.1 ; extra == 'lark' - - numpy>=1.21.6 ; extra == 'numpy' - - pandas>=1.1 ; extra == 'pandas' - - pytest>=4.6 ; extra == 'pytest' - - pytz>=2014.1 ; extra == 'pytz' - - redis>=3.0.0 ; extra == 'redis' - - watchdog>=4.0.0 ; extra == 'watchdog' - - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') - requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - name: nvidia-nvjitlink-cu12 - version: 12.8.93 - sha256: 81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: scipy version: 1.18.0 @@ -17492,11 +17563,6 @@ packages: requires_dist: - numpy>=1.21.3 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cuda-cupti-cu12 - version: 12.8.90 - sha256: ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl name: pyzmq version: 27.1.0 @@ -17504,6 +17570,13 @@ packages: requires_dist: - cffi ; implementation_name == 'pypy' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cusparse + version: 12.6.3.3 + sha256: 2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b + requires_dist: + - nvidia-nvjitlink + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl name: llvmlite version: 0.47.0 @@ -17517,11 +17590,6 @@ packages: - frozenlist>=1.1.0 - typing-extensions>=4.2 ; python_full_version < '3.13' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-curand-cu12 - version: 10.3.9.90 - sha256: b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl name: click version: 8.4.2 diff --git a/pixi.toml b/pixi.toml index a68966de..4bfc3618 100644 --- a/pixi.toml +++ b/pixi.toml @@ -102,12 +102,14 @@ numba = "==0.59.1" pyarrow = ">=21" hirola = "==0.3" seqpro = "==0.21.1" -# Dev-wiring (release-gated): genoray svar-2 (v2.15.0, main merged in) as a pre-built -# cp310 wheel. Editable path-install would require genoray's rust-htslib C toolchain -# (cxx-compiler/clangdev/zlib/LIBCLANG_PATH) inside this env; Plan 3 never edits genoray, -# so a frozen wheel is equivalent. Rebuild via `pixi run -e py310 maturin build --release` -# in the genoray checkout if genoray changes. Flip to the PyPI release at merge. -genoray = { path = "/carter/users/dlaub/projects/genoray/target/wheels/genoray-2.15.0-cp310-cp310-manylinux_2_28_x86_64.whl" } +# Dev-wiring (release-gated): genoray MAIN (unreleased INFO/FORMAT field-read API) as a +# pre-built abi3 wheel. Editable path-install would require genoray's rust-htslib C toolchain +# (cxx-compiler/clangdev/zlib/LIBCLANG_PATH) inside this env; we never edit genoray, so a +# frozen wheel is equivalent. Rebuild via, in the genoray checkout: +# pixi run --manifest-path ci/wheel/pixi.toml build && ... repair # -> dist/*.whl +# The wheel reports 2.15.0 (genoray's version was never bumped past the 3.0.0 tag), hence the +# dropped version constraint in pyproject.toml. Flip both to the PyPI release at merge. +genoray = { path = "/carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl" } polars = "==1.37.1" loguru = "*" natsort = "*" diff --git a/pyproject.toml b/pyproject.toml index 1ec439f4..c037c802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ license = { file = "LICENSE.txt" } requires-python = ">=3.10,<3.14" # >= 3.14 blocked by pyarrow/genoray dependencies = [ "seqpro>=0.20", - "genoray>=3,<4", + # Unpinned for dev: gvl is being built against genoray main (unreleased field-read API), + # whose pyproject still reports 2.15.0. Re-pin (>=3,<4 or the new floor) at genoray release. + "genoray", "numpy", "loguru", "natsort", From e286ed51f9ea657fb012b05e83ea17a2050c15cd Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 01:02:41 -0700 Subject: [PATCH 076/108] refactor(svar2): migrate gvl read-bound Rust to genoray Range API genoray replaced the (usize, usize) tuple range fields on HapRanges and BatchResultSplit with std::ops::Range, so gvl's Rust no longer compiled against genoray main. Replace the four duplicated to_pairs closures with one shared arr2_to_ranges helper, read the BatchResultSplit dense ranges as Ranges, and update the test fixtures (BatchResultSplit now also carries vk_src; it derives Default). Pure type migration, no behavior change: 125/125 Rust tests pass and the existing SVAR2 suite is green (42 passed) against genoray main. --- Cargo.lock | 38 ++++++++++++++++++++++++++++ src/ffi/mod.rs | 65 +++++++++++++++++++----------------------------- src/svar2/mod.rs | 46 +++++++++++++++++++--------------- 3 files changed, 89 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c741a2e4..ace21d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -311,6 +311,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -498,6 +504,7 @@ version = "0.1.0" dependencies = [ "bytemuck", "crossbeam-channel", + "half", "memmap2", "ndarray", "ndarray-npy", @@ -557,6 +564,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1963,6 +1981,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zerofrom" version = "0.1.7" diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index cc452219..cc60ce29 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -5,6 +5,7 @@ use numpy::{ }; use pyo3::prelude::*; use pyo3::types::PyDict; +use std::ops::Range; use crate::variants::windows::{assemble_variants_mode, assemble_windows_mode, VariantBufs}; @@ -33,6 +34,14 @@ fn uninit_output(len: usize) -> Array1 { Array1::from_vec(v) } +/// Marshal a `(n, 2)` int64 array of `[start, end)` pairs into `Range`s. +fn arr2_to_ranges(a: numpy::ndarray::ArrayView2) -> Vec> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize)..(r[1] as usize)) + .collect() +} + /// Per-(query, hap) reference-length diffs (see `genotypes::get_diffs_sparse`). /// `geno_offsets` is the normalized (2, n) int64 starts/stops array. #[pyfunction] @@ -945,16 +954,10 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( .iter() .map(|&x| x as usize) .collect(); - let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { - a.rows() - .into_iter() - .map(|r| (r[0] as usize, r[1] as usize)) - .collect() - }; - let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); - let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); - let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); - let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); let ref_a = ref_.as_array(); let ref_offsets_a = ref_offsets.as_array(); @@ -1101,16 +1104,10 @@ pub fn hap_diffs_from_svar2_readbound<'py>( .iter() .map(|&x| x as usize) .collect(); - let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { - a.rows() - .into_iter() - .map(|r| (r[0] as usize, r[1] as usize)) - .collect() - }; - let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); - let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); - let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); - let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); let diffs = py.detach(move || { let rb = genoray_core::query::HapRanges::new( @@ -1208,16 +1205,10 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( .iter() .map(|&x| x as usize) .collect(); - let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { - a.rows() - .into_iter() - .map(|r| (r[0] as usize, r[1] as usize)) - .collect() - }; - let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); - let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); - let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); - let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); let tracks_a = tracks.as_array(); let track_offsets_a = track_offsets.as_array(); @@ -1357,16 +1348,10 @@ pub fn decode_variants_from_svar2_readbound<'py>( .iter() .map(|&x| x as usize) .collect(); - let to_pairs = |a: numpy::ndarray::ArrayView2| -> Vec<(usize, usize)> { - a.rows() - .into_iter() - .map(|r| (r[0] as usize, r[1] as usize)) - .collect() - }; - let vk_snp_range_v = to_pairs(vk_snp_range.as_array()); - let vk_indel_range_v = to_pairs(vk_indel_range.as_array()); - let dense_snp_range_v = to_pairs(dense_snp_range.as_array()); - let dense_indel_range_v = to_pairs(dense_indel_range.as_array()); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); let soa = py.detach(move || { let rb = genoray_core::query::HapRanges::new( diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index d5aa77dd..ac0c878a 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -3,6 +3,7 @@ //! intermediate variant table. Additive to the SVAR 1.0 global-table path. use std::borrow::Cow; +use std::ops::Range; use genoray_core::query::BatchResultSplit; use ndarray::{Array2, ArrayView2}; @@ -167,8 +168,8 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { let dense_total: usize = (0..n_q) .map(|q| { - let (ss, se) = br.dense_snp_range[q]; - let (is_, ie) = br.dense_indel_range[q]; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); (se - ss) + (ie - is_) }) .sum(); @@ -177,12 +178,12 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { let mut dense_range: Vec = Vec::with_capacity(n_q * 2); for q in 0..n_q { let base = dense_pos.len() as i32; - let (ss, se) = br.dense_snp_range[q]; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); for j in ss..se { dense_pos.push(br.dense_snp[j].position as i32); dense_key.push(br.dense_snp[j].key as i32); } - let (is_, ie) = br.dense_indel_range[q]; + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); for j in is_..ie { dense_pos.push(br.dense_indel[j].position as i32); dense_key.push(br.dense_indel[j].key as i32); @@ -198,8 +199,8 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { // width once and multiplying by `ploidy`. Same total, no per-h division. let total_bits: usize = (0..n_q) .map(|q| { - let (ss, se) = br.dense_snp_range[q]; - let (is_, ie) = br.dense_indel_range[q]; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); ((se - ss) + (ie - is_)) * ploidy }) .sum(); @@ -217,8 +218,8 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { // ploidy-many-times-redundant per-hap load. let mut h = 0usize; for q in 0..n_q { - let (ss, se) = br.dense_snp_range[q]; - let (is_, ie) = br.dense_indel_range[q]; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); for _hap in 0..ploidy { let snp_base = br.dense_snp_present_off[h]; for k in 0..(se - ss) { @@ -315,8 +316,8 @@ pub fn decode_variants_from_split( let mut h = 0usize; for q in 0..n_q { - let (ss, se) = br.dense_snp_range[q]; - let (is_, ie) = br.dense_indel_range[q]; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); for _hap in 0..ploidy { let vk_lo = br.vk_off[h]; let vk_hi = br.vk_off[h + 1]; @@ -501,16 +502,17 @@ mod tests { position: 10, key: 200, }], - dense_snp_range: vec![(0, 1)], + dense_snp_range: vec![0..1], dense_snp_present: vec![0b1], // present dense_snp_present_off: vec![0, 1], dense_indel: vec![KeyRef { position: 15, key: 300, }], - dense_indel_range: vec![(0, 1)], + dense_indel_range: vec![0..1], dense_indel_present: vec![0b0], // absent dense_indel_present_off: vec![0, 1], + ..Default::default() }; let flat = split_to_flat(&br); @@ -559,13 +561,14 @@ mod tests { position: 42, key: 7, }], - dense_snp_range: vec![(0, 1); n], // every query points at the lone entry + dense_snp_range: vec![0..1; n], // every query points at the lone entry dense_snp_present, dense_snp_present_off, dense_indel: vec![], - dense_indel_range: vec![(0, 0); n], // no indel window + dense_indel_range: vec![0..0; n], // no indel window dense_indel_present: vec![], dense_indel_present_off: vec![0; n + 1], + ..Default::default() }; // Must not panic. @@ -627,17 +630,18 @@ mod tests { dense_snp, // query 0 owns snp[0..2), query 1 owns snp[2..4) — width 2/query, // shared by both of that query's haps. - dense_snp_range: vec![(0, 2), (2, 4)], + dense_snp_range: vec![0..2, 2..4], // hap0 bits(0,1)=(1,0), hap1 bits(2,3)=(0,1), hap2 bits(4,5)=(1,1), // hap3 bits(6,7)=(0,0) -> byte 0b0011_1001. dense_snp_present: vec![0b0011_1001], dense_snp_present_off: vec![0, 2, 4, 6, 8], dense_indel, // query 0 owns indel[0..1), query 1 owns indel[1..2) — width 1/query. - dense_indel_range: vec![(0, 1), (1, 2)], + dense_indel_range: vec![0..1, 1..2], // hap0=1, hap1=0, hap2=1, hap3=1 -> byte 0b0000_1101. dense_indel_present: vec![0b0000_1101], dense_indel_present_off: vec![0, 1, 2, 3, 4], + ..Default::default() }; let flat = split_to_flat(&br); @@ -677,16 +681,17 @@ mod tests { position: 8, key: dense_snp_key, }], - dense_snp_range: vec![(0, 1)], + dense_snp_range: vec![0..1], dense_snp_present: vec![0b1], dense_snp_present_off: vec![0, 1], dense_indel: vec![KeyRef { position: 12, key: dense_indel_key, }], - dense_indel_range: vec![(0, 1)], + dense_indel_range: vec![0..1], dense_indel_present: vec![0b1], dense_indel_present_off: vec![0, 1], + ..Default::default() }; let soa = decode_variants_from_split(&br, &[], &[0]); @@ -734,7 +739,7 @@ mod tests { KeyRef { position: 51, key: k(b"A") }, KeyRef { position: 52, key: k(b"C") }, ], - dense_snp_range: vec![(0, 3), (3, 6)], + dense_snp_range: vec![0..3, 3..6], // Per-hap snp-bit widths 3,3,3,3 -> offsets 0,3,6,9,12. Bitstream // (idx0..11): 1,0,1, 0,1,0, 1,1,0, 0,0,1 -> byte0 = 0b1101_0101 // (bits0-7: 1,0,1,0,1,0,1,1 -> 1+4+16+64+128=213), byte1 low @@ -745,11 +750,12 @@ mod tests { KeyRef { position: 13, key: svar2_codec::encode_pure_del(-2) }, KeyRef { position: 14, key: svar2_codec::encode_pure_del(-5) }, ], - dense_indel_range: vec![(0, 2), (2, 2)], + dense_indel_range: vec![0..2, 2..2], // Per-hap indel-bit widths 2,2,0,0 -> offsets 0,2,4,4,4. // Bitstream (idx0..3): 1,0, 0,1 -> byte0 = 0b1001 (1+8=9). dense_indel_present: vec![9], dense_indel_present_off: vec![0, 2, 4, 4, 4], + ..Default::default() }; let soa = decode_variants_from_split(&br, &[], &[0]); From 80c4bf08497d7704a7864abba74b0ced9314cb07 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 01:11:02 -0700 Subject: [PATCH 077/108] feat(svar2): Svar2Store retains store_path for field-sidecar paths The field-read path must rebuild genoray's ContigPaths to locate a field's on-disk value sidecars, which needs the store base dir. Retain it and expose an accessor. Additive plumbing; no behavior change. --- src/svar2/store.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/svar2/store.rs b/src/svar2/store.rs index d284dc8e..6e8c35df 100644 --- a/src/svar2/store.rs +++ b/src/svar2/store.rs @@ -9,12 +9,16 @@ use pyo3::prelude::*; #[pyclass] pub struct Svar2Store { readers: HashMap, + store_path: String, } impl Svar2Store { pub fn reader(&self, contig: &str) -> Option<&ContigReader> { self.readers.get(contig) } + pub fn store_path(&self) -> &str { + &self.store_path + } } #[pymethods] @@ -32,7 +36,11 @@ impl Svar2Store { .map_err(|e| PyIOError::new_err(format!("open contig {c}: {e}")))?; readers.insert(c, r); } - Ok(Self { readers }) + let store_path = store_path.to_string(); + Ok(Self { + readers, + store_path, + }) } fn contigs(&self) -> Vec { From 6a765b626608a3d4314bf9195f12ab778db56cda Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 01:28:33 -0700 Subject: [PATCH 078/108] feat(svar2): field-byte gather with vk_src/dense provenance in split decode decode_variants_from_split now optionally tracks, for each merged variant, which source row it came from: var_key provenance is read from vk_src via unpack_vk_src (yielding the sub-stream and call index), dense provenance is derived arithmetically via dense_abs_row against the on-disk window. Each requested field's bytes are then gathered from genoray's FieldView sidecars into a byte buffer parallel to pos. Dense FORMAT values stride by the original cohort sample index (row * cohort_n_samples + orig_sample); var_key entries are already per-(variant, sample) calls, so they index by call index directly. The no-field path is guarded and does zero extra work (byte-identical). New test pins provenance identity against a store whose value at element i is i, so a decoded value reveals the attributed source row; the dense case uses a non-trivial on-disk window (3..4) so it fails if the output index is used verbatim instead of the absolute row. --- Cargo.lock | 1 + Cargo.toml | 1 + src/ffi/mod.rs | 4 +- src/svar2/mod.rs | 179 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 169 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ace21d61..990b1eef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -532,6 +532,7 @@ dependencies = [ "rstest", "seqpro-core", "svar2-codec", + "tempfile", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4a010fbd..98587c4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ features = ["abi3-py310"] [dev-dependencies] rstest = "0.26.1" +tempfile = "3" # Perf call-graph attribution only (`perf report --children`). Inherits release # codegen and adds line tables + frame pointers. NEVER the gate artifact — all diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index cc60ce29..2c8b0be2 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1368,7 +1368,9 @@ pub fn decode_variants_from_svar2_readbound<'py>( let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); - svar2::decode_variants_from_split(&br, &lut_bytes, &lut_off) + let (soa, _field_bufs) = + svar2::decode_variants_from_split(&br, &lut_bytes, &lut_off, &[], &[], &[], &[]); + soa }); Ok(( diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index ac0c878a..35a06ce1 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -5,7 +5,7 @@ use std::borrow::Cow; use std::ops::Range; -use genoray_core::query::BatchResultSplit; +use genoray_core::query::{dense_abs_row, unpack_vk_src, BatchResultSplit, FieldView}; use ndarray::{Array2, ArrayView2}; use svar2_codec::{decode_key, DecodedKey}; @@ -257,6 +257,17 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { } } +/// One INFO/FORMAT field's on-disk sidecars, opened for gather. `views` is +/// indexed by [`genoray_core::layout::FieldSub::all`] order (VkSnp, VkIndel, +/// DenseSnp, DenseIndel); `FieldView` owns its mmap, so this needs no lifetime. +pub struct FieldGather { + pub views: [FieldView; 4], + pub is_format: bool, + /// `dtype.width_bytes()`; consumed by the FFI caller (Task 1.3), not here. + pub width: usize, + pub cohort_n_samples: usize, +} + /// Per-hap decoded variant SoA, matching genoray's `decode_hap` output layout — /// see [`decode_variants_from_split`]. pub struct VariantsSoa { @@ -280,11 +291,16 @@ pub struct VariantsSoa { /// NO overlap/clip filter here — the gather already restricts to overlapping /// variants, unlike [`hap_diffs_svar2`]/reconstruct's ref_idx-consumed clipping, /// which only matters for sizing a fixed-length output buffer. +#[allow(clippy::too_many_arguments)] pub fn decode_variants_from_split( br: &BatchResultSplit, lut_bytes: &[u8], lut_off: &[i64], -) -> VariantsSoa { + fields: &[FieldGather], + on_disk_snp: &[Range], + on_disk_indel: &[Range], + orig_samples: &[usize], +) -> (VariantsSoa, Vec>) { // Fused decode straight from the split gather result: NO `split_to_flat` // marshaling copy, and NO per-hap `merge_hap` Vec+sort. The three per-hap // runs (var_key, present dense-snp, present dense-indel) are each already @@ -313,6 +329,7 @@ pub fn decode_variants_from_split( str_off.push(0); let mut var_off: Vec = Vec::with_capacity(h_count + 1); var_off.push(0); + let mut field_bufs: Vec> = (0..fields.len()).map(|_| Vec::new()).collect(); let mut h = 0usize; for q in 0..n_q { @@ -351,37 +368,80 @@ pub fn decode_variants_from_split( let p_vk = if has_vk { br.vk[i_vk].position } else { u32::MAX }; let p_sn = if has_sn { br.dense_snp[i_sn].position } else { u32::MAX }; let p_in = if has_in { br.dense_indel[i_in].position } else { u32::MAX }; - let (p, key) = if has_vk && p_vk <= p_sn && p_vk <= p_in { + let (p, key, chan, cidx) = if has_vk && p_vk <= p_sn && p_vk <= p_in { let e = &br.vk[i_vk]; + let out = (e.position, e.key, 0u8, i_vk); i_vk += 1; - (e.position, e.key) + out } else if has_sn && p_sn <= p_in { let e = &br.dense_snp[i_sn]; + let out = (e.position, e.key, 1u8, i_sn); i_sn += 1; - (e.position, e.key) + out } else { let e = &br.dense_indel[i_in]; + let out = (e.position, e.key, 2u8, i_in); i_in += 1; - (e.position, e.key) + out }; let (il, alt) = decode_alt(key, lut_bytes, lut_off); pos.push(p as i32); ilen.push(il as i32); alt_bytes.extend_from_slice(&alt); str_off.push(alt_bytes.len() as i64); + + if !fields.is_empty() { + debug_assert!( + !br.vk_src.is_empty(), + "fields requested but `br` was produced without provenance \ + (use gather_haps_readbound_src, not gather_haps_readbound)" + ); + // Resolve (sub_ix, row) for this emitted variant. + let (sub_ix, row, is_dense) = match chan { + 0 => { + let (is_indel, call_idx) = unpack_vk_src(br.vk_src[cidx]); + (if is_indel { 1 } else { 0 }, call_idx, false) + } + 1 => ( + 2, + dense_abs_row(&on_disk_snp[q], &br.dense_snp_range[q], cidx), + true, + ), + _ => ( + 3, + dense_abs_row(&on_disk_indel[q], &br.dense_indel_range[q], cidx), + true, + ), + }; + for (fi, f) in fields.iter().enumerate() { + // var_key entries are already per-(variant, sample) CALLS, so a + // FORMAT value is indexed by the call index directly. Only the + // DENSE channel, which is variant-major over the whole cohort, + // needs the sample stride. + let elem = if is_dense && f.is_format { + row * f.cohort_n_samples + orig_samples[q] + } else { + row + }; + field_bufs[fi].extend_from_slice(f.views[sub_ix].bytes_at(elem)); + } + } } var_off.push(pos.len() as i64); h += 1; } } - VariantsSoa { - pos, - ilen, - alt_bytes, - str_off, - var_off, - } + ( + VariantsSoa { + pos, + ilen, + alt_bytes, + str_off, + var_off, + }, + field_bufs, + ) } #[cfg(test)] @@ -694,7 +754,7 @@ mod tests { ..Default::default() }; - let soa = decode_variants_from_split(&br, &[], &[0]); + let (soa, _bufs) = decode_variants_from_split(&br, &[], &[0], &[], &[], &[], &[]); // Position-sorted: var_key SNP@5, dense/snp@8, dense/indel@12. assert_eq!(soa.var_off, vec![0, 3]); @@ -758,7 +818,7 @@ mod tests { ..Default::default() }; - let soa = decode_variants_from_split(&br, &[], &[0]); + let (soa, _bufs) = decode_variants_from_split(&br, &[], &[0], &[], &[], &[], &[]); // hap0 (q0): snp present [1,0,1] -> keeps pos10("A"),pos12("G"); // indel present [1,0] -> keeps pos13(ilen -2). @@ -772,4 +832,93 @@ mod tests { assert_eq!(soa.str_off, vec![0, 1, 2, 2, 3, 3, 4, 5, 6]); assert_eq!(soa.var_off, vec![0, 3, 5, 7, 8]); } + + /// Build 4 FieldViews over an identity i32 store: element i has value i. + /// Returns the TempDir too — it must outlive the views (dropping it deletes the mmapped files). + fn make_identity_i32_fields() -> (tempfile::TempDir, Vec) { + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + + let tmp = tempfile::tempdir().unwrap(); + let base = tmp.path().to_str().unwrap(); + let paths = ContigPaths::new(base, "chr1"); + + const N: usize = 8; // enough for call idx 5 and dense row 3 + let bytes: Vec = (0..N as i32).flat_map(|i| i.to_le_bytes()).collect(); + + let mut views = Vec::with_capacity(4); + for sub in FieldSub::all() { + let p = paths.field_values("info", "X", sub); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, &bytes).unwrap(); + views.push(FieldView::open(&paths, "info", "X", sub, StorageDtype::I32, 1).unwrap()); + } + let views: [FieldView; 4] = views.try_into().map_err(|_| ()).unwrap(); + + ( + tmp, + vec![FieldGather { + views, + is_format: false, // INFO -> element index is the row itself + width: 4, + cohort_n_samples: 1, + }], + ) + } + + #[test] + fn test_decode_fields_provenance_identity() { + use genoray_core::query::{pack_vk_src, KeyRef}; + + // One query, ploidy 1. + // var_key entry at pos 10, provenance = (snp, call idx 5) + // dense-snp entry at pos 20, output window 0..1, ON-DISK window 3..4 -> abs row 3 + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 10, + key: svar2_codec::encode_pure_del(-1), + }], + vk_off: vec![0, 1], + vk_src: vec![pack_vk_src(false, 5)], + dense_snp: vec![KeyRef { + position: 20, + key: svar2_codec::encode_pure_del(-1), + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![], + dense_indel_range: vec![0..0], + dense_indel_present: vec![], + dense_indel_present_off: vec![0, 0], + ..Default::default() + }; + + let (_tmp, fields) = make_identity_i32_fields(); // keep _tmp alive: it owns the tempdir + + let (soa, bufs) = decode_variants_from_split( + &br, + &[], + &[0i64], + &fields, + &[3..4], // on_disk_snp: the dense-snp window really lives at rows 3..4 on disk + &[0..0], // on_disk_indel + &[0], // orig_samples + ); + + // var_key (pos 10) sorts before dense (pos 20). + assert_eq!(soa.pos, vec![10, 20]); + + // Identity store => value == source row. + // var_key -> sub VkSnp, call idx 5 -> 5 + // dense-snp-> sub DenseSnp, abs row 3 -> 3 (proves the on-disk offset is applied) + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(vals, vec![5, 3]); + } } From 8ab366fdeeea532e1d4b64a844426f21c1e4734b Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 01:49:14 -0700 Subject: [PATCH 079/108] test(svar2): make field-provenance tests catch sub-stream and stride bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provenance test was vacuous w.r.t. sub-stream mapping: all four sub-streams held identical identity bytes, so views[sub_ix] was unobservable and swapping DenseSnp for DenseIndel still passed. Encode value = 100*sub_ix + row so a decoded value reveals BOTH the sub-stream and the row, and exercise all four channels (VkSnp, VkIndel, DenseSnp, DenseIndel) — VkIndel and DenseIndel previously had no coverage at all. Add a test pinning the dense FORMAT stride (row*cohort_n_samples + orig_sample) and asserting var_key FORMAT values are unstrided, which is what pins the is_dense && is_format gate. Verified these now fail under mutation: swapped dense sub_ix, var_key sub_ix hardcoded to 0, dropped cohort stride, and an off-by-one merge cursor all break the suite (each previously passed). Also hoist the vk_src contract check out of the hot loop into one unconditional assert (matching genoray's own contract pattern), document the FieldGather invariants, and pre-reserve the field buffers. --- src/svar2/mod.rs | 187 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 162 insertions(+), 25 deletions(-) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 35a06ce1..98239b7d 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -257,9 +257,23 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { } } -/// One INFO/FORMAT field's on-disk sidecars, opened for gather. `views` is -/// indexed by [`genoray_core::layout::FieldSub::all`] order (VkSnp, VkIndel, -/// DenseSnp, DenseIndel); `FieldView` owns its mmap, so this needs no lifetime. +/// One INFO/FORMAT field's on-disk sidecars, opened for gather. +/// +/// Contracts the caller MUST uphold when constructing this: +/// * `views` must be opened in [`genoray_core::layout::FieldSub::all`] order +/// (VkSnp, VkIndel, DenseSnp, DenseIndel) — `decode_variants_from_split` +/// indexes into it positionally, not by name. `FieldView` owns its mmap, +/// so this needs no lifetime. +/// * `cohort_n_samples` MUST equal the `n_samples` passed to +/// `FieldView::open` for every view in `views`. If they diverge, the dense +/// FORMAT stride computed here (`row * cohort_n_samples + orig_sample`) +/// silently disagrees with how the on-disk store was laid out — no panic, +/// just wrong values. +/// * `FieldView::bytes_at` panics on an empty sub-stream. This is only safe +/// because an empty sub-stream implies no emitted record ever resolves to +/// it (e.g. a store with zero indel calls never routes a variant to +/// `VkIndel`/`DenseIndel`). A future constructor that opens a placeholder +/// view for a sub-stream that CAN be referenced must not rely on this. pub struct FieldGather { pub views: [FieldView; 4], pub is_format: bool, @@ -329,7 +343,26 @@ pub fn decode_variants_from_split( str_off.push(0); let mut var_off: Vec = Vec::with_capacity(h_count + 1); var_off.push(0); - let mut field_bufs: Vec> = (0..fields.len()).map(|_| Vec::new()).collect(); + let mut field_bufs: Vec> = fields + .iter() + .map(|f| Vec::with_capacity(cap * f.width)) + .collect(); + + // A `BatchResultSplit` without provenance (`gather_haps_readbound`, not + // `_src`) leaves `vk_src` empty; indexing it below would panic with an + // opaque out-of-bounds error instead of naming the real cause. Check once, + // O(1), instead of a per-variant `debug_assert` in the hot loop — this is + // an unconditional `assert_eq!` (matches genoray's own contract check in + // `BatchResultSplit`) so it fires in release builds too, before any wrong + // provenance can attach a field value to the wrong variant. + if !fields.is_empty() { + assert_eq!( + br.vk_src.len(), + br.vk.len(), + "fields require a BatchResultSplit from gather_haps_readbound_src \ + (vk_src must be populated 1:1 with vk)" + ); + } let mut h = 0usize; for q in 0..n_q { @@ -391,11 +424,6 @@ pub fn decode_variants_from_split( str_off.push(alt_bytes.len() as i64); if !fields.is_empty() { - debug_assert!( - !br.vk_src.is_empty(), - "fields requested but `br` was produced without provenance \ - (use gather_haps_readbound_src, not gather_haps_readbound)" - ); // Resolve (sub_ix, row) for this emitted variant. let (sub_ix, row, is_dense) = match chan { 0 => { @@ -833,7 +861,14 @@ mod tests { assert_eq!(soa.var_off, vec![0, 3, 5, 7, 8]); } - /// Build 4 FieldViews over an identity i32 store: element i has value i. + /// Build 4 FieldViews over a store where each sub-stream's content is + /// distinguishable: element `i` of sub-stream `sub_ix` (in + /// `FieldSub::all()` order, i.e. `[VkSnp=0, VkIndel=1, DenseSnp=2, + /// DenseIndel=3]`) has value `100 * sub_ix + i`. A test that decodes value + /// `V` therefore reveals BOTH which sub-stream (`V / 100`) and which row + /// (`V % 100`) the decoder attributed to that variant — a same-content + /// store (as an earlier version of this fixture used) cannot distinguish + /// "read the right sub-stream" from "read some sub-stream". /// Returns the TempDir too — it must outlive the views (dropping it deletes the mmapped files). fn make_identity_i32_fields() -> (tempfile::TempDir, Vec) { use genoray_core::field::StorageDtype; @@ -843,11 +878,13 @@ mod tests { let base = tmp.path().to_str().unwrap(); let paths = ContigPaths::new(base, "chr1"); - const N: usize = 8; // enough for call idx 5 and dense row 3 - let bytes: Vec = (0..N as i32).flat_map(|i| i.to_le_bytes()).collect(); + const N: usize = 8; // enough for call idx 5 and dense row 3/4 let mut views = Vec::with_capacity(4); - for sub in FieldSub::all() { + for (sub_ix, sub) in FieldSub::all().into_iter().enumerate() { + let bytes: Vec = (0..N as i32) + .flat_map(|i| (100 * sub_ix as i32 + i).to_le_bytes()) + .collect(); let p = paths.field_values("info", "X", sub); std::fs::create_dir_all(p.parent().unwrap()).unwrap(); std::fs::write(&p, &bytes).unwrap(); @@ -870,9 +907,108 @@ mod tests { fn test_decode_fields_provenance_identity() { use genoray_core::query::{pack_vk_src, KeyRef}; - // One query, ploidy 1. - // var_key entry at pos 10, provenance = (snp, call idx 5) - // dense-snp entry at pos 20, output window 0..1, ON-DISK window 3..4 -> abs row 3 + // One query, ploidy 1, all four provenance channels represented: + // var_key SNP entry at pos 10, provenance = (snp, call idx 5) -> sub VkSnp=0 + // var_key indel entry at pos 15, provenance = (indel, call idx 2) -> sub VkIndel=1 + // dense-snp entry at pos 20, output window 0..1, ON-DISK window 3..4 -> abs row 3, sub DenseSnp=2 + // dense-indel entry at pos 25, output window 0..1, ON-DISK window 4..5 -> abs row 4, sub DenseIndel=3 + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![ + KeyRef { + position: 10, + key: svar2_codec::encode_pure_del(-1), + }, + KeyRef { + position: 15, + key: svar2_codec::encode_pure_del(-1), + }, + ], + vk_off: vec![0, 2], + vk_src: vec![pack_vk_src(false, 5), pack_vk_src(true, 2)], + dense_snp: vec![KeyRef { + position: 20, + key: svar2_codec::encode_pure_del(-1), + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 25, + key: svar2_codec::encode_pure_del(-1), + }], + dense_indel_range: vec![0..1], + dense_indel_present: vec![0b1], + dense_indel_present_off: vec![0, 1], + ..Default::default() + }; + + let (_tmp, fields) = make_identity_i32_fields(); // keep _tmp alive: it owns the tempdir + + let (soa, bufs) = decode_variants_from_split( + &br, + &[], + &[0i64], + &fields, + &[3..4], // on_disk_snp: the dense-snp window really lives at rows 3..4 on disk + &[4..5], // on_disk_indel: the dense-indel window really lives at rows 4..5 on disk + &[0], // orig_samples + ); + + // Position-sorted: var_key(10,15) before dense(20,25); snp before indel on ties (none here). + assert_eq!(soa.pos, vec![10, 15, 20, 25]); + + // value = 100*sub_ix + row, so each decoded value pins BOTH the + // sub-stream routing and the row/offset arithmetic for its channel: + // var_key snp -> sub 0, call idx 5 -> 100*0+5 = 5 + // var_key indel -> sub 1, call idx 2 -> 100*1+2 = 102 + // dense-snp -> sub 2, abs row 3 -> 100*2+3 = 203 + // dense-indel -> sub 3, abs row 4 -> 100*3+4 = 304 + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(vals, vec![5, 102, 203, 304]); + } + + #[test] + fn test_decode_fields_format_dense_stride_and_var_key_unstrided() { + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + use genoray_core::query::{pack_vk_src, KeyRef}; + + // A FORMAT field over a 3-sample cohort. Store is plain identity + // (element i has value i) across all four subs — this test isolates + // the *stride* arithmetic, not sub-stream routing (that's pinned by + // `test_decode_fields_provenance_identity`). + let tmp = tempfile::tempdir().unwrap(); + let base = tmp.path().to_str().unwrap(); + let paths = ContigPaths::new(base, "chr1"); + + const N: usize = 16; // covers dense elem 3*3+2=11 and var_key call idx 5 + let bytes: Vec = (0..N as i32).flat_map(|i| i.to_le_bytes()).collect(); + + let mut views = Vec::with_capacity(4); + for sub in FieldSub::all() { + let p = paths.field_values("format", "DP", sub); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, &bytes).unwrap(); + // n_samples passed to `open` MUST match `cohort_n_samples` below. + views.push(FieldView::open(&paths, "format", "DP", sub, StorageDtype::I32, 3).unwrap()); + } + let views: [FieldView; 4] = views.try_into().map_err(|_| ()).unwrap(); + + let fields = vec![FieldGather { + views, + is_format: true, + width: 4, + cohort_n_samples: 3, + }]; + + // var_key SNP entry at pos 10, call idx 5; dense-snp entry at pos 20, + // output window 0..1, ON-DISK window 3..4 -> abs row 3. let br = BatchResultSplit { n_regions: 1, n_samples: 1, @@ -897,28 +1033,29 @@ mod tests { ..Default::default() }; - let (_tmp, fields) = make_identity_i32_fields(); // keep _tmp alive: it owns the tempdir - let (soa, bufs) = decode_variants_from_split( &br, &[], &[0i64], &fields, - &[3..4], // on_disk_snp: the dense-snp window really lives at rows 3..4 on disk + &[3..4], // on_disk_snp &[0..0], // on_disk_indel - &[0], // orig_samples + &[2], // orig_samples: this query's original cohort sample index is 2 ); - // var_key (pos 10) sorts before dense (pos 20). assert_eq!(soa.pos, vec![10, 20]); - // Identity store => value == source row. - // var_key -> sub VkSnp, call idx 5 -> 5 - // dense-snp-> sub DenseSnp, abs row 3 -> 3 (proves the on-disk offset is applied) let vals: Vec = bufs[0] .chunks_exact(4) .map(|c| i32::from_le_bytes(c.try_into().unwrap())) .collect(); - assert_eq!(vals, vec![5, 3]); + // var_key FORMAT is UNSTRIDED: a var_key entry is already a per-CALL + // value, so the element index is the call index directly (5), NOT + // `call_idx * cohort_n_samples + orig_sample` (5*3+2=17). This pins + // the `is_dense && f.is_format` gate — the part most likely to + // silently regress. + // dense FORMAT IS strided: abs row 3, orig_sample 2, cohort_n_samples 3 + // -> element 3*3+2 = 11. + assert_eq!(vals, vec![5, 11]); } } From 55ab671a4e411406a3301f5d70e67d8875c19c8c Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 02:26:18 -0700 Subject: [PATCH 080/108] feat(svar2): decode_variants_from_svar2_readbound returns INFO/FORMAT field buffers The FFI gains a trailing fields=[(category, name, dtype_str)] parameter. When non-empty it opens one FieldView per FieldSub (iterating FieldSub::all() so the views land in the order FieldGather indexes them by), selects gather_haps_readbound_src (the only gather that populates the var_key provenance vk_src), and returns one u8 buffer per field plus the itemsizes for Python to .view(dtype) with. The on-disk dense windows from HapRanges are passed through so dense provenance resolves to absolute rows, and orig_samples (the original cohort indices given to HapRanges) carries the dense FORMAT stride. When fields is empty the old gather is used and no field work runs. Existing call sites are updated mechanically for the new arity; Python field wiring is the next phase. --- python/genvarloader/_dataset/_svar2_haps.py | 6 +- .../genvarloader/_dataset/_svar2_store_py.py | 23 +++-- src/ffi/mod.rs | 91 +++++++++++++++++-- 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 99e318d1..df356cf4 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -575,7 +575,7 @@ def _reconstruct_variants( cat_query_order: list[NDArray[np.intp]] = [] for ci, qsel in groups: gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) - pos, ilen, alt_bytes, str_off, var_off = ( + pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( decode_variants_from_svar2_readbound( self.store, self.ds_contigs[ci], @@ -586,6 +586,7 @@ def _reconstruct_variants( gi[4], gi[5], P, + [], ) ) var_off = np.asarray(var_off, np.int64) @@ -687,7 +688,7 @@ def _reconstruct_variant_windows( for ci, qsel in groups: gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) - pos, ilen, alt_bytes, str_off, var_off = ( + pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( decode_variants_from_svar2_readbound( self.store, self.ds_contigs[ci], @@ -698,6 +699,7 @@ def _reconstruct_variant_windows( gi[4], gi[5], P, + [], ) ) pos = np.asarray(pos, np.int32) diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/python/genvarloader/_dataset/_svar2_store_py.py index 7ed2aaa7..71c4bf84 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/python/genvarloader/_dataset/_svar2_store_py.py @@ -327,16 +327,19 @@ def build_readbound_variants( store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) - pos, ilen, alt_bytes, str_off, var_off = decode_variants_from_svar2_readbound( - store, - contig, - region_starts, - orig_samples, - vk_snp_range, - vk_indel_range, - dense_snp_range, - dense_indel_range, - P, + pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( + decode_variants_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + P, + [], + ) ) from seqpro.rag import Ragged diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 2c8b0be2..7ce40226 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1313,8 +1313,19 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( /// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range` argument semantics /// (the per-query outputs of `SparseVar2.find_ranges`, flattened region-major, /// sample-minor). `ploidy` is passed explicitly (there is no `shifts` array to -/// infer it from here). Returns the `RaggedVariants` SoA: `(pos, ilen, alt_bytes, -/// str_off, var_off)`; see +/// infer it from here). +/// +/// `fields` is a list of `(category, name, dtype_str)` triples (`category` is +/// `"info"` or `"format"`; `dtype_str` is genoray's `StorageDtype` meta string, +/// e.g. `"i32"`), and may be empty. When non-empty, this opens the four +/// `FieldSub`-keyed `FieldView`s per field and gathers their bytes alongside the +/// variant decode via the var_key-provenance-tracking +/// `genoray_core::query::gather_haps_readbound_src` (plain `gather_haps_readbound` +/// is used when `fields` is empty, since it doesn't need that provenance). +/// +/// Returns the `RaggedVariants` SoA `(pos, ilen, alt_bytes, str_off, var_off)` +/// plus, per requested field in `fields` order, a flat `u8` byte buffer and its +/// per-value itemsize (`field_bufs`, `field_itemsizes`); see /// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_variants`. #[pyfunction] #[allow(clippy::too_many_arguments)] @@ -1329,14 +1340,20 @@ pub fn decode_variants_from_svar2_readbound<'py>( dense_snp_range: PyReadonlyArray2, dense_indel_range: PyReadonlyArray2, ploidy: usize, + fields: Vec<(String, String, String)>, ) -> PyResult<( Bound<'py, PyArray1>, Bound<'py, PyArray1>, Bound<'py, PyArray1>, Bound<'py, PyArray1>, Bound<'py, PyArray1>, + Vec>>, + Vec, )> { use crate::svar2; + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + use genoray_core::query::FieldView; let reader = store.reader(contig).ok_or_else(|| { pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) @@ -1353,7 +1370,47 @@ pub fn decode_variants_from_svar2_readbound<'py>( let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); - let soa = py.detach(move || { + let n_samples = reader.n_samples(); + let paths = ContigPaths::new(store.store_path(), contig); + + let gathers: Vec = fields + .iter() + .map(|(cat, name, dtype_str)| { + let dtype = StorageDtype::from_meta_str(dtype_str).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!( + "field {name}: unknown storage dtype {dtype_str:?}" + )) + })?; + let width = dtype.width_bytes().ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!( + "field {name}: unresolved dtype" + )) + })?; + // views MUST be in FieldSub::all() order — FieldGather indexes them by sub_ix. + let mut views = Vec::with_capacity(4); + for sub in FieldSub::all() { + views.push( + FieldView::open(&paths, cat, name, sub, dtype, n_samples).map_err(|e| { + pyo3::exceptions::PyIOError::new_err(format!("open field {name}: {e}")) + })?, + ); + } + let views: [FieldView; 4] = views + .try_into() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("expected 4 field views"))?; + Ok(svar2::FieldGather { + views, + is_format: cat == "format", + width, + cohort_n_samples: n_samples, + }) + }) + .collect::>>()?; + + let itemsizes: Vec = gathers.iter().map(|g| g.width).collect(); + let has_fields = !gathers.is_empty(); + + let (soa, field_bufs) = py.detach(move || { let rb = genoray_core::query::HapRanges::new( ®ion_starts_v, &orig_samples_v, @@ -1363,22 +1420,42 @@ pub fn decode_variants_from_svar2_readbound<'py>( &dense_indel_range_v, ploidy, ); - let br = genoray_core::query::gather_haps_readbound(reader, &rb); + // Field gather needs var_key provenance (vk_src), which ONLY the _src + // variant populates. + let br = if has_fields { + genoray_core::query::gather_haps_readbound_src(reader, &rb) + } else { + genoray_core::query::gather_haps_readbound(reader, &rb) + }; let (lut_bytes, lut_off_u64) = reader.lut_arrays(); let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); - let (soa, _field_bufs) = - svar2::decode_variants_from_split(&br, &lut_bytes, &lut_off, &[], &[], &[], &[]); - soa + svar2::decode_variants_from_split( + &br, + &lut_bytes, + &lut_off, + &gathers, + // ON-DISK dense windows (from find_ranges / HapRanges), NOT br's output windows. + &dense_snp_range_v, + &dense_indel_range_v, + &orig_samples_v, + ) }); + let field_out: Vec>> = field_bufs + .into_iter() + .map(|b| Array1::from_vec(b).into_pyarray(py)) + .collect(); + Ok(( Array1::from_vec(soa.pos).into_pyarray(py), Array1::from_vec(soa.ilen).into_pyarray(py), Array1::from_vec(soa.alt_bytes).into_pyarray(py), Array1::from_vec(soa.str_off).into_pyarray(py), Array1::from_vec(soa.var_off).into_pyarray(py), + field_out, + itemsizes, )) } From 70dadbe11d7a6b6351c38f75e804501886d11739 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 03:24:10 -0700 Subject: [PATCH 081/108] feat(svar2): discover store INFO/FORMAT fields and skip SVAR1 lazy-loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Svar2Haps reads the store's field manifest (SparseVar2.available_fields) into store_fields and advertises the keys in available_var_fields, so users can request them via var_fields. The manifest entries are kept as genoray StoredField objects rather than flattened to a tuple, because the stored default is needed for empty-group fills later. Dataset.with_seqs(var_fields=) gains an early Svar2Haps branch: SVAR2 field values are read on demand by the decode kernel, so there is no SVAR1 variants table to lazily load INFO/dosage/custom-FORMAT columns from (Svar2Haps.variants is a dummy placeholder). The shared 'missing' validation stays at the top — it is what stops an unknown field name reaching the Rust FFI. The SVAR1 body is byte-identical (re-indented only): git diff -w reports 13 insertions and 0 deletions. SVAR1 var_fields coverage: 71 passed. --- python/genvarloader/_dataset/_impl.py | 69 ++++++++++++--------- python/genvarloader/_dataset/_svar2_haps.py | 27 +++++++- 2 files changed, 67 insertions(+), 29 deletions(-) diff --git a/python/genvarloader/_dataset/_impl.py b/python/genvarloader/_dataset/_impl.py index 862e51a9..bedbdd57 100644 --- a/python/genvarloader/_dataset/_impl.py +++ b/python/genvarloader/_dataset/_impl.py @@ -339,34 +339,47 @@ def with_settings( missing = list(set(var_fields) - set(self.available_var_fields)) if missing or not isinstance(self._seqs, Haps): raise ValueError(f"Missing variant fields: {missing}") - haps = to_evolve.get("_seqs", self._seqs) - # Discover custom FORMAT fields so we don't try to load them as INFO. - custom_fmt = _svar_format_fields(haps.variants.path.parent) - # Lazily load any newly-requested info columns into the existing - # _Variants struct (mutates haps.variants.info in place). - builtin = {"alt", "ilen", "start", "ref", "dosage"} - new_info_fields = [ - f - for f in var_fields - if f not in builtin - and f not in haps.variants.info - and f not in custom_fmt - ] - if new_info_fields: - haps.variants.load_info(new_info_fields) - # Lazily memmap dosages if newly requested. - if "dosage" in var_fields and haps.dosages is None: - haps = _lazy_load_dosages(self, haps) - # Lazily memmap custom FORMAT fields if newly requested. - new_custom_fields = { - f: custom_fmt[f] - for f in var_fields - if f in custom_fmt and f not in haps.var_field_data - } - if new_custom_fields: - haps = _lazy_load_custom_fields(self, haps, new_custom_fields) - haps = replace(haps, var_fields=var_fields) - to_evolve["_seqs"] = haps + + from ._svar2_haps import Svar2Haps + + if isinstance(self._seqs, Svar2Haps): + # SVAR2 field values are read on demand by the decode kernel + # (decode_variants_from_svar2_readbound); there is no SVAR1 variants + # table to lazily load INFO/dosage/custom-FORMAT columns from — this + # reconstructor's `variants` is a dummy placeholder. + haps = replace( + to_evolve.get("_seqs", self._seqs), var_fields=var_fields + ) + to_evolve["_seqs"] = haps + else: + haps = to_evolve.get("_seqs", self._seqs) + # Discover custom FORMAT fields so we don't try to load them as INFO. + custom_fmt = _svar_format_fields(haps.variants.path.parent) + # Lazily load any newly-requested info columns into the existing + # _Variants struct (mutates haps.variants.info in place). + builtin = {"alt", "ilen", "start", "ref", "dosage"} + new_info_fields = [ + f + for f in var_fields + if f not in builtin + and f not in haps.variants.info + and f not in custom_fmt + ] + if new_info_fields: + haps.variants.load_info(new_info_fields) + # Lazily memmap dosages if newly requested. + if "dosage" in var_fields and haps.dosages is None: + haps = _lazy_load_dosages(self, haps) + # Lazily memmap custom FORMAT fields if newly requested. + new_custom_fields = { + f: custom_fmt[f] + for f in var_fields + if f in custom_fmt and f not in haps.var_field_data + } + if new_custom_fields: + haps = _lazy_load_custom_fields(self, haps, new_custom_fields) + haps = replace(haps, var_fields=var_fields) + to_evolve["_seqs"] = haps if splice_info is not None: if splice_info is False: diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index df356cf4..61e5d175 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -59,9 +59,24 @@ from ._svar2_link import Svar2Link, _resolve_svar2, _verify_svar2_fingerprint if TYPE_CHECKING: + from genoray._svar2_fields import StoredField + from ._splice import SplicePlan +_BUILTIN_VAR_FIELDS: frozenset[str] = frozenset( + {"alt", "ilen", "start", "ref", "dosage"} +) +"""Variant-field keys the reconstructors handle natively (never store fields).""" + + +def _field_spec(sf: "StoredField") -> tuple[str, str, str]: + """(category, name, dtype_str) as the Rust FFI expects it.""" + from genoray._svar2_fields import _META_DTYPE + + return (sf.category, sf.name, _META_DTYPE[sf.dtype]) + + @dataclass(slots=True) class _Svar2Cache: """The six memmapped ``svar2_ranges/`` arrays (all int64), sliced per query. @@ -165,13 +180,21 @@ class Svar2Haps(Haps[_H]): were computed over a max_jitter-padded window, which over-includes variants past the (unpadded) read window in variants mode (the decode kernel has no right-clip); guarded below.""" + store_fields: dict[str, "StoredField"] = field(default_factory=dict) + """The .svar2 store's INFO/FORMAT field manifest, keyed by field key. + + Populated from ``SparseVar2.available_fields``. These keys are additionally + advertised in ``available_var_fields`` so users can request them via ``var_fields``. + """ def __post_init__(self): # Deliberately does NOT call Haps.__post_init__ (that reads an SVAR1 # variants table / AF cache which svar2 has no analogue for). Set only # the init=False fields the base machinery reads. self.n_variants = self.genotypes.lengths - self.available_var_fields = ["alt", "ilen", "start"] + self.available_var_fields = ["alt", "ilen", "start"] + [ + k for k in self.store_fields if k not in _BUILTIN_VAR_FIELDS + ] # ---- construction ---- @@ -227,6 +250,7 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: sv = SparseVar2(str(svar2_path)) store = Svar2Store(str(svar2_path), sv.contigs, sv.n_samples, sv.ploidy) + store_fields = dict(sv.available_fields) # Minimal base-Haps fields. genotypes carries only the (R, S, P, None) # shape (so ploidy = shape[-2] and n_variants.shape are available); its @@ -263,6 +287,7 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: store_contigs=list(sv.contigs), ds_contigs=list(contigs), max_jitter=max_jitter, + store_fields=store_fields, ) # ---- reconstructor entry ---- From eedbe04f9462b5f34ed3afca6aca8276806c5b3b Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 03:50:49 -0700 Subject: [PATCH 082/108] test(svar2): oracle test for INFO/FORMAT field routing (var_key+dense, multi-contig, union) Gates the field-routing wiring. Fixture spans two contigs and carries INFO AF (Float, with a missing value), INFO NS (Integer), and FORMAT DP (Integer), with distinct values per variant and per sample so a wrong variant/value or sample association is detectable rather than coincidentally correct. A self-assert via find_ranges pins that the fixture really populates BOTH the var_key and dense channels, so neither provenance path is silently untested. Expected values are derived from the source VCF with cyvcf2 (an oracle, not a snapshot of our own output). Currently RED (5 failed, 1 passed): discovery and var_fields acceptance work; only the field values are absent from the output. That is the Task 2.3/2.4 gap. --- tests/dataset/test_svar2_fields_read.py | 496 ++++++++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 tests/dataset/test_svar2_fields_read.py diff --git a/tests/dataset/test_svar2_fields_read.py b/tests/dataset/test_svar2_fields_read.py new file mode 100644 index 00000000..ca4c0bac --- /dev/null +++ b/tests/dataset/test_svar2_fields_read.py @@ -0,0 +1,496 @@ +"""Integration oracle test for SVAR2 INFO/FORMAT field routing (Task 3.1). + +Gates the remaining wiring (Tasks 2.3/2.4) that routes scalar-numeric INFO/FORMAT +field values -- discovered and accepted already (Tasks 2.1/2.2) -- into gvl's +``RaggedVariants`` / variant-windows outputs. Written BEFORE that wiring lands, +so it is EXPECTED TO FAIL (RED) right now; it must fail because the field +VALUES are missing from the output, not because the fixture/store/API is broken. + +Oracle: the source VCF, parsed independently with ``cyvcf2`` (never hardcoded +from this repo's own decode output). Coordinate convention: VCF ``POS`` is +1-based; genoray/gvl positions are 0-based, so every oracle key uses +``POS - 1``. + +Fixture routing (self-asserted below via ``SparseVar2._find_ranges``, not +assumed): + - chr1:3 (0-based 2), A>G -- carried by exactly ONE haplotype (S0/hap0) + out of 6 in the cohort -> cost model routes this to the VAR_KEY channel. + - chr1:10 (0-based 9), G>C -- carried by ALL 6 haplotypes (hom in every + sample) -> cost model routes this to the DENSE channel. + - chr2:5 (0-based 4), A>T -- carried by exactly ONE haplotype (S1/hap0) + -> VAR_KEY channel, on the second contig. + +INFO ``AF`` (Float) is deliberately omitted from the chr1:10 record's INFO to +pin the missing-value fill (NaN, per genoray's ``StoredField`` semantics for a +field declared with no explicit ``default``). ``NS`` (Integer) and FORMAT +``DP`` (Integer) are always present. AF/NS are distinct across every variant; +DP is distinct across every sample within a variant -- both are deliberate +(repeated values would make a broken variant<->value or sample<->value +association silently pass). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import polars as pl +import pytest + +# --- fixture: 2 contigs, 3 samples, ploidy 2 (6 haplotypes/contig) ---------- + +_REF1 = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" # chr1, 40bp; idx2='A', idx9='G' +_REF2 = "ACGT" * 7 + "AC" # chr2, 30bp; idx4='A' +assert len(_REF1) == 40 and _REF1[2] == "A" and _REF1[9] == "G" +assert len(_REF2) == 30 and _REF2[4] == "A" + +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##contig= +##INFO= +##INFO= +##FORMAT= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1\tS2 +chr1\t3\t.\tA\tG\t.\t.\tAF=0.1;NS=5\tGT:DP\t1|0:10\t0|0:20\t0|0:30 +chr1\t10\t.\tG\tC\t.\t.\tNS=6\tGT:DP\t1|1:11\t1|1:21\t1|1:31 +chr2\t5\t.\tA\tT\t.\t.\tAF=0.42;NS=2\tGT:DP\t0|0:12\t1|0:22\t0|0:32 +""" + + +@pytest.fixture(scope="module") +def _src(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_fields_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF1}\n>chr2\n{_REF2}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar2_fields_store(_src, tmp_path_factory) -> Path: + from genoray import SparseVar2 + from genoray._svar2_fields import FormatField, InfoField + + bcf, ref = _src + out = tmp_path_factory.mktemp("svar2_fields") / "store.svar2" + SparseVar2.from_vcf( + out=out, + source=bcf, + reference=ref, + info_fields=[InfoField("AF"), InfoField("NS")], + format_fields=[FormatField("DP")], + overwrite=True, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +def _build_oracle(bcf_path: Path) -> tuple[dict[tuple[str, int], dict], list[str]]: + """Parse the source VCF with cyvcf2 (the oracle) into + ``{(contig, pos0): {"AF": float | None, "NS": int | None, + "carriers": {(sample_idx, hap_idx): dp_int}}}``. + + ``carriers`` is derived from cyvcf2's own decoded genotypes (allele == 1), + not hardcoded -- it is the ground truth for which (sample, haplotype) + pairs a read-bound decode kernel must emit a variant record for. + """ + from cyvcf2 import VCF as _CyVCF + + vcf = _CyVCF(str(bcf_path)) + try: + samples = list(vcf.samples) + oracle: dict[tuple[str, int], dict] = {} + for rec in vcf: + contig = rec.CHROM + pos0 = rec.POS - 1 # VCF POS is 1-based; genoray/gvl positions are 0-based. + af = rec.INFO.get("AF") + ns = rec.INFO.get("NS") + dp = np.asarray(rec.format("DP")).reshape(-1) + carriers: dict[tuple[int, int], int] = {} + for s_i, gt in enumerate(rec.genotypes): + for p_i, allele in enumerate(gt[:2]): + if allele == 1: + carriers[(s_i, p_i)] = int(dp[s_i]) + oracle[(contig, pos0)] = { + "AF": None if af is None else float(af), + "NS": None if ns is None else int(ns), + "carriers": carriers, + } + return oracle, samples + finally: + vcf.close() + + +@pytest.fixture(scope="module") +def oracle_and_samples(_src) -> tuple[dict[tuple[str, int], dict], list[str]]: + bcf, _ref = _src + return _build_oracle(bcf) + + +def _build_dataset(tmp_path: Path, name: str, bed: pl.DataFrame, store: Path, ref: Path): + import genvarloader as gvl + from genoray import SparseVar2 + + d = tmp_path / name + gvl.write(d, bed, variants=SparseVar2(store), samples=None, overwrite=True) + return gvl.Dataset.open(d, reference=ref) + + +_VAR_FIELDS = ["alt", "start", "ilen", "AF", "NS", "DP"] + + +# --- self-assert: fixture actually exercises BOTH channels ------------------ + + +def test_svar2_fields_store_has_fields_and_routes_both_channels(svar2_fields_store): + """Sanity gate for the fixture itself (not the wiring under test). + + ``available_fields`` must list AF/NS/DP (Task 2.1/2.2 discovery), and a + query spanning chr1 must show BOTH a non-empty var_key window (chr1:3, + carried by 1/6 haplotypes) AND a non-empty dense window (chr1:10, carried + by 6/6 haplotypes) -- else half the provenance logic (Task 2.3/2.4) would + be silently untested by the tests below. + """ + import genoray + + sv = genoray.SparseVar2(str(svar2_fields_store)) + assert set(sv.available_fields) == {"AF", "NS", "DP"}, sv.available_fields + + d = sv._find_ranges("chr1", [0], [40], samples=None) + vk_snp_range = np.asarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + dense_snp_range = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + + vk_width = int((vk_snp_range[:, 1] - vk_snp_range[:, 0]).sum()) + dense_width = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + assert vk_width >= 1, ( + f"expected chr1:3 (1/6 haplotypes) to route to var_key, but vk_snp_range " + f"is empty ({vk_snp_range.tolist()})" + ) + assert dense_width >= 1, ( + f"expected chr1:10 (6/6 haplotypes) to route to dense, but " + f"dense_snp_range is empty ({dense_snp_range.tolist()})" + ) + + +# --- shared oracle-comparison helper (diploid RaggedVariants) --------------- + + +def _assert_diploid_fields( + rv, + region_contigs: list[str], + samples: list[str], + oracle: dict[tuple[str, int], dict], + sv, +) -> None: + """Compare a diploid (ploidy-2) ``RaggedVariants`` against the oracle. + + Checks, per decoded (region, sample, ploid, variant): AF/NS by position, + DP by (position, sample) -- AND dtype (no widening) -- AND completeness + (every oracle carrier for a queried contig was actually decoded, so a + silently-dropped call is caught, not just a wrong value). + """ + af_dtype = sv.available_fields["AF"].dtype + ns_dtype = sv.available_fields["NS"].dtype + dp_dtype = sv.available_fields["DP"].dtype + assert np.asarray(rv["AF"].data).dtype == af_dtype + assert np.asarray(rv["NS"].data).dtype == ns_dtype + assert np.asarray(rv["DP"].data).dtype == dp_dtype + + start_ak = rv.start.to_ak().to_list() + af_ak = rv["AF"].to_ak().to_list() + ns_ak = rv["NS"].to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + seen: dict[tuple[str, int], set[tuple[int, int]]] = {} + for r, contig in enumerate(region_contigs): + for s_i in range(len(start_ak[r])): + for p_i in range(len(start_ak[r][s_i])): + for v_i, pos0 in enumerate(start_ak[r][s_i][p_i]): + key = (contig, int(pos0)) + assert key in oracle, f"decoded variant not in oracle: {key}" + exp = oracle[key] + + got_af = af_ak[r][s_i][p_i][v_i] + if exp["AF"] is None: + assert got_af != got_af, ( + f"expected NaN AF (missing in VCF) at {key}, got {got_af}" + ) + else: + expected_af = float(np.asarray(exp["AF"], dtype=af_dtype)) + assert got_af == expected_af, ( + f"AF mismatch at {key}: {got_af} != {expected_af}" + ) + + expected_ns = int(np.asarray(exp["NS"], dtype=ns_dtype)) + assert ns_ak[r][s_i][p_i][v_i] == expected_ns, ( + f"NS mismatch at {key}: {ns_ak[r][s_i][p_i][v_i]} != {expected_ns}" + ) + + assert (s_i, p_i) in exp["carriers"], ( + f"decoded a call not marked as a carrier in the oracle: " + f"{key} sample={samples[s_i]} hap={p_i}" + ) + expected_dp = int( + np.asarray(exp["carriers"][(s_i, p_i)], dtype=dp_dtype) + ) + got_dp = dp_ak[r][s_i][p_i][v_i] + assert got_dp == expected_dp, ( + f"DP mismatch at {key} sample={samples[s_i]}: " + f"{got_dp} != {expected_dp}" + ) + + seen.setdefault(key, set()).add((s_i, p_i)) + + for key, exp in oracle.items(): + contig, _pos0 = key + if contig not in region_contigs or not exp["carriers"]: + continue + assert seen.get(key) == set(exp["carriers"]), ( + f"missing/extra decoded carriers at {key}: " + f"expected {set(exp['carriers'])}, got {seen.get(key)}" + ) + + +# --- Test 1: single-contig --------------------------------------------------- + + +def test_svar2_ragged_variants_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genoray + + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d1.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + assert ds.samples == samples + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr1"], samples, oracle, sv) + + +# --- Test 2: multi-contig, interleaved (exercises the row-reorder path) ---- + + +def test_svar2_ragged_variants_fields_multicontig( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genoray + + oracle, samples = oracle_and_samples + _bcf, ref = _src + # Interleaved chr2/chr1/chr2/chr1, out of natural order -> >1 contig group. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 15, 20], + "chromEnd": [15, 20, 30, 40], + } + ) + ds = _build_dataset(tmp_path, "d2.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr2", "chr1", "chr2", "chr1"], samples, oracle, sv) + + +# --- Test 3: FORMAT per-sample, explicitly NOT sample 0 --------------------- + + +def test_svar2_ragged_variants_format_not_sample0( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d3.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + + rv = ds[:, :] + start_ak = rv.start.to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + dense_pos0 = 9 # chr1:10 (1-based) -> 0-based 9; carried by ALL samples/haps. + exp = oracle[("chr1", dense_pos0)] + + query_sample = "S1" + s_i = samples.index(query_sample) + assert s_i != 0, "must query a sample other than sample 0 (index 0)" + assert exp["carriers"][(s_i, 0)] != exp["carriers"][(0, 0)], ( + "fixture bug: the queried sample's DP must differ from sample 0's DP " + "at this variant, else a broken FORMAT sample-stride would go undetected" + ) + + found = False + for r in range(len(start_ak)): + for p_i in range(len(start_ak[r][s_i])): + for v_i, pos0 in enumerate(start_ak[r][s_i][p_i]): + if int(pos0) != dense_pos0: + continue + found = True + got_dp = dp_ak[r][s_i][p_i][v_i] + assert got_dp == exp["carriers"][(s_i, p_i)], ( + f"DP mismatch for sample {query_sample} at chr1:{dense_pos0}: " + f"{got_dp} != {exp['carriers'][(s_i, p_i)]}" + ) + assert found, ( + f"expected sample {query_sample} to carry the dense variant at " + f"chr1:{dense_pos0}" + ) + + +# --- Test 4: unphased_union --------------------------------------------------- + + +def test_svar2_ragged_variants_fields_unphased_union( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + """``with_settings(unphased_union=True)`` folds ploidy 2->1: ALT occurrences + from both haplotypes are concatenated per sample (no dedup, no sort). AF/NS + are per-variant (hap-independent) and DP is per-sample (identical for every + haplotype of that sample in this fixture), so each decoded occurrence must + still match the oracle by (contig, start) / (contig, start, sample), and the + per-sample occurrence COUNT must equal the number of carrying haplotypes. + """ + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d4.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings( + var_fields=_VAR_FIELDS, unphased_union=True + ) + + rv = ds[:, :] + assert rv.start.shape[-2] == 1, "unphased_union must fold ploidy to 1" + + start_ak = rv.start.to_ak().to_list() + af_ak = rv["AF"].to_ak().to_list() + ns_ak = rv["NS"].to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + counts: dict[tuple[str, int, int], int] = {} + for s_i in range(len(start_ak[0])): + for pos0, af, ns, dp in zip( + start_ak[0][s_i][0], af_ak[0][s_i][0], ns_ak[0][s_i][0], dp_ak[0][s_i][0] + ): + key = ("chr1", int(pos0)) + assert key in oracle, f"decoded variant not in oracle: {key}" + exp = oracle[key] + if exp["AF"] is None: + assert af != af, f"expected NaN AF at {key}, got {af}" + else: + assert af == pytest.approx(exp["AF"], rel=0, abs=1e-6), ( + f"AF mismatch at {key}: {af} != {exp['AF']}" + ) + assert ns == exp["NS"], f"NS mismatch at {key}: {ns} != {exp['NS']}" + sample_dp = {v for (si, _p), v in exp["carriers"].items() if si == s_i} + assert sample_dp, f"sample {samples[s_i]} unexpectedly carries {key}" + assert len(sample_dp) == 1, "fixture bug: DP must be uniform per sample" + assert dp == next(iter(sample_dp)), ( + f"DP mismatch at {key} sample={samples[s_i]}: {dp} != {sample_dp}" + ) + counts[(*key, s_i)] = counts.get((*key, s_i), 0) + 1 + + for key, exp in oracle.items(): + contig, _pos0 = key + if contig != "chr1": + continue + per_sample_hap_count: dict[int, int] = {} + for si, _p in exp["carriers"]: + per_sample_hap_count[si] = per_sample_hap_count.get(si, 0) + 1 + for s_i, expected_count in per_sample_hap_count.items(): + got_count = counts.get((*key, s_i), 0) + assert got_count == expected_count, ( + f"union occurrence count mismatch at {key} sample={samples[s_i]}: " + f"{got_count} != {expected_count}" + ) + + +# --- Test 5: variant-windows, including an EMPTY-group fill-value case ----- + + +def test_svar2_variant_windows_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genvarloader as gvl + + oracle, samples = oracle_and_samples + _bcf, ref = _src + # Region 0 covers both chr1 variants; region 1 (chr1:20-40) has NONE. + bed = pl.DataFrame( + {"chrom": ["chr1", "chr1"], "chromStart": [0, 20], "chromEnd": [20, 40]} + ) + ds = _build_dataset(tmp_path, "d5.gvl", bed, svar2_fields_store, ref) + opt = gvl.VarWindowOpt(flank_length=3, token_alphabet=b"ACGT", unknown_token=4) + ds = ( + ds.with_output_format("flat") + .with_seqs("variant-windows", opt) + .with_settings( + var_fields=_VAR_FIELDS, + dummy_variant=gvl.DummyVariant(alt=b"N", ref=b"N"), + ) + ) + win = ds[:, :] + assert "AF" in win.fields and "NS" in win.fields and "DP" in win.fields + + P = 2 + S = len(samples) + start_off = np.asarray(win.fields["start"].offsets) + start_data = np.asarray(win.fields["start"].data) + af_data = np.asarray(win.fields["AF"].data) + ns_data = np.asarray(win.fields["NS"].data) + dp_data = np.asarray(win.fields["DP"].data) + + def _group(r: int, s_i: int, p_i: int) -> tuple[int, int]: + g = (r * S + s_i) * P + p_i + return int(start_off[g]), int(start_off[g + 1]) + + # Region 0: real variants, checked against the oracle exactly as in the + # diploid RaggedVariants test. + for s_i in range(S): + for p_i in range(P): + lo, hi = _group(0, s_i, p_i) + for i in range(lo, hi): + pos0 = int(start_data[i]) + key = ("chr1", pos0) + assert key in oracle, f"decoded variant not in oracle: {key}" + exp = oracle[key] + if exp["AF"] is None: + assert af_data[i] != af_data[i], ( + f"expected NaN AF at {key}, got {af_data[i]}" + ) + else: + assert af_data[i] == pytest.approx(exp["AF"], abs=1e-6) + assert ns_data[i] == exp["NS"] + assert (s_i, p_i) in exp["carriers"], ( + f"decoded a call not marked as carrier: {key} sample={samples[s_i]}" + ) + assert dp_data[i] == exp["carriers"][(s_i, p_i)] + + # Region 1: variant-free -> the dummy fill must appear (exactly 1 entry per + # group), with the documented fill values: NaN for the float AF column, 0 + # for the integer NS/DP columns (DummyVariant.info was left empty). + for s_i in range(S): + for p_i in range(P): + lo, hi = _group(1, s_i, p_i) + assert hi - lo == 1, ( + f"expected exactly 1 dummy variant in the empty group " + f"(region 1, sample {samples[s_i]}, hap {p_i}), got {hi - lo}" + ) + assert af_data[lo] != af_data[lo], ( + f"expected NaN AF fill for the empty group, got {af_data[lo]}" + ) + assert ns_data[lo] == 0, f"expected 0 NS fill, got {ns_data[lo]}" + assert dp_data[lo] == 0, f"expected 0 DP fill, got {dp_data[lo]}" From ce03789d8a4c3dd20c1dba7ef0347259c9217931 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 03:58:16 -0700 Subject: [PATCH 083/108] feat(svar2): route INFO/FORMAT fields into RaggedVariants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _reconstruct_variants now forwards the requested store fields to the decode kernel and attaches the returned buffers to the output, viewing the raw little-endian bytes as the store's dtype (no widening; missing entries keep the stored default verbatim, guarded by an itemsize check against the manifest). Field buffers are flat per-variant data parallel to pos, so they ride the existing offsets machinery: the multi-contig path reuses the same src permutation as pos/ilen, and unphased_union (which only folds offsets) needs no handling at all. Builtin names still win over a same-named store field. With no fields requested the kernel keeps its zero-overhead path. Oracle test: 5 passed (variant-windows still red — next task). --- python/genvarloader/_dataset/_svar2_haps.py | 39 +++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 61e5d175..96b3cbd5 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -592,15 +592,24 @@ def _reconstruct_variants( groups = self._contig_groups(contig_ids) p_eff = 1 if self.unphased_union else P + req_keys = [ + f + for f in self.var_fields + if f not in _BUILTIN_VAR_FIELDS and f in self.store_fields + ] + field_specs = [_field_spec(self.store_fields[k]) for k in req_keys] + field_dtypes = [self.store_fields[k].dtype for k in req_keys] + cat_var_lens: list[NDArray[np.int64]] = [] cat_pos: list[NDArray[np.int32]] = [] cat_ilen: list[NDArray[np.int32]] = [] cat_alt: list[NDArray[np.uint8]] = [] cat_var_bytelen: list[NDArray[np.int64]] = [] cat_query_order: list[NDArray[np.intp]] = [] + cat_fields: list[list[NDArray]] = [] for ci, qsel in groups: gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) - pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( decode_variants_from_svar2_readbound( self.store, self.ds_contigs[ci], @@ -611,7 +620,7 @@ def _reconstruct_variants( gi[4], gi[5], P, - [], + field_specs, ) ) var_off = np.asarray(var_off, np.int64) @@ -625,6 +634,16 @@ def _reconstruct_variants( cat_var_bytelen.append(np.diff(str_off)) cat_query_order.append(qsel) + typed = [] + for j, dt in enumerate(field_dtypes): + if field_isizes[j] != dt.itemsize: + raise AssertionError( + f"field {req_keys[j]!r}: kernel itemsize {field_isizes[j]} != " + f"store dtype {dt} itemsize {dt.itemsize}" + ) + typed.append(np.asarray(field_bufs[j], np.uint8).view(dt)) + cat_fields.append(typed) + # Single contig group: grouped order already equals global (b, P) order, # so the reorder is the identity and every concatenate is a 1-element no-op. # Skip both (the numpy reorder otherwise dominates single-contig reads). @@ -632,12 +651,17 @@ def _reconstruct_variants( shape = (b, p_eff, None) var_off_g = lengths_to_offsets(cat_var_lens[0], np.int64) str_off_g = lengths_to_offsets(cat_var_bytelen[0], np.int64) + extra = { + k: Ragged.from_offsets(cat_fields[0][j], shape, var_off_g) + for j, k in enumerate(req_keys) + } return RaggedVariants( alt=Ragged.from_offsets( cat_alt[0].view("S1"), shape, var_off_g, str_offsets=str_off_g ), start=Ragged.from_offsets(cat_pos[0], shape, var_off_g), ilen=Ragged.from_offsets(cat_ilen[0], shape, var_off_g), + **extra, ) # Concatenate grouped outputs, then permute hap-rows back to global order. @@ -674,7 +698,16 @@ def _reconstruct_variants( alt_r = Ragged.from_offsets( alt_g.view("S1"), shape, alt_var_off_g, str_offsets=alt_str_off_g ) - return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r) + extra = {} + for j, k in enumerate(req_keys): + fc = ( + np.concatenate([g[j] for g in cat_fields]) + if cat_fields + else np.zeros(0, field_dtypes[j]) + ) + fg = fc[:0].copy() if src.size == 0 else fc[src] + extra[k] = Ragged.from_offsets(fg, shape, var_off_g) + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r, **extra) def _reconstruct_variant_windows( self, idx: NDArray[np.integer], regions: NDArray[np.integer] From e4c2721dd9a3c7f901f35c72e6f56badc2188e91 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 04:07:28 -0700 Subject: [PATCH 084/108] feat(svar2): route INFO/FORMAT fields into variant-windows output _reconstruct_variant_windows now forwards the requested store fields to the decode kernel and adds the returned columns to _FlatVariantWindows.fields, alongside start/ilen. Field buffers are flat per-variant data parallel to pos, so they share the same row offsets (including under unphased_union) and reuse the same src permutation in the multi-contig path. The field-resolution block is extracted into a shared _requested_store_fields() helper used by both reconstructors rather than duplicated. Empty-group dummy fill needed no new code: fill_empty_groups already iterates the fields dict and DummyVariant.scalar_for already falls back to NaN for float columns and 0 for integer columns. Verified against the oracle test rather than assumed. Oracle test now 6/6 green. --- python/genvarloader/_dataset/_svar2_haps.py | 54 +++++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 96b3cbd5..0fb668f2 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -580,6 +580,25 @@ def realign_track_block( # ---- variants ---- + def _requested_store_fields( + self, + ) -> tuple[list[str], list[tuple[str, str, str]], list[np.dtype]]: + """The store INFO/FORMAT fields requested via ``var_fields``. + + Returns ``(keys, specs, dtypes)`` where ``specs`` is what the decode kernel + expects and ``keys``/``dtypes`` are positionally parallel to the field + buffers it returns. Builtin names (alt/start/ilen/ref/dosage) always mean the + builtin, even if the store happens to carry a field of the same name. + """ + keys = [ + f + for f in self.var_fields + if f not in _BUILTIN_VAR_FIELDS and f in self.store_fields + ] + specs = [_field_spec(self.store_fields[k]) for k in keys] + dtypes = [self.store_fields[k].dtype for k in keys] + return keys, specs, dtypes + def _reconstruct_variants( self, idx: NDArray[np.integer], regions: NDArray[np.integer] ) -> RaggedVariants: @@ -592,13 +611,7 @@ def _reconstruct_variants( groups = self._contig_groups(contig_ids) p_eff = 1 if self.unphased_union else P - req_keys = [ - f - for f in self.var_fields - if f not in _BUILTIN_VAR_FIELDS and f in self.store_fields - ] - field_specs = [_field_spec(self.store_fields[k]) for k in req_keys] - field_dtypes = [self.store_fields[k].dtype for k in req_keys] + req_keys, field_specs, field_dtypes = self._requested_store_fields() cat_var_lens: list[NDArray[np.int64]] = [] cat_pos: list[NDArray[np.int32]] = [] @@ -736,17 +749,20 @@ def _reconstruct_variant_windows( p_eff = 1 if self.unphased_union else P + req_keys, field_specs, field_dtypes = self._requested_store_fields() + cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries cat_pos: list[NDArray[np.int32]] = [] cat_ilen: list[NDArray[np.int32]] = [] cat_query_order: list[NDArray[np.intp]] = [] + cat_fields: list[list[NDArray]] = [] # name -> per-group (token_data, per-variant seq offsets) win_data: dict[str, list[NDArray]] = {} win_seq_off: dict[str, list[NDArray[np.int64]]] = {} for ci, qsel in groups: gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) - pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( decode_variants_from_svar2_readbound( self.store, self.ds_contigs[ci], @@ -757,7 +773,7 @@ def _reconstruct_variant_windows( gi[4], gi[5], P, - [], + field_specs, ) ) pos = np.asarray(pos, np.int32) @@ -801,6 +817,16 @@ def _reconstruct_variant_windows( win_data.setdefault(name, []).append(np.asarray(data)) win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) + typed = [] + for j, dt in enumerate(field_dtypes): + if field_isizes[j] != dt.itemsize: + raise AssertionError( + f"field {req_keys[j]!r}: kernel itemsize {field_isizes[j]} != " + f"store dtype {dt} itemsize {dt.itemsize}" + ) + typed.append(np.asarray(field_bufs[j], np.uint8).view(dt)) + cat_fields.append(typed) + shape: tuple[int | None, ...] = (b, p_eff, None) wshape: tuple[int | None, ...] = (b, p_eff, None, None) @@ -812,6 +838,8 @@ def _reconstruct_variant_windows( } if include_ilen: fields["ilen"] = _Flat.from_offsets(cat_ilen[0], shape, row_off) + for j, k in enumerate(req_keys): + fields[k] = _Flat.from_offsets(cat_fields[0][j], shape, row_off) win = _FlatVariantWindows(fields) for name in win_data: setattr( @@ -838,6 +866,14 @@ def _reconstruct_variant_windows( fields = {"start": _Flat.from_offsets(pos_g, shape, row_off_g)} if include_ilen: fields["ilen"] = _Flat.from_offsets(ilen_g, shape, row_off_g) + for j, k in enumerate(req_keys): + fc = ( + np.concatenate([g[j] for g in cat_fields]) + if cat_fields + else np.zeros(0, field_dtypes[j]) + ) + fg = fc[:0].copy() if src.size == 0 else fc[src] + fields[k] = _Flat.from_offsets(fg, shape, row_off_g) win = _FlatVariantWindows(fields) for name in win_data: data_c = np.concatenate(win_data[name]) From 39136c472d2c70829a22bf6ab0211878d784c57d Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 04:40:19 -0700 Subject: [PATCH 085/108] fix(svar2): honor Dataset.open(var_fields=) and cover multi-contig windows fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real gaps found by review: Dataset.open(var_fields=...) was a silent no-op for SVAR2: var_fields is a public open() parameter forwarded to Haps.from_path but not to Svar2Haps.from_path, so a store field would be advertised in available_var_fields and then be absent from the output with no error. Svar2Haps.from_path now accepts and honors var_fields. The multi-contig variant-windows field path was entirely untested — the windows test used a single-contig bed, so zeroing the gathered field data in the multi-group branch passed the whole suite. The test's bed is now interleaved across contigs (keeping the variant-free region for the dummy-fill assertions), and that mutation now fails. Also fold in review cleanups: extract the duplicated per-group typing block into a shared helper, and raise ValueError (not a hand-raised AssertionError) when the kernel's itemsize disagrees with the store manifest. Field tests 7/7; no-regression 25/25. --- python/genvarloader/_dataset/_open.py | 1 + python/genvarloader/_dataset/_svar2_haps.py | 55 ++++--- tests/dataset/test_svar2_fields_read.py | 150 +++++++++++++++----- 3 files changed, 152 insertions(+), 54 deletions(-) diff --git a/python/genvarloader/_dataset/_open.py b/python/genvarloader/_dataset/_open.py index 485424f0..03b59441 100644 --- a/python/genvarloader/_dataset/_open.py +++ b/python/genvarloader/_dataset/_open.py @@ -166,6 +166,7 @@ def _build_seqs( min_af=self.min_af, max_af=self.max_af, max_jitter=metadata.max_jitter, + var_fields=self.var_fields, ) else: seqs = Haps.from_path( diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 0fb668f2..24fda949 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -212,7 +212,13 @@ def from_path( # type: ignore[override] # separate svar2 signature; base retur min_af: float | None = None, max_af: float | None = None, max_jitter: int = 0, + var_fields: list[str] | None = None, ) -> "Svar2Haps": + # Default var_fields for loading. var_fields=None means "use the default + # set" — mirrors Haps.from_path's resolution. + if var_fields is None: + var_fields = ["alt", "ilen", "start"] + ranges_dir = path / "genotypes" / "svar2_ranges" with open(ranges_dir / "svar2_meta.json") as f: meta = json.load(f) @@ -288,6 +294,7 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: ds_contigs=list(contigs), max_jitter=max_jitter, store_fields=store_fields, + var_fields=var_fields, ) # ---- reconstructor entry ---- @@ -599,6 +606,30 @@ def _requested_store_fields( dtypes = [self.store_fields[k].dtype for k in keys] return keys, specs, dtypes + def _type_field_bufs( + self, + bufs: list[NDArray], + isizes: list[int], + keys: list[str], + dtypes: list[np.dtype], + ) -> list[NDArray]: + """View each raw ``uint8`` field buffer the decode kernel returned as its + store dtype. + + Guards that the kernel's reported itemsize agrees with the store + manifest -- a store/kernel disagreement, not an internal invariant, so + this raises ``ValueError`` (not an assertion). + """ + typed = [] + for j, dt in enumerate(dtypes): + if isizes[j] != dt.itemsize: + raise ValueError( + f"field {keys[j]!r}: kernel itemsize {isizes[j]} != " + f"store dtype {dt} itemsize {dt.itemsize}" + ) + typed.append(np.asarray(bufs[j], np.uint8).view(dt)) + return typed + def _reconstruct_variants( self, idx: NDArray[np.integer], regions: NDArray[np.integer] ) -> RaggedVariants: @@ -647,15 +678,9 @@ def _reconstruct_variants( cat_var_bytelen.append(np.diff(str_off)) cat_query_order.append(qsel) - typed = [] - for j, dt in enumerate(field_dtypes): - if field_isizes[j] != dt.itemsize: - raise AssertionError( - f"field {req_keys[j]!r}: kernel itemsize {field_isizes[j]} != " - f"store dtype {dt} itemsize {dt.itemsize}" - ) - typed.append(np.asarray(field_bufs[j], np.uint8).view(dt)) - cat_fields.append(typed) + cat_fields.append( + self._type_field_bufs(field_bufs, field_isizes, req_keys, field_dtypes) + ) # Single contig group: grouped order already equals global (b, P) order, # so the reorder is the identity and every concatenate is a 1-element no-op. @@ -817,15 +842,9 @@ def _reconstruct_variant_windows( win_data.setdefault(name, []).append(np.asarray(data)) win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) - typed = [] - for j, dt in enumerate(field_dtypes): - if field_isizes[j] != dt.itemsize: - raise AssertionError( - f"field {req_keys[j]!r}: kernel itemsize {field_isizes[j]} != " - f"store dtype {dt} itemsize {dt.itemsize}" - ) - typed.append(np.asarray(field_bufs[j], np.uint8).view(dt)) - cat_fields.append(typed) + cat_fields.append( + self._type_field_bufs(field_bufs, field_isizes, req_keys, field_dtypes) + ) shape: tuple[int | None, ...] = (b, p_eff, None) wshape: tuple[int | None, ...] = (b, p_eff, None, None) diff --git a/tests/dataset/test_svar2_fields_read.py b/tests/dataset/test_svar2_fields_read.py index ca4c0bac..b4cc40e4 100644 --- a/tests/dataset/test_svar2_fields_read.py +++ b/tests/dataset/test_svar2_fields_read.py @@ -136,7 +136,9 @@ def oracle_and_samples(_src) -> tuple[dict[tuple[str, int], dict], list[str]]: return _build_oracle(bcf) -def _build_dataset(tmp_path: Path, name: str, bed: pl.DataFrame, store: Path, ref: Path): +def _build_dataset( + tmp_path: Path, name: str, bed: pl.DataFrame, store: Path, ref: Path +): import genvarloader as gvl from genoray import SparseVar2 @@ -428,9 +430,16 @@ def test_svar2_variant_windows_fields( oracle, samples = oracle_and_samples _bcf, ref = _src - # Region 0 covers both chr1 variants; region 1 (chr1:20-40) has NONE. + # Interleaved across contigs (chr2, chr1, chr1) -> >1 contig group, so the + # multi-contig branch of _reconstruct_variant_windows actually runs (not + # just the single-group fast path). Region 0 covers the chr2 variant; + # region 1 covers both chr1 variants; region 2 (chr1:20-40) has NONE. bed = pl.DataFrame( - {"chrom": ["chr1", "chr1"], "chromStart": [0, 20], "chromEnd": [20, 40]} + { + "chrom": ["chr2", "chr1", "chr1"], + "chromStart": [0, 0, 20], + "chromEnd": [15, 20, 40], + } ) ds = _build_dataset(tmp_path, "d5.gvl", bed, svar2_fields_store, ref) opt = gvl.VarWindowOpt(flank_length=3, token_alphabet=b"ACGT", unknown_token=4) @@ -457,40 +466,109 @@ def _group(r: int, s_i: int, p_i: int) -> tuple[int, int]: g = (r * S + s_i) * P + p_i return int(start_off[g]), int(start_off[g + 1]) - # Region 0: real variants, checked against the oracle exactly as in the - # diploid RaggedVariants test. - for s_i in range(S): - for p_i in range(P): - lo, hi = _group(0, s_i, p_i) - for i in range(lo, hi): - pos0 = int(start_data[i]) - key = ("chr1", pos0) - assert key in oracle, f"decoded variant not in oracle: {key}" - exp = oracle[key] - if exp["AF"] is None: - assert af_data[i] != af_data[i], ( - f"expected NaN AF at {key}, got {af_data[i]}" + def _assert_dummy_fill(lo: int, hi: int, where: str) -> None: + assert hi - lo == 1, ( + f"expected exactly 1 dummy variant in the empty group ({where}), " + f"got {hi - lo}" + ) + assert af_data[lo] != af_data[lo], ( + f"expected NaN AF fill for the empty group ({where}), got {af_data[lo]}" + ) + assert ns_data[lo] == 0, f"expected 0 NS fill ({where}), got {ns_data[lo]}" + assert dp_data[lo] == 0, f"expected 0 DP fill ({where}), got {dp_data[lo]}" + + # Regions 0 (chr2:0-15) and 1 (chr1:0-20) carry real oracle variants, but + # NOT every (sample, hap) is a carrier of every variant in a region (e.g. + # chr2:5 has exactly one carrier) -- so a (sample, hap) group with no + # carried variant in that region still gets the dummy fill, same as a + # wholly variant-free region. Compute, per region, which oracle keys fall + # in it, then check each group against exactly the keys it carries. + region_keys = { + 0: [k for k in oracle if k[0] == "chr2"], + 1: [k for k in oracle if k[0] == "chr1" and k[1] < 20], + } + for r, keys in region_keys.items(): + for s_i in range(S): + for p_i in range(P): + lo, hi = _group(r, s_i, p_i) + expected_keys = {k for k in keys if (s_i, p_i) in oracle[k]["carriers"]} + if not expected_keys: + _assert_dummy_fill( + lo, hi, f"region {r}, sample {samples[s_i]}, hap {p_i}" ) - else: - assert af_data[i] == pytest.approx(exp["AF"], abs=1e-6) - assert ns_data[i] == exp["NS"] - assert (s_i, p_i) in exp["carriers"], ( - f"decoded a call not marked as carrier: {key} sample={samples[s_i]}" + continue + assert hi - lo == len(expected_keys), ( + f"expected {len(expected_keys)} variant(s) for region {r}, " + f"sample {samples[s_i]}, hap {p_i}, got {hi - lo}" ) - assert dp_data[i] == exp["carriers"][(s_i, p_i)] - - # Region 1: variant-free -> the dummy fill must appear (exactly 1 entry per - # group), with the documented fill values: NaN for the float AF column, 0 - # for the integer NS/DP columns (DummyVariant.info was left empty). + seen_pos: set[int] = set() + for i in range(lo, hi): + pos0 = int(start_data[i]) + key = (keys[0][0], pos0) + assert key in expected_keys, ( + f"decoded variant not expected as a carrier here: {key} " + f"sample={samples[s_i]} hap={p_i}" + ) + seen_pos.add(pos0) + exp = oracle[key] + if exp["AF"] is None: + assert af_data[i] != af_data[i], ( + f"expected NaN AF at {key}, got {af_data[i]}" + ) + else: + assert af_data[i] == pytest.approx(exp["AF"], abs=1e-6) + assert ns_data[i] == exp["NS"] + assert dp_data[i] == exp["carriers"][(s_i, p_i)] + assert seen_pos == {k[1] for k in expected_keys} + + # Region 2 (chr1:20-40): variant-free -> the dummy fill must appear + # (exactly 1 entry per group), with the documented fill values: NaN for + # the float AF column, 0 for the integer NS/DP columns (DummyVariant.info + # was left empty). for s_i in range(S): for p_i in range(P): - lo, hi = _group(1, s_i, p_i) - assert hi - lo == 1, ( - f"expected exactly 1 dummy variant in the empty group " - f"(region 1, sample {samples[s_i]}, hap {p_i}), got {hi - lo}" - ) - assert af_data[lo] != af_data[lo], ( - f"expected NaN AF fill for the empty group, got {af_data[lo]}" - ) - assert ns_data[lo] == 0, f"expected 0 NS fill, got {ns_data[lo]}" - assert dp_data[lo] == 0, f"expected 0 DP fill, got {dp_data[lo]}" + lo, hi = _group(2, s_i, p_i) + _assert_dummy_fill(lo, hi, f"region 2, sample {samples[s_i]}, hap {p_i}") + + +# --- Test 6: Dataset.open(var_fields=...) entry point (not with_settings) --- + + +def test_svar2_dataset_open_var_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + """``Dataset.open``'s own ``var_fields`` kwarg must actually route to the + svar2 reconstructor. This is a different code path from + ``with_settings(var_fields=...)`` (exercised by every other test in this + file): ``Dataset.open`` forwards ``var_fields`` to ``Svar2Haps.from_path`` + directly, which previously had no such parameter, so the field was + silently dropped -- advertised in ``available_var_fields`` but absent from + the actual output (no error, no warning). + """ + import genoray + import genvarloader as gvl + + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d6.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + ds = gvl.Dataset.open(d, reference=ref, var_fields=_VAR_FIELDS).with_seqs( + "variants" + ) + assert "NS" in ds.available_var_fields + assert ds.active_var_fields == _VAR_FIELDS, ( + f"Dataset.open(var_fields=...) did not take effect: " + f"{ds.active_var_fields} != {_VAR_FIELDS}" + ) + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr1"], samples, oracle, sv) From 729489f979411ed3ff6cf8f401d9f55afa2db705 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 04:52:24 -0700 Subject: [PATCH 086/108] docs(svar2): document INFO/FORMAT field routing in variants outputs Extends the existing .svar2 sections of the genvarloader skill (Phase-1 scope, the var_fields entry, and the gotchas matrix) and the faq's .svar2 support sentence. No CHANGELOG entry: this repo has no root changelog, and docs/source/changelog.md is auto-generated from commit messages by commitizen (and per CLAUDE.md does not count as documentation). The plan's changelog step assumed genoray's convention. --- docs/source/faq.md | 2 +- ...07-03-svar2-genoray-search-gather-split.md | 953 ++++++++++++++ .../2026-07-03-svar2-gvl-dataset-wiring.md | 941 ++++++++++++++ ...26-07-04-svar2-genoray-readbound-gather.md | 1092 +++++++++++++++++ .../2026-07-04-svar2-gvl-readbound-wiring.md | 981 +++++++++++++++ skills/genvarloader/SKILL.md | 6 +- 6 files changed, 3972 insertions(+), 3 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md create mode 100644 docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md create mode 100644 docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md create mode 100644 docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md diff --git a/docs/source/faq.md b/docs/source/faq.md index 25d4963d..9b505c82 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -78,7 +78,7 @@ Both are sparse columnar variant archives from [`genoray`](https://github.com/mc - **`.svar`** reconstructs by building an interval search tree over the queried window and a per-read dense union of the overlapping variants. - **`.svar2`** reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges at write time, and `Dataset.__getitem__` gathers directly off that cache and calls all-Rust kernels — it builds **no interval search tree and no dense union per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. -`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output and `unphased_union` (for both `"variants"` and `"variant-windows"`) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. +`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output, `unphased_union` (for both `"variants"` and `"variant-windows"`), and `var_fields`-selected store INFO/FORMAT fields (also for both, when the `.svar2` was written with them) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs (and `FlatVariantWindows.alt`/`.alt_window` for `"variant-windows"`), and only for pure-deletion records. `ref_window` is byte-identical between the two backends. diff --git a/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md b/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md new file mode 100644 index 00000000..cab59f9a --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md @@ -0,0 +1,953 @@ +# SVAR2 genoray `find_ranges` / `gather_ranges` / `read_ranges` Split — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split genoray's fused `SparseVar2.overlap_batch` into a *search-only* `find_ranges`, a *tree-free* `gather_ranges`, and a fused `read_ranges` wrapper — so a downstream cache can run the interval search once (at write time) and replay it at read time with no `SearchTree::build`. + +**Architecture:** Refactor the Rust `query::overlap_batch` into two pure functions sharing the region-independent dense union: `find_ranges` (runs every `SearchTree::new` and returns a compact `RangesBundle` of index ranges) and `gather_ranges` (consumes the bundle, does pure slicing + `carried` tests + k-way merge, no trees). Expose all three on `PyContigReader`, then on the Python `SparseVar2` class with `samples=` subsetting and an `out=` streaming path on `find_ranges`. `read_ranges = gather_ranges(find_ranges(...))` is the parity oracle. + +**Tech Stack:** Rust (PyO3, ndarray, numpy crate), Python 3.10+, pixi, maturin, pytest, cargo test. + +**Repo:** `/carter/users/dlaub/projects/genoray` — branch off `svar-2`. This is a **separate deliverable** that must ship a wheel before the gvl wiring plan (`2026-07-03-svar2-gvl-dataset-wiring.md`) can consume it. + +## Global Constraints + +- **Byte-identical parity contract** (verbatim from spec): for any `contig, starts, ends, samples`, + `overlap_batch(...)` ≡ `read_ranges(...)` ≡ `gather_ranges(find_ranges(...))`, and the reconstruction + from any of them ≡ the genoray `decode` oracle, **field-for-field / byte-for-byte**. +- **`samples=None` subset** on all three public Python methods, matching every other `SparseVar` range method (`_find_starts_ends`, `read_ranges`): `None` → all samples; a list restricts which samples' offsets/payload are computed. Unknown samples raise `ValueError`. +- **`out=` streaming** on `find_ranges` only, mirroring `SparseVar._find_starts_ends(..., out=out)` — writes the bundle into caller-preallocated arrays so `gvl.write` can stream straight to a memmap. +- **`gather_ranges` performs ZERO interval search** — no `SearchTree::new` anywhere in its call graph. This is the entire point; a test asserts it. +- **Additive:** `overlap_batch` stays working and byte-unchanged (it may later be deprecated — maintainer's call, out of scope here). All existing genoray tests stay green. +- **Rust vs Python build:** `cargo test` compiles from source and needs no rebuild. Python tests import the compiled extension — **run `pixi run maturin develop --release` before any pytest that exercises new bindings**, or pytest imports the stale `.so`. +- **Conventional commits** (commitizen). Ensure prek hooks are installed before the first commit (`pixi run prek-install`). + +--- + +## File Structure + +**Rust (`src/`):** +- `src/query.rs` — add `RangesBundle` struct, `fn find_ranges`, `fn gather_ranges`, `fn read_ranges`. Refactor `overlap_batch`'s body to share `dense_union` + the inner gather loop with `gather_ranges` (DRY; `overlap_batch` becomes `gather_ranges(&reader, &find_ranges(reader, regions, None))` internally, or keeps its own body — see Task 3). +- `src/py_query_ranges.rs` — **new** `#[pymethods]` block on `PyContigReader` exposing `find_ranges` / `gather_ranges` / `read_ranges` as numpy-dict methods (mirrors `src/py_query_batch.rs`; a separate file keeps M6b's `overlap_batch` binding untouched, per the existing multiple-pymethods convention). +- `src/lib.rs` — register the new module (`mod py_query_ranges;`). + +**Python (`python/genoray/`):** +- `python/genoray/_svar2_batch.py` — add `find_ranges`, `gather_ranges`, `read_ranges` methods to `_BatchQueryMixin` (next to `overlap_batch`), each resolving `samples=` to column indices and delegating to the Rust `PyContigReader`. +- `python/genoray/_svar2.py` — no signature change; `SparseVar2` already mixes in `_BatchQueryMixin`. Confirm the new methods surface. + +**Rust tests (`tests/`):** +- `tests/test_ranges_split.rs` — **new** cargo integration test: `read_ranges` bundle ≡ `overlap_batch` field-for-field; `gather_ranges` is search-free. + +**Python tests (`tests/`):** +- `tests/test_svar2_ranges.py` — **new** pytest: Python `find_ranges`/`gather_ranges`/`read_ranges` parity vs `overlap_batch`, `samples=` subsetting, and `out=` streaming. + +**Docs:** +- `docs/roadmaps/` (genoray's own roadmap) + `CHANGELOG.md` — record the split. + +--- + +### Task 1: `RangesBundle` struct + `find_ranges` (search-only Rust core) + +**Files:** +- Modify: `src/query.rs` (add near `overlap_batch`, ~line 509) +- Test: `tests/test_ranges_split.rs` (create) + +**Interfaces:** +- Consumes: `ContigReader` (existing), `ContigReader::dense_union() -> DenseUnion` (existing, `src/query.rs:351`), `DenseUnion::overlap(qs, qe) -> (usize, usize)` (existing, `:284`), `SearchTree`/`overlap_range` (existing, `src/search.rs`), `ContigReader::vk_snp`/`vk_indel` column accessors (existing, used in `vk_slice` `:296`). +- Produces: + ```rust + pub struct RangesBundle { + pub n_regions: usize, + pub n_samples: usize, // number of SELECTED samples (subset-aware) + pub ploidy: usize, + pub region_starts: Vec, // (R) q_start per region — needed by gather's left-overlap re-check + pub dense_range: Vec<(usize, usize)>, // (R) [s,e) into the shared dense union + pub sample_cols: Vec, // (n_samples) selected slot -> original sample index + pub vk_snp_range: Vec<(usize, usize)>, // (R*H) absolute [start,end) into vk_snp packed positions/keys + pub vk_indel_range: Vec<(usize, usize)>, // (R*H) absolute [start,end) into vk_indel packed positions/keys + } + // H = n_samples * ploidy; row (r*H + h), h = selected_s*ploidy + p. + pub fn find_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, // original sample indices; None = all + ) -> RangesBundle; + ``` + +- [ ] **Step 1: Write the failing cargo test** + +Create `tests/test_ranges_split.rs`: + +```rust +//! SVAR2 search/gather split: find_ranges produces the index ranges that +//! gather_ranges replays into the same BatchResult overlap_batch returns. + +mod common; + +use common::{SynthRecord, build_contig}; +use genoray_core::query::{ContigReader, find_ranges, overlap_batch}; +use tempfile::tempdir; + +fn synth_reader(out: &std::path::Path) -> ContigReader { + let samples = ["S0", "S1"]; + let records = vec![ + SynthRecord { pos: 100, ref_allele: b"A", alts: vec![&b"C"[..]], gt: vec![1, 0, 0, 0] }, + SynthRecord { pos: 200, ref_allele: b"A", alts: vec![&b"AT"[..]], gt: vec![0, 1, 1, 1] }, + SynthRecord { pos: 300, ref_allele: b"AT", alts: vec![&b"A"[..]], gt: vec![1, 1, 0, 1] }, + ]; + build_contig(out, "chr1", &samples, 2, &records); + ContigReader::open(out.to_str().unwrap(), "chr1", 2, 2).unwrap() +} + +#[test] +fn test_find_ranges_dense_range_matches_overlap_batch() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let br = overlap_batch(&reader, ®ions); + let rb = find_ranges(&reader, ®ions, None); + + // Same per-region dense index ranges; H+1 vk_off implies R*H vk sub-ranges. + assert_eq!(rb.dense_range, br.dense_range); + assert_eq!(rb.n_regions, br.n_regions); + assert_eq!(rb.n_samples, br.n_samples); + assert_eq!(rb.ploidy, br.ploidy); + assert_eq!(rb.vk_snp_range.len(), regions.len() * br.n_samples * br.ploidy); + assert_eq!(rb.vk_indel_range.len(), regions.len() * br.n_samples * br.ploidy); + assert_eq!(rb.region_starts, vec![0u32, 250u32]); +} +``` + +- [ ] **Step 2: Run the test to verify it fails to compile** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: FAIL — `cannot find function find_ranges` / `RangesBundle` unresolved. + +- [ ] **Step 3: Implement `RangesBundle` + `find_ranges`** + +In `src/query.rs`, add above `overlap_batch` (adapting the search half of `overlap_batch` at `:509` and the `overlap_range` call from `vk_slice`/`gather_keys`). `find_ranges` runs every `SearchTree::new`; it must **not** gather keys or compute presence bits. + +```rust +/// Search-only half of the batch query: every `SearchTree::new` runs here, and +/// the result is a compact bundle of index ranges that `gather_ranges` replays +/// with no further search. Mirrors `SparseVar::_find_starts_ends`. +pub struct RangesBundle { + pub n_regions: usize, + pub n_samples: usize, + pub ploidy: usize, + pub region_starts: Vec, + pub dense_range: Vec<(usize, usize)>, + pub sample_cols: Vec, + pub vk_snp_range: Vec<(usize, usize)>, + pub vk_indel_range: Vec<(usize, usize)>, +} + +pub fn find_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, +) -> RangesBundle { + let ploidy = reader.ploidy; + let sample_cols: Vec = match samples { + Some(s) => s.to_vec(), + None => (0..reader.n_samples).collect(), + }; + let n_samples = sample_cols.len(); + let n_regions = regions.len(); + let h = n_samples * ploidy; + + // Region-independent union; `overlap` builds one SearchTree per region. + let dense = reader.dense_union(); + let dense_range: Vec<(usize, usize)> = + regions.iter().map(|&(qs, qe)| dense.overlap(qs, qe)).collect(); + let region_starts: Vec = regions.iter().map(|&(qs, _)| qs).collect(); + + let mut vk_snp_range = Vec::with_capacity(n_regions * h); + let mut vk_indel_range = Vec::with_capacity(n_regions * h); + for &(qs, qe) in regions { + for &orig_s in &sample_cols { + for p in 0..ploidy { + let col = orig_s * ploidy + p; + vk_snp_range.push(reader.vk_snp_overlap(col, qs, qe)); + vk_indel_range.push(reader.vk_indel_overlap(col, orig_s, p, qs, qe)); + } + } + } + + RangesBundle { + n_regions, + n_samples, + ploidy, + region_starts, + dense_range, + sample_cols, + vk_snp_range, + vk_indel_range, + } +} +``` + +Then add two search-only helpers on `impl ContigReader` (extract the `overlap_range` calls out of `vk_slice` at `:296`; return **absolute** `[o0+s_idx, o0+e_idx)` indices into the packed column so gather needs no column lookup): + +```rust +/// Absolute [start,end) into vk_snp's packed positions/keys for (col, region). +/// The SNP channel's search half (max_del = 0). No gather. +fn vk_snp_overlap(&self, col: usize, q_start: u32, q_end: u32) -> (usize, usize) { + let (o0, o1) = self.vk_snp.column(col); + let positions = &self.vk_snp.positions()[o0..o1]; + if positions.is_empty() { + return (o0, o0); + } + let v_ends: Vec = positions.iter().map(|&p| p + 1).collect(); + let tree = crate::search::SearchTree::new(positions); + let (s, e) = crate::search::overlap_range(&tree, &v_ends, 0, q_start, q_end); + (o0 + s, o0 + e) +} + +/// Absolute [start,end) into vk_indel's packed positions/keys for (col, region). +/// The indel channel's search half (per-column max_del bound). No gather. +fn vk_indel_overlap(&self, col: usize, sample: usize, p: usize, q_start: u32, q_end: u32) -> (usize, usize) { + let (o0, o1) = self.vk_indel.column(col); + let positions = &self.vk_indel.positions()[o0..o1]; + if positions.is_empty() { + return (o0, o0); + } + let keys = &as_u32(&self.vk_indel.keys)[o0..o1]; + let max_del = self.vk_indel_max_del[[sample, p]]; + let v_ends: Vec = positions + .iter() + .enumerate() + .map(|(i, &pos)| pos + 1 + rvk::deletion_len(keys[i])) + .collect(); + let tree = crate::search::SearchTree::new(positions); + let (s, e) = crate::search::overlap_range(&tree, &v_ends, max_del, q_start, q_end); + (o0 + s, o0 + e) +} +``` + +> Implementation note: `overlap_range` currently lives behind `spine::gather_keys` (`src/spine.rs:48`). Confirm `overlap_range` and `SearchTree` are `pub` in `src/search.rs` (they are used across modules already); if `vk_snp.column`/`.positions()`/`vk_indel.keys` are private to a sibling module, add `pub(crate)` accessors — do not widen further than needed. + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (1 test). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +pixi run prek-install +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add find_ranges search-only query core" +``` + +--- + +### Task 2: `gather_ranges` (tree-free Rust core) + +**Files:** +- Modify: `src/query.rs` +- Test: `tests/test_ranges_split.rs` (extend) + +**Interfaces:** +- Consumes: `RangesBundle` (Task 1), `ContigReader`, `ContigReader::dense_union()`, `spine::merge_keys` (existing, `src/spine.rs:63`), `KeyRef` (existing), `DenseTable::carried(hap, col)` (existing, used in `overlap_batch` at `:548`), `ContigReader::lut_arrays()` (existing, `:260`), `rvk::snp_code_to_key`/`unpack_snp_key_at`/`deletion_len` (existing). +- Produces: + ```rust + pub fn gather_ranges(reader: &ContigReader, rb: &RangesBundle) -> BatchResult; + ``` + Returns the **exact same `BatchResult`** shape `overlap_batch` returns (`vk`, `vk_off`, `dense`, `dense_range`, `dense_present`, `dense_present_off`, `n_regions`, `n_samples`, `ploidy`) — so all downstream numpy conversion and the SVAR2 kernels are unchanged. + +- [ ] **Step 1: Write the failing test (extend `tests/test_ranges_split.rs`)** + +```rust +use genoray_core::query::gather_ranges; + +#[test] +fn test_gather_ranges_reproduces_overlap_batch_field_for_field() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let got = gather_ranges(&reader, &find_ranges(&reader, ®ions, None)); + + assert_eq!(got.n_regions, oracle.n_regions); + assert_eq!(got.n_samples, oracle.n_samples); + assert_eq!(got.ploidy, oracle.ploidy); + assert_eq!(got.vk, oracle.vk); + assert_eq!(got.vk_off, oracle.vk_off); + assert_eq!(got.dense, oracle.dense); + assert_eq!(got.dense_range, oracle.dense_range); + assert_eq!(got.dense_present, oracle.dense_present); + assert_eq!(got.dense_present_off, oracle.dense_present_off); +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_gather_ranges_reproduces_overlap_batch_field_for_field` +Expected: FAIL — `cannot find function gather_ranges`. + +- [ ] **Step 3: Implement `gather_ranges`** + +Adapt the inner triple loop of `overlap_batch` (`src/query.rs:526-567`) to consume `rb` sub-ranges instead of building trees. The **only** change from `overlap_batch`'s body is that `vk_slice`'s two `spine::gather_keys` calls (which each build a `SearchTree`) are replaced by direct slices of the precomputed `rb.vk_snp_range` / `rb.vk_indel_range`, filtered by `carried` + the per-element left-overlap check `q_start < v_end`, then `merge_keys`. The dense-presence loop is copied verbatim (it never built a tree). + +```rust +/// Tree-free gather: replay a `RangesBundle` into the same `BatchResult` that +/// `overlap_batch` produces. Contains NO `SearchTree::new` — the search already +/// happened in `find_ranges`. +pub fn gather_ranges(reader: &ContigReader, rb: &RangesBundle) -> BatchResult { + let ploidy = rb.ploidy; + let n_samples = rb.n_samples; + let n_regions = rb.n_regions; + let hpr = n_samples * ploidy; // haps per region + + let dense = reader.dense_union(); + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_present: Vec = Vec::new(); + let mut dense_present_off: Vec = vec![0]; + + for r in 0..n_regions { + let qs = rb.region_starts[r]; + let (ds, de) = rb.dense_range[r]; + for si in 0..n_samples { + let orig_s = rb.sample_cols[si]; + for p in 0..ploidy { + let col = orig_s * ploidy + p; + let hap = col; + let row = r * hpr + si * ploidy + p; + + // --- var_key gather (no search) --- + let mut snp_run: Vec = Vec::new(); + let (ss, se) = rb.vk_snp_range[row]; + { + let positions = self_positions_snp(reader); + let keys = as_bytes(&reader.vk_snp.keys); + for i in ss..se { + // SNP v_end = pos + 1; left-overlap re-check. + if reader.vk_snp.carried_column_bit(col, i) && qs < positions[i] + 1 { + snp_run.push(KeyRef { + position: positions[i], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, i)), + }); + } + } + } + let mut indel_run: Vec = Vec::new(); + let (is_, ie_) = rb.vk_indel_range[row]; + { + let positions = reader.vk_indel.positions(); + let keys = as_u32(&reader.vk_indel.keys); + for i in is_..ie_ { + let v_end = positions[i] + 1 + rvk::deletion_len(keys[i]); + if reader.vk_indel.carried_column_bit(col, i) && qs < v_end { + indel_run.push(KeyRef { position: positions[i], key: keys[i] }); + } + } + } + let merged = spine::merge_keys(vec![snp_run, indel_run]); + vk.extend_from_slice(&merged); + vk_off.push(vk.len()); + + // --- dense presence bits (verbatim from overlap_batch) --- + let nbits = de - ds; + let bit_base = *dense_present_off.last().unwrap(); + let need_bytes = (bit_base + nbits).div_ceil(8); + if dense_present.len() < need_bytes { + dense_present.resize(need_bytes, 0); + } + for (k, j) in (ds..de).enumerate() { + let (is_indel, dcol) = dense.src[j]; + let carried = if is_indel { + reader.dense_indel.as_ref().expect("indel src implies table").carried(hap, dcol) + } else { + reader.dense_snp.as_ref().expect("snp src implies table").carried(hap, dcol) + }; + if carried && dense.v_ends[j] > qs { + bits::set_bit(&mut dense_present, bit_base + k); + } + } + dense_present_off.push(bit_base + nbits); + } + } + } + + BatchResult { + n_regions, + n_samples, + ploidy, + vk, + vk_off, + dense: dense.refs, + dense_range: rb.dense_range.clone(), + dense_present, + dense_present_off, + } +} +``` + +> Implementation note: the exact per-element `carried` accessor for a **var_key** column (`carried_column_bit`) and the `positions`/`keys` slice accessors are genoray-internal. The existing `vk_slice` (`:296`) reaches them via `spine::gather_keys`'s `carried: impl Fn(usize) -> bool` closure — in the current code that closure is `|_| true` (var_key channel carries every stored key by construction; the presence filter is the dense channel's job). **Verify this**: if `vk_slice` passes `|_| true`, then the var_key gather needs no per-element carried test at all — drop `carried_column_bit` and keep only the `qs < v_end` left-overlap re-check. Let the field-for-field parity test (Step 1) pin the correct behavior; do not guess — match `overlap_batch` byte-for-byte. Replace the `self_positions_snp(reader)` placeholder with the real `reader.vk_snp.positions()` accessor once confirmed `pub(crate)`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (both tests). If the field-for-field test fails, the divergence is in the var_key `carried`/left-overlap handling — reconcile against `vk_slice`/`gather_keys` until byte-identical. + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add gather_ranges tree-free query core" +``` + +--- + +### Task 3: `read_ranges` fused wrapper + search-free assertion + +**Files:** +- Modify: `src/query.rs` +- Test: `tests/test_ranges_split.rs` (extend) + +**Interfaces:** +- Consumes: `find_ranges` (Task 1), `gather_ranges` (Task 2). +- Produces: `pub fn read_ranges(reader: &ContigReader, regions: &[(u32, u32)], samples: Option<&[usize]>) -> BatchResult;` + +- [ ] **Step 1: Write the failing test** + +```rust +use genoray_core::query::read_ranges; + +#[test] +fn test_read_ranges_equals_overlap_batch() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let got = read_ranges(&reader, ®ions, None); + assert_eq!(got.vk, oracle.vk); + assert_eq!(got.vk_off, oracle.vk_off); + assert_eq!(got.dense_present, oracle.dense_present); + assert_eq!(got.dense_present_off, oracle.dense_present_off); + assert_eq!(got.dense_range, oracle.dense_range); +} + +// Subset parity: read_ranges over a sample subset equals the corresponding +// hap-rows of the full overlap_batch. For samples=[1] (original index 1), +// region r's hap rows are r*H + [ploidy .. 2*ploidy) of the full result. +#[test] +fn test_read_ranges_sample_subset_matches_full() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 400u32)]; + + let full = overlap_batch(&reader, ®ions); + let sub = read_ranges(&reader, ®ions, Some(&[1])); + assert_eq!(sub.n_samples, 1); + // hap rows for sample 1 in the full result: h in [1*ploidy, 2*ploidy). + let ploidy = full.ploidy; + for p in 0..ploidy { + let full_h = 1 * ploidy + p; + let sub_h = 0 * ploidy + p; + assert_eq!( + &sub.vk[sub.vk_off[sub_h]..sub.vk_off[sub_h + 1]], + &full.vk[full.vk_off[full_h]..full.vk_off[full_h + 1]], + ); + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_read_ranges` +Expected: FAIL — `cannot find function read_ranges`. + +- [ ] **Step 3: Implement `read_ranges`** + +```rust +/// Fused search+gather: the public/live-query analog of `SparseVar::read_ranges` +/// and the parity oracle for the split. Byte-identical to `overlap_batch` for +/// `samples = None`. +pub fn read_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, +) -> BatchResult { + gather_ranges(reader, &find_ranges(reader, regions, samples)) +} +``` + +Optionally (DRY, keep `overlap_batch` byte-identical): leave `overlap_batch` as-is — do **not** re-route it through the split in this task, to avoid perturbing the existing oracle while the split stabilizes. A follow-up may collapse them once parity is locked. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (all Task 1–3 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add read_ranges fused wrapper + subset parity" +``` + +--- + +### Task 4: PyO3 bindings — `find_ranges` / `gather_ranges` / `read_ranges` on `PyContigReader` + +**Files:** +- Create: `src/py_query_ranges.rs` +- Modify: `src/lib.rs` (add `mod py_query_ranges;`) +- Test: `tests/test_ranges_split.rs` (add a binding round-trip test mirroring `tests/test_batch_raw.rs`) + +**Interfaces:** +- Consumes: `PyContigReader` (existing, `src/py_query.rs:12`), `find_ranges`/`gather_ranges`/`read_ranges` (Tasks 1–3), the numpy helpers `u8_to_pyarray`/`u32_to_i32_pyarray`/`usize_to_i64_pyarray` (existing, `src/py_convert.rs`). +- Produces on `PyContigReader`: + - `read_ranges(regions, samples=None) -> PyDict` — the **same key/dtype contract** as `overlap_batch` (`vk_pos`, `vk_key`, `vk_off`, `dense_pos`, `dense_key`, `dense_range`, `dense_present`, `dense_present_off`, `lut_bytes`, `lut_off`, `n_regions`, `n_samples`, `ploidy`). + - `find_ranges(regions, samples=None, out=None) -> PyDict` — the compact bundle: `dense_range (R,2) i32`, `region_starts (R) i32`, `sample_cols (n_samples) i64`, `vk_snp_range (R*H,2) i64`, `vk_indel_range (R*H,2) i64`, `n_regions`, `n_samples`, `ploidy`. `out` (optional dict of preallocated arrays) receives the ranges in place. + - `gather_ranges(bundle: PyDict, samples=None) -> PyDict` — same output contract as `read_ranges`; `bundle` is a `find_ranges` dict. + +- [ ] **Step 1: Write the failing binding test** + +Add to `tests/test_ranges_split.rs` (uses `Python::with_gil`, mirrors `tests/test_batch_raw.rs`): + +```rust +use genoray_core::py_query::PyContigReader; +use numpy::{PyArray1, PyArrayMethods}; +use pyo3::prelude::*; +use pyo3::types::PyDictMethods; + +#[test] +fn test_py_read_ranges_dict_matches_overlap_batch_dict() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let _reader = synth_reader(&out); + let base = out.to_str().unwrap().to_string(); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + Python::with_gil(|py| { + let pr = PyContigReader::new(&base, "chr1", 2, 2).unwrap(); + let d_ob = pr.overlap_batch(py, regions.clone()).unwrap(); + let d_rr = pr.read_ranges(py, regions.clone(), None).unwrap(); + for k in ["vk_pos", "vk_key", "dense_pos", "dense_key"] { + let a = d_ob.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + let b = d_rr.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + assert_eq!(a.as_slice().unwrap(), b.as_slice().unwrap(), "key {k}"); + } + }); +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_py_read_ranges_dict_matches_overlap_batch_dict` +Expected: FAIL — no method `read_ranges` on `PyContigReader`. + +- [ ] **Step 3: Implement the bindings** + +Create `src/py_query_ranges.rs` (mirror `src/py_query_batch.rs` for the `read_ranges`/`gather_ranges` output dict; add the compact `find_ranges` dict + `out=` streaming): + +```rust +//! SVAR2 search/gather split: numpy-dict bindings on `PyContigReader`. +//! Separate #[pymethods] block (multiple-pymethods) so the M6b overlap_batch +//! binding in py_query_batch.rs is untouched. + +use ndarray::Array2; +use numpy::{PyArray1, PyArray2, PyArrayMethods, ToPyArray}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::py_convert::{u8_to_pyarray, u32_to_i32_pyarray, usize_to_i64_pyarray}; +use crate::py_query::PyContigReader; +use crate::query::{find_ranges, gather_ranges, read_ranges, BatchResult, RangesBundle}; + +fn batch_result_to_dict<'py>( + py: Python<'py>, + reader_lut: (Vec, Vec), + br: &BatchResult, +) -> PyResult> { + // Identical to py_query_batch.rs::overlap_batch's dict assembly. + let vk_pos: Vec = br.vk.iter().map(|k| k.position).collect(); + let vk_key: Vec = br.vk.iter().map(|k| k.key).collect(); + let dense_pos: Vec = br.dense.iter().map(|k| k.position).collect(); + let dense_key: Vec = br.dense.iter().map(|k| k.key).collect(); + let r = br.dense_range.len(); + let mut dr: Vec = Vec::with_capacity(r * 2); + for &(s, e) in &br.dense_range { dr.push(s as i32); dr.push(e as i32); } + let dense_range = Array2::from_shape_vec((r, 2), dr).expect("dense_range").to_pyarray(py); + let (lut_bytes, lut_off_u64) = reader_lut; + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let d = PyDict::new(py); + d.set_item("vk_pos", u32_to_i32_pyarray(py, &vk_pos))?; + d.set_item("vk_key", u32_to_i32_pyarray(py, &vk_key))?; + d.set_item("vk_off", usize_to_i64_pyarray(py, &br.vk_off))?; + d.set_item("dense_pos", u32_to_i32_pyarray(py, &dense_pos))?; + d.set_item("dense_key", u32_to_i32_pyarray(py, &dense_key))?; + d.set_item("dense_range", dense_range)?; + d.set_item("dense_present", u8_to_pyarray(py, &br.dense_present))?; + d.set_item("dense_present_off", usize_to_i64_pyarray(py, &br.dense_present_off))?; + d.set_item("lut_bytes", u8_to_pyarray(py, &lut_bytes))?; + d.set_item("lut_off", PyArray1::from_slice(py, &lut_off))?; + d.set_item("n_regions", br.n_regions)?; + d.set_item("n_samples", br.n_samples)?; + d.set_item("ploidy", br.ploidy)?; + Ok(d) +} + +fn bundle_to_dict<'py>(py: Python<'py>, rb: &RangesBundle) -> PyResult> { + let pairs2 = |v: &[(usize, usize)]| -> Vec { + let mut o = Vec::with_capacity(v.len() * 2); + for &(a, b) in v { o.push(a as i64); o.push(b as i64); } + o + }; + let dr: Vec = rb.dense_range.iter().flat_map(|&(a, b)| [a as i32, b as i32]).collect(); + let d = PyDict::new(py); + d.set_item("dense_range", Array2::from_shape_vec((rb.n_regions, 2), dr).unwrap().to_pyarray(py))?; + d.set_item("region_starts", u32_to_i32_pyarray(py, &rb.region_starts))?; + d.set_item("sample_cols", usize_to_i64_pyarray(py, &rb.sample_cols))?; + let h = rb.n_samples * rb.ploidy; + d.set_item("vk_snp_range", + Array2::from_shape_vec((rb.n_regions * h, 2), pairs2(&rb.vk_snp_range)).unwrap().to_pyarray(py))?; + d.set_item("vk_indel_range", + Array2::from_shape_vec((rb.n_regions * h, 2), pairs2(&rb.vk_indel_range)).unwrap().to_pyarray(py))?; + d.set_item("n_regions", rb.n_regions)?; + d.set_item("n_samples", rb.n_samples)?; + d.set_item("ploidy", rb.ploidy)?; + Ok(d) +} + +fn bundle_from_dict(py: Python<'_>, d: &Bound<'_, PyDict>) -> RangesBundle { + let get_i64 = |k: &str| -> Vec { + d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly().as_slice().unwrap().to_vec() + }; + let get_i32 = |k: &str| -> Vec { + d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly().as_slice().unwrap().to_vec() + }; + let get_i32_2d = |k: &str| -> Vec<(usize, usize)> { + let a = d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + a.as_array().rows().into_iter().map(|r| (r[0] as usize, r[1] as usize)).collect() + }; + let get_i64_2d = |k: &str| -> Vec<(usize, usize)> { + let a = d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + a.as_array().rows().into_iter().map(|r| (r[0] as usize, r[1] as usize)).collect() + }; + let n_regions = d.get_item("n_regions").unwrap().unwrap().extract().unwrap(); + let n_samples = d.get_item("n_samples").unwrap().unwrap().extract().unwrap(); + let ploidy = d.get_item("ploidy").unwrap().unwrap().extract().unwrap(); + RangesBundle { + n_regions, n_samples, ploidy, + region_starts: get_i32("region_starts").into_iter().map(|x| x as u32).collect(), + dense_range: get_i32_2d("dense_range"), + sample_cols: get_i64("sample_cols").into_iter().map(|x| x as usize).collect(), + vk_snp_range: get_i64_2d("vk_snp_range"), + vk_indel_range: get_i64_2d("vk_indel_range"), + } +} + +#[pymethods] +impl PyContigReader { + pub fn read_ranges<'py>(&self, py: Python<'py>, regions: Vec<(u32, u32)>, samples: Option>) -> PyResult> { + let br = read_ranges(&self.inner, ®ions, samples.as_deref()); + batch_result_to_dict(py, self.inner.lut_arrays(), &br) + } + pub fn find_ranges<'py>(&self, py: Python<'py>, regions: Vec<(u32, u32)>, samples: Option>) -> PyResult> { + let rb = find_ranges(&self.inner, ®ions, samples.as_deref()); + bundle_to_dict(py, &rb) + } + pub fn gather_ranges<'py>(&self, py: Python<'py>, bundle: Bound<'py, PyDict>) -> PyResult> { + let rb = bundle_from_dict(py, &bundle); + let br = gather_ranges(&self.inner, &rb); + batch_result_to_dict(py, self.inner.lut_arrays(), &br) + } +} +``` + +Add `mod py_query_ranges;` to `src/lib.rs` (next to `mod py_query_batch;`). + +> Implementation note: the `out=` streaming variant is deferred to the Python layer (Task 5) — `find_ranges` returns freshly-allocated numpy arrays here; the Python `SparseVar2.find_ranges(..., out=...)` copies them into a caller memmap. This keeps the Rust binding simple and matches how gvl actually uses it (write once). If profiling later shows the copy matters, push `out=` into Rust then. + +- [ ] **Step 4: Rebuild the extension and run tests** + +Run: +```bash +cd /carter/users/dlaub/projects/genoray +pixi run cargo test --test test_ranges_split +pixi run maturin develop --release +``` +Expected: cargo PASS; maturin builds cleanly. + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/py_query_ranges.rs src/lib.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): PyContigReader find/gather/read_ranges bindings" +``` + +--- + +### Task 5: Python `SparseVar2` methods with `samples=` and `out=` + +**Files:** +- Modify: `python/genoray/_svar2_batch.py` +- Test: `tests/test_svar2_ranges.py` (create) + +**Interfaces:** +- Consumes: `SparseVar2._readers[contig]` (existing `PyContigReader` per contig, `python/genoray/_svar2_batch.py:25`), `SparseVar2.samples`/`.ploidy`/`.n_samples` (existing, `_svar2.py:44-46`), the new `PyContigReader.read_ranges`/`find_ranges`/`gather_ranges` (Task 4). +- Produces on `SparseVar2` (via `_BatchQueryMixin`): + ```python + def find_ranges(self, contig, starts, ends, samples=None, out=None) -> dict[str, np.ndarray] + def gather_ranges(self, contig, ranges, samples=None) -> dict[str, np.ndarray] + def read_ranges(self, contig, starts, ends, samples=None) -> dict[str, np.ndarray] + ``` + `ranges` is a `find_ranges` dict. `read_ranges`'s output dict is the **exact same contract** as `overlap_batch`. + +- [ ] **Step 1: Write the failing pytest** + +Create `tests/test_svar2_ranges.py` (reuse the `svar2_store` fixture that `tests/test_svar2_batch.py` uses): + +```python +import numpy as np +from genoray import SparseVar2 + + +def _assert_dicts_equal(a: dict, b: dict, keys): + for k in keys: + np.testing.assert_array_equal(np.asarray(a[k]), np.asarray(b[k]), err_msg=k) + + +PAYLOAD_KEYS = [ + "vk_pos", "vk_key", "vk_off", "dense_pos", "dense_key", "dense_range", + "dense_present", "dense_present_off", "lut_bytes", "lut_off", +] + + +def test_read_ranges_matches_overlap_batch(svar2_store): + sv = SparseVar2(svar2_store) + starts, ends = [0, 5], [40, 20] + ob = sv.overlap_batch("chr1", list(zip(starts, ends))) + rr = sv.read_ranges("chr1", starts, ends) + _assert_dicts_equal(ob, rr, PAYLOAD_KEYS) + assert int(rr["n_regions"]) == 2 + + +def test_gather_of_find_matches_read(svar2_store): + sv = SparseVar2(svar2_store) + starts, ends = [0], [40] + ranges = sv.find_ranges("chr1", starts, ends) + gathered = sv.gather_ranges("chr1", ranges) + read = sv.read_ranges("chr1", starts, ends) + _assert_dicts_equal(read, gathered, PAYLOAD_KEYS) + + +def test_read_ranges_sample_subset(svar2_store): + sv = SparseVar2(svar2_store) + full = sv.overlap_batch("chr1", [(0, 40)]) + sub = sv.read_ranges("chr1", [0], [40], samples=[sv.samples[1]]) + assert int(sub["n_samples"]) == 1 + ploidy = sv.ploidy + for p in range(ploidy): + fh = 1 * ploidy + p + sh = 0 * ploidy + p + np.testing.assert_array_equal( + full["vk_pos"][full["vk_off"][fh]:full["vk_off"][fh + 1]], + sub["vk_pos"][sub["vk_off"][sh]:sub["vk_off"][sh + 1]], + ) + + +def test_find_ranges_out_streaming(svar2_store): + sv = SparseVar2(svar2_store) + ranges = sv.find_ranges("chr1", [0], [40]) + # Pre-allocate matching-shape buffers and stream into them. + out = {k: np.empty_like(np.asarray(ranges[k])) for k in + ("dense_range", "region_starts", "sample_cols", "vk_snp_range", "vk_indel_range")} + ranges2 = sv.find_ranges("chr1", [0], [40], out=out) + for k in out: + np.testing.assert_array_equal(np.asarray(ranges2[k]), np.asarray(ranges[k])) + # out= wrote in place: returned array shares the buffer. + assert np.asarray(ranges2[k]).base is out[k] or ranges2[k] is out[k] +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run pytest tests/test_svar2_ranges.py -q` +Expected: FAIL — `SparseVar2` has no attribute `read_ranges`. + +- [ ] **Step 3: Implement the Python methods** + +Add to `_BatchQueryMixin` in `python/genoray/_svar2_batch.py`. Resolve `samples=` names → original integer indices via `self.samples.index(...)`; validate membership. + +```python + def _sample_idxs(self, samples): + if samples is None: + return None + idxs = [] + for s in np.atleast_1d(np.asarray(samples)).tolist(): + if s not in self.samples: + raise ValueError(f"Sample {s!r} not found in the dataset.") + idxs.append(self.samples.index(s)) + return idxs + + def read_ranges(self, contig, starts, ends, samples=None): + """Fused search+gather query (byte-identical to ``overlap_batch`` for + ``samples=None``). See ``overlap_batch`` for the returned dict contract.""" + reg = self._regions(starts, ends) + return self._readers[contig].read_ranges(reg, self._sample_idxs(samples)) + + def find_ranges(self, contig, starts, ends, samples=None, out=None): + """Search-only step: returns the compact ranges bundle to be replayed by + ``gather_ranges``. When ``out`` is a dict of preallocated arrays keyed by + the bundle field names, the ranges are written into it in place.""" + reg = self._regions(starts, ends) + d = self._readers[contig].find_ranges(reg, self._sample_idxs(samples)) + if out is not None: + for k, buf in out.items(): + np.asarray(buf)[...] = np.asarray(d[k]) + d[k] = buf + return d + + def gather_ranges(self, contig, ranges, samples=None): + """Tree-free gather step: replay a ``find_ranges`` bundle into the full + ``overlap_batch`` payload dict. ``samples`` is accepted for symmetry but + the subset is already fixed by the bundle; passing a different subset is + a ValueError.""" + return self._readers[contig].gather_ranges(ranges) +``` + +Add the `_regions` helper (shared with `overlap_batch`, which currently inlines `[(int(s), int(e)) for s, e in regions]`): + +```python + @staticmethod + def _regions(starts, ends): + s = np.atleast_1d(np.asarray(starts)) + e = np.atleast_1d(np.asarray(ends)) + return [(int(a), int(b)) for a, b in zip(s, e)] +``` + +Import numpy at module top (`import numpy as np`) — currently only imported under `TYPE_CHECKING`; move it to a real import since the methods use it at runtime. + +- [ ] **Step 4: Rebuild + run** + +Run: +```bash +cd /carter/users/dlaub/projects/genoray +pixi run maturin develop --release +pixi run pytest tests/test_svar2_ranges.py -q +``` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add python/genoray/_svar2_batch.py tests/test_svar2_ranges.py +rtk git commit -m "feat(svar2): SparseVar2 find/gather/read_ranges with samples= and out=" +``` + +--- + +### Task 6: Reconstruction parity vs the `decode` oracle + +**Files:** +- Test: `tests/test_svar2_ranges.py` (extend) + +**Interfaces:** +- Consumes: `SparseVar2.decode(...)` (existing oracle, `_svar2.py`), the M6b→numpy contract, and the split methods (Task 5). This task adds no production code — it hardens the byte-identical contract end-to-end so the gvl plan can depend on it. + +- [ ] **Step 1: Write the failing/again-green oracle test** + +Extend `tests/test_svar2_ranges.py`. Mirror whatever reconstruction check `tests/test_svar2_batch.py` / `tests/test_svar2_decode.py` already use against `decode`; assert the `read_ranges` and `gather_ranges(find_ranges)` payloads reconstruct to the identical `decode` output. If `tests/test_svar2_decode.py` exposes a helper that turns an `overlap_batch` dict into per-hap calls, reuse it verbatim on the three payloads. + +```python +def test_split_reconstructs_like_decode_oracle(svar2_store): + from tests.test_svar2_decode import decode_from_payload # reuse existing helper + sv = SparseVar2(svar2_store) + starts, ends = [0], [40] + ob = sv.overlap_batch("chr1", list(zip(starts, ends))) + rr = sv.read_ranges("chr1", starts, ends) + gr = sv.gather_ranges("chr1", sv.find_ranges("chr1", starts, ends)) + oracle = sv.decode("chr1", starts[0], ends[0]) # adapt to real decode signature + for payload in (ob, rr, gr): + assert decode_from_payload(sv, payload) == oracle +``` + +> If no reusable `decode_from_payload` helper exists, this test degrades to the field-for-field payload equality already covered in Task 5 plus the Rust `test_gather_ranges_reproduces_overlap_batch_field_for_field` — in that case, delete this task's test and rely on those, since `overlap_batch` is itself the decode-validated reference (per `tests/test_batch_raw.rs`'s header comment). **Check `tests/test_svar2_decode.py` first**; do not invent a helper name. + +- [ ] **Step 2: Run** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run pytest tests/test_svar2_ranges.py tests/test_svar2_batch.py tests/test_svar2_decode.py -q` +Expected: PASS. + +- [ ] **Step 3: Full regression** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test && pixi run test` +Expected: all green (no existing test regressed by the additive methods). + +- [ ] **Step 4: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add tests/test_svar2_ranges.py +rtk git commit -m "test(svar2): search/gather split reconstructs like decode oracle" +``` + +--- + +### Task 7: Docs, roadmap, and wheel release + +**Files:** +- Modify: `CHANGELOG.md` (or let commitizen generate it) +- Modify: genoray roadmap doc (whichever tracks SVAR2 / M6b — grep `docs/` for `overlap_batch` / `M6b`) +- Modify: `python/genoray/_svar2.py` / `_svar2_batch.py` docstrings (already added in Task 5) + +**Interfaces:** +- Produces: a released genoray wheel/version that the gvl plan pins. + +- [ ] **Step 1: Update the roadmap** + +Find the genoray SVAR2 roadmap section (`rtk grep "overlap_batch" docs/` and `rtk grep "M6b" docs/`) and record: the search/gather split shipped (`find_ranges`/`gather_ranges`/`read_ranges` with `samples=`, `out=` on `find_ranges`), and note the **open question** the spec flags — whether to convert the read path to fully read-bound (dense union caching) — as a follow-up. + +- [ ] **Step 2: Verify the public API doc / API stubs** + +If genoray publishes a Sphinx `api.md`/autodoc or `.pyi` stubs for `SparseVar2`, add `find_ranges`/`gather_ranges`/`read_ranges`. Run `cd /carter/users/dlaub/projects/genoray && pixi run -e doc doc` if a docs env exists and confirm it builds. + +- [ ] **Step 3: Bump version + build the wheel** + +```bash +cd /carter/users/dlaub/projects/genoray +pixi run bump-dry # preview the version bump +# then perform the real bump per genoray's release process (commitizen) +pixi run maturin build --release +``` +Record the released version string — the gvl plan's Global Constraints pin `genoray >= `. + +- [ ] **Step 4: Final commit / tag** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add -A +rtk git commit -m "docs(svar2): record find/gather/read_ranges split + roadmap follow-up" +``` + +--- + +## Self-Review + +- **Spec coverage (Component A):** `find_ranges` (Task 1), `gather_ranges` (Task 2), `read_ranges` (Task 3), `samples=` on all three (Tasks 1/3/5), `out=` on `find_ranges` (Tasks 4-note/5), PyO3 bindings (Task 4), Python surface (Task 5), byte-identical parity vs `overlap_batch` and `decode` (Tasks 2/3/6), docs+roadmap+wheel (Task 7). ✅ +- **Deferred correctly:** dense-union caching / fully read-bound conversion is logged as a genoray follow-up (Task 7), matching the spec's open question. The var_key `carried` semantics are pinned by the field-for-field parity test rather than guessed (Tasks 2-3). +- **Type consistency:** `RangesBundle` fields, `H = n_samples * ploidy`, and the row index `r*H + selected_s*ploidy + p` are used identically across `find_ranges`, `gather_ranges`, and both binding dicts. diff --git a/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md b/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md new file mode 100644 index 00000000..d200cb9c --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md @@ -0,0 +1,941 @@ +# SVAR2 gvl Dataset Wiring — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the SVAR2 format into the gvl `Dataset` the way SVAR1 is — cache genoray's interval-search result at `gvl.write` time and replay it at read time via `gather_ranges` + the existing SVAR2 kernels — so SVAR2 haplotype/track reads stop paying a per-query `SearchTree::build`. + +**Architecture:** Mirror the SVAR1 write/read wiring. `_write_from_svar2` streams `SparseVar2.find_ranges(..., out=...)` into a compact per-region/per-hap ranges cache under `genotypes/`, plus a `Svar2Link` back-reference in `metadata.json`. On read, a new `HapsSvar2` reconstructor loads the cached ranges for the requested `(region, sample)` block, calls `SparseVar2.gather_ranges` (tree-free), and feeds the payload to `reconstruct_haplotypes_from_svar2` / `shift_and_realign_tracks_from_svar2`. A source discriminant (`svar2_link` present) selects `HapsSvar2` over the SVAR1 `Haps`. + +**Tech Stack:** Python 3.10+, numpy, polars, pydantic, seqpro `Ragged`, genoray `SparseVar2` (from the genoray split plan), pixi, maturin, pytest. + +**Repo:** `/carter/users/dlaub/projects/GenVarLoader` worktree `svar2-m6b-kernel`. **Depends on** the shipped genoray wheel from `2026-07-03-svar2-genoray-search-gather-split.md` (`find_ranges`/`gather_ranges`/`read_ranges` with `samples=`/`out=`). + +## Global Constraints + +- **Depends on genoray >= ``.** Before starting, confirm `pixi run -e dev python -c "from genoray import SparseVar2; SparseVar2.find_ranges; SparseVar2.gather_ranges"` succeeds; if not, bump the genoray pin in `pixi.toml` / `pyproject.toml` and `pixi run -e dev install`. +- **Byte-identical parity contract** (verbatim from spec): cached-path reconstruct ≡ live `read_ranges`/`overlap_batch` reconstruct ≡ `decode` oracle, on the M6b matrix (SNP/INS/DEL × samples × ploids) + real chr21 germline & somatic stores. Track re-alignment matched the same way. +- **Additive:** the SVAR1 path is **byte-unchanged**. The full SVAR1 regression suite stays green (`pixi run -e dev pytest tests -q`; `pixi run -e dev cargo-test` for kernels). Follows the rust-migration byte-identical parity contract and the numba-oracle-bug policy (if the cached path and a numba oracle disagree, check whether numba is the buggy one before "fixing" the new path). +- **Scope:** haplotypes + tracks **only**. SVAR2 `variants` and `annotated` output modes are out of scope (raise a clear `NotImplementedError`); the same cache extends to them later. +- **REBUILD RUST BEFORE PYTHON TESTS:** these changes are pure-Python (no `src/` edits), so `maturin develop` is *not* required for this plan — but if any parity test imports the SVAR2 kernels and you touched `src/`, run `pixi run -e dev maturin develop --release` first. +- **Docs gates:** any public-API change updates `skills/genvarloader/SKILL.md`; `api.md` stays in sync with `__all__`; user-facing docs audited (`README.md`, `docs/source/{api,write,format,faq}.md`). See CLAUDE.md's skill-maintenance + docs-audit rules. +- **Conventional commits** (commitizen). Ensure prek hooks installed before committing. + +--- + +## File Structure + +- `python/genvarloader/_dataset/_svar2_link.py` — **new**. `Svar2Fingerprint`, `Svar2Link` pydantic models, `_resolve_svar2`, `_verify_fingerprint2`. Mirrors `_svar_link.py`. +- `python/genvarloader/_dataset/_write.py` — **modify**. Add `.svar2` detection in the variant-source dispatch (~`:225`), a `SparseVar2` branch in the genotype-writing dispatch (~`:325`), and a new `_write_from_svar2(...)` (mirrors `_write_from_svar` at `:961`). Add `svar2_link` to the `Metadata` model (`:86`). +- `python/genvarloader/_dataset/_svar2_source.py` — **modify**. Add a cache-load + `gather_ranges` path; keep the live `overlap_batch` path as the parity oracle. +- `python/genvarloader/_dataset/_haps.py` — **modify**. Add `HapsSvar2` reconstructor (haplotypes-only, `Reconstructor[RaggedSeqs]`) that loads cached ranges + gathers + calls the SVAR2 kernels. +- `python/genvarloader/_dataset/_reconstruct.py` — **modify**. Add `HapsSvar2Tracks` and route `HapsSvar2` through `_build_reconstructor`. +- `python/genvarloader/_dataset/_open.py` — **modify**. `_build_seqs` constructs `HapsSvar2` when `metadata.svar2_link` is present. +- `python/genvarloader/_dataset/_migrate.py` — **verify** `svar2_link` is tolerated (additive metadata key). +- Tests: `tests/dataset/test_write_svar2.py`, `tests/dataset/test_svar2_dataset.py`, `tests/unit/dataset/test_svar2_link.py` — **new**. + +--- + +### Task 1: `Svar2Link` model + resolution/fingerprint + +**Files:** +- Create: `python/genvarloader/_dataset/_svar2_link.py` +- Test: `tests/unit/dataset/test_svar2_link.py` + +**Interfaces:** +- Produces: + ```python + class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + def _resolve_svar2(gvl_path: Path, link: Svar2Link | None, override: Path | str | None) -> Path + def _verify_fingerprint2(svar2_path: Path, link: Svar2Link | None) -> None + ``` +- Consumes: nothing gvl-internal beyond the same pattern as `_svar_link.py`. Fingerprint identity for `.svar2` = `n_variants` (from the SparseVar2 index) + a byte count of a canonical store file (see Step 3). + +- [ ] **Step 1: Write the failing test** + +Create `tests/unit/dataset/test_svar2_link.py`: + +```python +from pathlib import Path + +import pytest + +from genvarloader._dataset._svar2_link import ( + Svar2Fingerprint, + Svar2Link, + _resolve_svar2, +) + + +def _mk_svar2_dir(tmp_path: Path) -> Path: + d = tmp_path / "cohort.svar2" + d.mkdir() + (d / "index.arrow").write_bytes(b"stub") + return d + + +def test_resolve_prefers_override(tmp_path): + d = _mk_svar2_dir(tmp_path) + other = tmp_path / "other.svar2" + other.mkdir() + link = Svar2Link( + relative_path="cohort.svar2", + absolute_path=str(d), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + gvl = tmp_path # pretend the gvl dataset lives here + assert _resolve_svar2(gvl, link, other) == other + + +def test_resolve_falls_back_to_relative_then_absolute(tmp_path): + d = _mk_svar2_dir(tmp_path) + gvl = tmp_path / "ds.gvl" + gvl.mkdir() + import os + rel = os.path.relpath(d, start=gvl).replace(os.sep, "/") + link = Svar2Link( + relative_path=rel, absolute_path=str(d), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + assert _resolve_svar2(gvl, link, None) == d + + +def test_resolve_raises_when_unfindable(tmp_path): + gvl = tmp_path / "ds.gvl" + gvl.mkdir() + link = Svar2Link( + relative_path="missing.svar2", absolute_path=str(tmp_path / "missing.svar2"), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + with pytest.raises(FileNotFoundError): + _resolve_svar2(gvl, link, None) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: FAIL — `No module named ..._svar2_link`. + +- [ ] **Step 3: Implement `_svar2_link.py`** + +Copy `_svar_link.py`'s structure. Fingerprint on a **stable** `.svar2` identity: `n_variants` from the SparseVar2 index (`pl.scan_ipc(/index.arrow).select(pl.len())`) and the byte size of a canonical store file. **Verify the actual on-disk `.svar2` layout first** — `rtk ls ` (e.g. under `tmp/svar2_mvp/`) to confirm the index filename (`index.arrow`?) and pick one always-present store file for `store_bytes` (e.g. the largest `*.npy`/packed buffer). Do not assume `variant_idxs.npy` — that is SVAR1. + +```python +"""Resolution and integrity for the GVL dataset → SVAR2 back-reference. + +Mirrors _svar_link.py. SVAR2 fingerprint identity = n_variants (from the +SparseVar2 index) + byte count of a canonical store file. +""" +from __future__ import annotations + +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +# The canonical store file used for the byte-count fingerprint. VERIFY against a +# real .svar2 directory and update if the layout differs. +_STORE_FILE = "index.arrow" + + +def _resolve_svar2( + gvl_path: Path, link: Svar2Link | None, override: Path | str | None +) -> Path: + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_fingerprint2(svar2_path: Path, link: Svar2Link | None) -> None: + if link is None: + return + store = svar2_path / _STORE_FILE + if not store.exists(): + raise FileNotFoundError( + f"Expected {store}; resolved svar2 is malformed." + ) + import polars as pl + + n_variants_observed = ( + pl.scan_ipc(svar2_path / "index.arrow").select(pl.len()).collect().item() + ) + observed_bytes = store.stat().st_size + exp = link.fingerprint + mismatches: list[str] = [] + if n_variants_observed != exp.n_variants: + mismatches.append( + f"n_variants: expected {exp.n_variants}, observed {n_variants_observed}" + ) + if observed_bytes != exp.store_bytes: + mismatches.append( + f"store_bytes: expected {exp.store_bytes}, observed {observed_bytes}" + ) + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +rtk git add python/genvarloader/_dataset/_svar2_link.py tests/unit/dataset/test_svar2_link.py +rtk git commit -m "feat(svar2): Svar2Link resolution + fingerprint" +``` + +--- + +### Task 2: `Metadata.svar2_link` field + migration tolerance + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` (`Metadata` model, `:86-95`) +- Test: `tests/unit/dataset/test_svar2_link.py` (extend) + +**Interfaces:** +- Consumes: `Svar2Link` (Task 1). +- Produces: `Metadata.svar2_link: Svar2Link | None = None`; unchanged datasets (no `svar2_link` key) still validate; `_check_dataset_format_version` and `_migrate.migrate` tolerate the additive key. + +- [ ] **Step 1: Write the failing test** + +Add to `tests/unit/dataset/test_svar2_link.py`: + +```python +def test_metadata_roundtrips_svar2_link(): + from genvarloader._dataset._write import Metadata + from genvarloader._dataset._svar2_link import Svar2Fingerprint, Svar2Link + + link = Svar2Link( + relative_path="c.svar2", absolute_path="/abs/c.svar2", + fingerprint=Svar2Fingerprint(n_variants=5, store_bytes=99), + ) + m = Metadata( + contigs=["chr1"], samples=["s0"], ploidy=2, n_regions=1, + svar2_link=link, + ) + m2 = Metadata.model_validate_json(m.model_dump_json()) + assert m2.svar2_link is not None + assert m2.svar2_link.fingerprint.n_variants == 5 + + +def test_metadata_without_svar2_link_still_valid(): + from genvarloader._dataset._write import Metadata + + m = Metadata(contigs=["chr1"], samples=["s0"], ploidy=2, n_regions=1) + assert m.svar2_link is None +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -k metadata -q` +Expected: FAIL — `Metadata` has no field `svar2_link` (unexpected-keyword / validation error). + +- [ ] **Step 3: Add the field** + +In `python/genvarloader/_dataset/_write.py`, import `Svar2Link` next to `from ._svar_link import SvarLink` and add the field to `Metadata` (after `svar_link`, `:94`): + +```python +from ._svar2_link import Svar2Link # noqa: E402 (near the SvarLink import) +... + svar_link: SvarLink | None = None + svar2_link: Svar2Link | None = None +``` + +- [ ] **Step 4: Run to verify it passes, then confirm migration tolerance** + +Run: +```bash +pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q +pixi run -e dev pytest tests/unit/dataset -k migrate -q # existing migration tests still green +``` +Expected: PASS. `_migrate.migrate` reads `metadata.json` as raw JSON and only touches `format_version`, so the additive `svar2_link` key passes through untouched — the migration tests confirm no regression. No `_migrate.py` code change needed; note this in the commit. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_write.py tests/unit/dataset/test_svar2_link.py +rtk git commit -m "feat(svar2): additive svar2_link metadata field" +``` + +--- + +### Task 3: `_write_from_svar2` + write dispatch + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` +- Test: `tests/dataset/test_write_svar2.py` (create) + +**Interfaces:** +- Consumes: `SparseVar2` (genoray), `SparseVar2.find_ranges(contig, starts, ends, samples=, out=)` (genoray split plan), `Svar2Link`/`Svar2Fingerprint` (Task 1), the existing `_reject_unsupported_variants` (`_write.py`), `atomic_dir`/`_prep_bed`/`_write_regions` (existing write pipeline). +- Produces: + ```python + def _write_from_svar2( + path: Path, bed: pl.DataFrame, svar2: "SparseVar2", + samples: list[str], extend_to_length: bool, + ) -> tuple[pl.DataFrame, Svar2Link] + ``` + Writes cache memmaps + `svar2_meta.json` under `/genotypes/`. The cache layout (all int arrays, region-ordered to match `regions.npy`): + - `svar2_dense_range.npy` int32 `(R, 2)` + - `svar2_region_starts.npy` int32 `(R,)` + - `svar2_vk_snp_range.npy` int64 `(R, S, P, 2)` + - `svar2_vk_indel_range.npy` int64 `(R, S, P, 2)` + - `svar2_meta.json` — shapes/dtypes + `sample_cols` + `n_samples`/`ploidy` + +- [ ] **Step 1: Write the failing test** + +Create `tests/dataset/test_write_svar2.py`. Build a small `.svar2` (reuse whatever fixture/helper the existing `tests/test_svar2_reconstruct.py` uses to synthesize a `SparseVar2` store; grep it) and assert `gvl.write` produces the cache + link. + +```python +import json +from pathlib import Path + +import numpy as np +import polars as pl + +import genvarloader as gvl + + +def test_write_svar2_produces_ranges_cache(svar2_store, tmp_path): + # svar2_store: path to a small .svar2 directory (shared fixture; see conftest). + from genoray import SparseVar2 + + sv = SparseVar2(svar2_store) + bed = pl.DataFrame( + {"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]} + ) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + + geno = out / "genotypes" + assert (geno / "svar2_meta.json").exists() + meta = json.loads((geno / "svar2_meta.json").read_text()) + R, S, P = 1, sv.n_samples, sv.ploidy + dr = np.load(geno / "svar2_dense_range.npy") + assert dr.shape == (R, 2) + vks = np.load(geno / "svar2_vk_snp_range.npy") + assert vks.shape == (R, S, P, 2) + + ds_meta = json.loads((out / "metadata.json").read_text()) + assert ds_meta["svar2_link"] is not None + assert ds_meta["ploidy"] == P +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: FAIL — `write` does not recognize `.svar2` (raises the "unrecognized file extension" ValueError, or SparseVar2 branch missing). + +- [ ] **Step 3: Implement detection + dispatch + `_write_from_svar2`** + +3a. Import `SparseVar2` at the top of `_write.py` (next to `from genoray import ..., SparseVar`). + +3b. In the variant-source dispatch (`_write.py:225`, the `elif variants.is_dir() and variants.suffix == ".svar"` branch), add **before** it: + +```python + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) +``` + +3c. In the genotype-writing dispatch (`_write.py:325`, the `elif isinstance(variants, SparseVar):` branch), add a sibling branch: + +```python + elif isinstance(variants, SparseVar2): + from ._svar2_link import Svar2Link # local import ok + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link.model_dump() +``` + +`metadata["ploidy"] = variants.ploidy` at `:330` already runs for any `variants` (SparseVar2 exposes `.ploidy`). Confirm `SparseVar2.available_samples` exists (used at `:233`); if the attribute is `.samples`, add `available_samples` as an alias or special-case the `available_samples` assignment for `SparseVar2`. + +3d. Add `_write_from_svar2` (mirror `_write_from_svar` at `:961`; partition bed by contig, stream `find_ranges` into the cache memmaps): + +```python +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: "SparseVar2", + samples: list[str], + extend_to_length: bool, +) -> tuple[pl.DataFrame, "Svar2Link"]: + import json + import os + + from ._svar2_link import Svar2Fingerprint, Svar2Link + + _reject_unsupported_variants(svar2.index, "SVAR2") # verify svar2.index exists + + out_dir = path / "genotypes" + out_dir.mkdir(parents=True, exist_ok=True) + + R = bed.height + S = len(samples) + P = svar2.ploidy + + dense_range = np.memmap(out_dir / "svar2_dense_range.npy", np.int32, "w+", shape=(R, 2)) + region_starts = np.memmap(out_dir / "svar2_region_starts.npy", np.int32, "w+", shape=(R,)) + vk_snp = np.memmap(out_dir / "svar2_vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap(out_dir / "svar2_vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + + sample_cols: list[int] | None = None + contig_offset = 0 + for (c,), df in bed.partition_by("chrom", as_dict=True, maintain_order=True).items(): + c = cast(str, c) + rc = df.height + rows = slice(contig_offset, contig_offset + rc) + # find_ranges returns a dict bundle; stream into the cache slices via out=. + out = { + "dense_range": dense_range[rows], + "region_starts": region_starts[rows], + "sample_cols": np.empty(S, np.int64), + "vk_snp_range": vk_snp[rows].reshape(rc * S * P, 2), + "vk_indel_range": vk_indel[rows].reshape(rc * S * P, 2), + } + bundle = svar2.find_ranges( + c, df["chromStart"], df["chromEnd"], samples=samples, out=out + ) + if sample_cols is None: + sample_cols = np.asarray(bundle["sample_cols"], np.int64).tolist() + contig_offset += rc + + for m in (dense_range, region_starts, vk_snp, vk_indel): + m.flush() + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "n_regions": R, "n_samples": S, "ploidy": P, + "sample_cols": sample_cols, + "dense_range": {"shape": [R, 2], "dtype": " Implementation notes: (1) `find_ranges`'s `out=` writes into the provided arrays; passing memmap **slices** streams straight to disk. Confirm the genoray `out=` copy handles the `(rc*S*P, 2)` reshape of a memmap slice (it is C-contiguous per contig-block since the outer axis is region). If a slice is non-contiguous, allocate a per-contig scratch array, `find_ranges(..., out=scratch)`, then assign into the memmap slice. (2) `_reject_unsupported_variants(svar2.index, ...)` — verify `SparseVar2` exposes `.index` (a polars frame with the same reject-able columns); if the schema differs, adapt or skip with a comment (upstream `-V other,bnd` already filters symbolic/breakend ALTs per the spec). (3) `svar2.path` — confirm the attribute name (`SparseVar2.__init__` stores `self.path`). + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +rtk git commit -m "feat(svar2): write dispatch + _write_from_svar2 ranges cache" +``` + +--- + +### Task 4: `HapsSvar2` reconstructor — cached ranges + gather + kernel + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_source.py` (add cache-load + `gather_ranges` path) +- Modify: `python/genvarloader/_dataset/_haps.py` (add `HapsSvar2`) +- Test: `tests/dataset/test_svar2_dataset.py` (create) + +**Interfaces:** +- Consumes: the cache memmaps + `svar2_meta.json` (Task 3), `SparseVar2.gather_ranges(contig, bundle)` (genoray split plan), `SparseVar2Source.reconstruct`/`realign_tracks` marshalling (existing, `_svar2_source.py`), `Reference` (for `ref_`/`ref_offsets`/`pad_char`), the `Reconstructor[_H]` protocol (`_protocol.py`). +- Produces: + ```python + @dataclass(slots=True) + class HapsSvar2(Reconstructor[RaggedSeqs]): + path: Path + reference: Reference + svar2: "SparseVar2" + contigs: list[str] + samples: list[str] + ploidy: int + # cache memmaps + sample_cols loaded in from_path + def __call__(self, idx, r_idx, regions, output_length, jitter, rng, + deterministic, splice_plan=None, flat=False, to_rc=None) -> RaggedSeqs + @classmethod + def from_path(cls, path, reference, contigs, samples, ploidy, svar2_link, svar2_override) -> "HapsSvar2" + def gather_block(self, contig, region_rows, sample_slot_idxs) -> dict # bundle -> gather_ranges payload + ``` + +- [ ] **Step 1: Write the failing test (cached ≡ live oracle)** + +Create `tests/dataset/test_svar2_dataset.py`. Oracle = the **live** `SparseVar2Source` path (already decode-validated); assert the cached `HapsSvar2` reconstructs byte-identically. + +```python +from pathlib import Path + +import numpy as np +import polars as pl + +import genvarloader as gvl + + +def test_svar2_dataset_haps_match_live_source(svar2_store, reference_fasta, tmp_path): + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + + sv = SparseVar2(svar2_store) + regions = [(0, 40)] + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes") + cached = ds[0, :] # (S, P, ~L) ragged bytes for region 0, all samples + + # Live oracle via the adapter (contig ref bytes + offsets from the fasta). + ref_bytes, ref_off, pad = _load_contig_ref(reference_fasta, "chr1") + live = SparseVar2Source(sv).reconstruct( + "chr1", regions, ref_bytes, ref_off, pad, output_length=-1 + ) + # Compare byte-for-byte over every (sample, ploid). + np.testing.assert_array_equal(cached.to_packed().data, live.to_packed().data) + np.testing.assert_array_equal( + np.asarray(cached.offsets), np.asarray(live.offsets) + ) +``` + +Add the `_load_contig_ref` helper + `reference_fasta`/`svar2_store` fixtures to `tests/dataset/conftest.py` if absent (mirror the fixtures in `tests/test_svar2_reconstruct.py`). + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: FAIL — `Dataset.open` builds a plain `Haps` (no svar2 dispatch yet) or errors on the missing SVAR1 `svar_meta.json`/`offsets.npy`. + +- [ ] **Step 3a: Add the cache-load + gather path to `SparseVar2Source`** + +In `_svar2_source.py`, add a classmethod-free helper that builds a `gather_ranges` bundle dict from cached memmap slices and calls `self.svar2.gather_ranges`. Refactor `_query` so its `d` (payload dict) can come from **either** live `overlap_batch` (kept as oracle) or `gather_ranges(bundle)`: + +```python + def _query_cached(self, contig, regions, dense_range, region_starts, + vk_snp_range, vk_indel_range, sample_cols): + """Build a find_ranges bundle from cached slices and gather it (tree-free). + Shapes: dense_range (R,2), region_starts (R,), vk_*_range (R,S,P,2).""" + R = dense_range.shape[0] + S = vk_snp_range.shape[1] + P = vk_snp_range.shape[2] + bundle = { + "dense_range": np.ascontiguousarray(dense_range, np.int32), + "region_starts": np.ascontiguousarray(region_starts, np.int32), + "sample_cols": np.ascontiguousarray(sample_cols, np.int64), + "vk_snp_range": np.ascontiguousarray(vk_snp_range.reshape(R * S * P, 2), np.int64), + "vk_indel_range": np.ascontiguousarray(vk_indel_range.reshape(R * S * P, 2), np.int64), + "n_regions": R, "n_samples": S, "ploidy": P, + } + d = self.svar2.gather_ranges(contig, bundle) + reg = np.asarray(regions, np.int32).reshape(R, 2) + reg_rs = np.repeat(reg, S, axis=0) + regions_gvl = np.zeros((R * S, 3), np.int32) + regions_gvl[:, 1:] = reg_rs + dense_range_gvl = np.ascontiguousarray( + np.repeat(np.asarray(d["dense_range"], np.int32), S, axis=0), np.int32 + ) + return d, R, S, P, regions_gvl, dense_range_gvl +``` + +Extract the kernel-call bodies of `reconstruct`/`realign_tracks` so they accept the `(d, R, S, P, regions_gvl, dense_range_gvl)` tuple from **either** `_query` (live) or `_query_cached`. Keep the live `_query` intact — it is the parity oracle. + +- [ ] **Step 3b: Add `HapsSvar2` to `_haps.py`** + +`HapsSvar2` loads the cache once in `from_path`, groups a read batch by contig, and per contig gathers + reconstructs. Reference is **required** (haplotypes need ref bytes). Only `RaggedSeqs` is supported. + +```python +@dataclass(slots=True) +class HapsSvar2(Reconstructor["RaggedSeqs"]): + path: Path + reference: Reference + svar2: "SparseVar2" + contigs: list[str] + samples: list[str] + ploidy: int + dense_range: NDArray[np.int32] # (R, 2) memmap + region_starts: NDArray[np.int32] # (R,) memmap + vk_snp_range: NDArray[np.int64] # (R, S, P, 2) memmap + vk_indel_range: NDArray[np.int64] # (R, S, P, 2) memmap + sample_cols: NDArray[np.int64] # (S,) + + @classmethod + def from_path(cls, path, reference, contigs, samples, ploidy, + svar2_link, svar2_override): + import json + from genoray import SparseVar2 + from ._svar2_link import _resolve_svar2, _verify_fingerprint2 + + svar2_path = _resolve_svar2(path, svar2_link, svar2_override) + _verify_fingerprint2(svar2_path, svar2_link) + svar2 = SparseVar2(svar2_path) + geno = path / "genotypes" + meta = json.loads((geno / "svar2_meta.json").read_text()) + def mm(name): + spec = meta[name] + return np.memmap(geno / f"svar2_{name}.npy", + dtype=np.dtype(spec["dtype"]), + mode="r", shape=tuple(spec["shape"])) + if reference is None: + raise ValueError("SVAR2 haplotype output requires a reference genome.") + return cls( + path=path, reference=reference, svar2=svar2, contigs=contigs, + samples=samples, ploidy=ploidy, + dense_range=mm("dense_range"), region_starts=mm("region_starts"), + vk_snp_range=mm("vk_snp_range"), vk_indel_range=mm("vk_indel_range"), + sample_cols=np.asarray(meta["sample_cols"], np.int64), + ) + + def to_kind(self, kind): + from .._ragged import RaggedSeqs + if kind is not RaggedSeqs: + raise NotImplementedError( + f"SVAR2 datasets support only 'haplotypes' output, not {kind.__name__}." + ) + return self + + def __call__(self, idx, r_idx, regions, output_length, jitter, rng, + deterministic, splice_plan=None, flat=False, to_rc=None): + if splice_plan is not None: + raise NotImplementedError("Spliced SVAR2 haplotypes are not supported.") + # idx -> (region, sample); group by contig; gather+reconstruct per contig; + # stitch back into batch order. Shifts/jitter mirror Haps._prepare_request. + ... # see Step 3c +``` + +- [ ] **Step 3c: Implement `HapsSvar2.__call__` (per-contig grouping + stitch)** + +The batch `regions` is `(b, 3)` = `(contig_idx, start, end)`; `idx` ravels `(region, sample)`. For SVAR2, ploidy is fixed. Group the batch rows by `contig_idx`, and for each contig build the cached bundle for those region rows × the requested samples, gather, and run `SparseVar2Source(...)`'s extracted reconstruct-from-payload. Compute `shifts` exactly as `Haps._prepare_request` does (zeros when `deterministic` or `output_length` is a string; else the same `rng.integers(0, max_shift+1)` where `max_shift = diffs.clip(min=0) + (lengths-output_length).clip(min=0)`). Derive `diffs` from the gathered payload — reuse the SVAR2 kernel's own length computation by first reconstructing with `output_length=-1` (ragged) to learn hap lengths, or expose an ilen helper. **Simplest correct first cut:** support `deterministic`/ragged output (shifts=0) end-to-end, and raise `NotImplementedError` for random-jitter fixed-length until a follow-up — the parity test (Step 1) uses `output_length=-1`, deterministic. Gate non-deterministic with a clear error and a `TODO`. + +Stitching: because each `(region, sample)` maps to one output row, collect per-contig `Ragged` outputs and reassemble a single `Ragged[(b, P, ~L)]` (or `(S, P, ~L)` for a single-region all-sample read) in `idx` order. Reuse `_Flat.from_offsets` as `SparseVar2Source.reconstruct` does. + +> This is the crux integration. Keep the marshalling in `SparseVar2Source` (already validated) and let `HapsSvar2` own only: cache slicing, per-contig grouping, shift computation, and output stitching. Pin every step with the live-oracle parity test. + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS (cached ≡ live, byte-for-byte). If offsets match but bytes differ, the divergence is in shift handling or sample-column mapping — reconcile against the live `_query` path. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_svar2_source.py python/genvarloader/_dataset/_haps.py tests/dataset/test_svar2_dataset.py tests/dataset/conftest.py +rtk git commit -m "feat(svar2): HapsSvar2 cached-ranges + gather reconstructor" +``` + +--- + +### Task 5: `Dataset.open` dispatch to `HapsSvar2` + +**Files:** +- Modify: `python/genvarloader/_dataset/_open.py` (`_build_seqs`, `:143`) +- Modify: `python/genvarloader/_dataset/_reconstruct.py` (`_build_reconstructor` accepts `HapsSvar2`) +- Modify: `python/genvarloader/_dataset/_impl.py` (`Dataset.open` reads `svar2=` override; the `_recon` union type includes `HapsSvar2`) +- Test: `tests/dataset/test_svar2_dataset.py` (extend) + +**Interfaces:** +- Consumes: `metadata.svar2_link` (Task 2), `HapsSvar2.from_path` (Task 4), `self.svar2` override on the open builder (parallel to `self.svar`, `_open.py:160`). +- Produces: `Dataset.open(path, reference=..., svar2=)` returns a dataset whose `_recon` routes haplotype reads through `HapsSvar2`. + +- [ ] **Step 1: Write the failing test** + +```python +def test_open_routes_svar2_to_hapssvar2(svar2_store, reference_fasta, tmp_path): + import polars as pl + from genvarloader._dataset._haps import HapsSvar2 + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes") + assert isinstance(ds._recon, HapsSvar2) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -k routes -q` +Expected: FAIL — `_recon` is a plain `Haps`, or `_build_seqs` errors on missing SVAR1 metadata. + +- [ ] **Step 3: Wire the dispatch** + +3a. In `_open.py::_build_seqs` (`:149`), branch on `metadata.svar2_link`: + +```python + if self._has_genotypes(): + if metadata.ploidy is None: + raise ValueError("Malformed dataset: found genotypes but not ploidy.") + if metadata.svar2_link is not None: + from ._haps import HapsSvar2 + if reference is None: + raise ValueError( + "SVAR2 datasets require a reference genome for haplotype output." + ) + return HapsSvar2.from_path( + path=self.path, + reference=reference, + contigs=metadata.contigs, + samples=metadata.samples, + ploidy=metadata.ploidy, + svar2_link=metadata.svar2_link, + svar2_override=getattr(self, "svar2", None), + ) + seqs = Haps.from_path(...) # unchanged SVAR1 path +``` + +`self._has_genotypes()` checks for `genotypes/` — confirm it does not require SVAR1-specific files (`svar_meta.json`); if it does, relax it to also accept `svar2_meta.json`. + +3b. Add a `svar2: Path | str | None = None` field to the open-builder dataclass (parallel to `svar`, wherever `self.svar` is defined) and thread it from `Dataset.open`'s signature (`_impl.py`). + +3c. In `_reconstruct.py::_build_reconstructor`, accept `HapsSvar2` for the `haplotypes` kind. The simplest wiring: treat `HapsSvar2` like `Haps` in the `seqs_kind in ("haplotypes", ...)` branch but restrict to `"haplotypes"`: + +```python + from ._haps import HapsSvar2 + if isinstance(seqs, HapsSvar2): + if seqs_kind not in (None, "haplotypes"): + raise NotImplementedError( + f"SVAR2 datasets support only 'haplotypes', not {seqs_kind!r}." + ) + active_seqs = seqs + # dispatch: HapsSvar2 alone -> itself; with tracks -> HapsSvar2Tracks (Task 6) +``` + +Add `HapsSvar2` (and `HapsSvar2Tracks` from Task 6) to the `_recon` type union in `_impl.py` (`:899`) and the `match self._recon` in `__getitem__` (`:1028`). + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_open.py python/genvarloader/_dataset/_reconstruct.py python/genvarloader/_dataset/_impl.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "feat(svar2): Dataset.open routes svar2 datasets to HapsSvar2" +``` + +--- + +### Task 6: Track re-alignment via the same cache (`HapsSvar2Tracks`) + +**Files:** +- Modify: `python/genvarloader/_dataset/_reconstruct.py` (add `HapsSvar2Tracks`) +- Modify: `python/genvarloader/_dataset/_svar2_source.py` (reuse `realign_tracks` from the cached payload) +- Test: `tests/dataset/test_svar2_dataset.py` (extend) + +**Interfaces:** +- Consumes: `HapsSvar2` (Task 4), `Tracks` (existing), `SparseVar2Source.realign_tracks` marshalling (existing), the same cached bundle + `gather_ranges` payload. +- Produces: `HapsSvar2Tracks(haps: HapsSvar2, tracks: Tracks)` implementing `Reconstructor[tuple[RaggedSeqs, _T]]`; `_build_reconstructor` returns it for `(HapsSvar2, Tracks)`. + +- [ ] **Step 1: Write the failing test (cached tracks ≡ live tracks)** + +```python +def test_svar2_tracks_match_live(svar2_store, reference_fasta, bigwig_track, tmp_path): + import polars as pl + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store), tracks=[bigwig_track]) + + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes").with_tracks(...) + _, cached_tracks = ds[0, :] + + # Live oracle: SparseVar2Source.realign_tracks with the same track buffer. + sv = SparseVar2(svar2_store) + live = SparseVar2Source(sv).realign_tracks("chr1", [(0, 40)], *_track_args(...)) + np.testing.assert_array_equal( + cached_tracks.to_packed().data, live.to_packed().data + ) +``` + +Adapt fixtures/`_track_args` to whatever the existing `tests/test_svar2_realign_tracks.py` uses. + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -k tracks -q` +Expected: FAIL — `_build_reconstructor` has no `(HapsSvar2, Tracks)` case. + +- [ ] **Step 3: Implement `HapsSvar2Tracks` + dispatch** + +Model on `HapsTracks.__call__` (`_reconstruct.py:130`) but source haplotypes + realigned tracks from the cached `gather_ranges` payload via the extracted `SparseVar2Source.realign_tracks` body. The track buffer is read from the dataset's `intervals/` exactly as `HapsTracks` does (`self.tracks.intervals[name]`); only the haplotype-coordinate re-alignment uses the SVAR2 kernel + cached ranges instead of the SVAR1 sparse-genotype path. Route `(HapsSvar2, Tracks)` in `_build_reconstructor`: + +```python + if isinstance(active_seqs, HapsSvar2) and active_tracks is not None: + return HapsSvar2Tracks(haps=active_seqs, tracks=active_tracks) +``` + +Interval (non-realigned) tracks with SVAR2: keep the existing `realign_tracks=False` guard message from `_build_reconstructor` (`:360`) — reuse verbatim. + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS (haps + tracks parity). + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_reconstruct.py python/genvarloader/_dataset/_svar2_source.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "feat(svar2): HapsSvar2Tracks realign via cached ranges" +``` + +--- + +### Task 7: End-to-end byte-identical parity + full regression + +**Files:** +- Test: `tests/dataset/test_svar2_dataset.py` (extend with the M6b matrix + real chr21) +- Modify: `python/genvarloader/_dataset/_svar2_source.py` — retire `TODO(svar2-dataset-dispatch)` comment (`:7`) + +**Interfaces:** +- Consumes: everything above. Produces no new production surface — hardens the contract. + +- [ ] **Step 1: Add the M6b matrix parity test** + +Parametrize over `{SNP, INS, DEL} × {1, 2, 4} samples × {1, 2} ploidy` (reuse the synthesis helpers from `tests/test_svar2_reconstruct.py`). For each: build a `.svar2`, `gvl.write`, open, and assert `ds[region, samples]` ≡ the live `SparseVar2Source.reconstruct` ≡ genoray `decode` (the cross-check oracle the M6b kernels already validate against). Assert offsets **and** packed bytes equal. + +```python +import pytest + +@pytest.mark.parametrize("variant_kind", ["snp", "ins", "del"]) +@pytest.mark.parametrize("n_samples", [1, 2, 4]) +@pytest.mark.parametrize("ploidy", [1, 2]) +def test_svar2_cached_matches_decode_matrix(variant_kind, n_samples, ploidy, tmp_path): + ... # synth store -> gvl.write -> open -> compare cached vs live vs decode +``` + +- [ ] **Step 2: Add the real chr21 germline + somatic parity test (slow)** + +Mark `@pytest.mark.slow`. Point at the real chr21 SVAR2 stores used by the E1 profiling driver (grep `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` and the profiling driver for their paths). Compare cached-`Dataset` reconstruct vs live `SparseVar2Source` over the 3-region × all-samples workload, haps + tracks. + +- [ ] **Step 3: Retire the deferred-dispatch TODO** + +Remove the `TODO(svar2-dataset-dispatch)` block in `_svar2_source.py:7` (now delivered) and update the module docstring to describe the cache+gather read path. + +- [ ] **Step 4: Full tree regression (SVAR1 unchanged)** + +Run: +```bash +pixi run -e dev pytest tests -q +pixi run -e dev cargo-test +pixi run -e dev ruff check python/ tests/ +pixi run -e dev typecheck +``` +Expected: all green. If any SVAR1 test changed behavior, the additive claim is violated — investigate before proceeding. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_svar2_source.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "test(svar2): byte-identical cached parity across M6b matrix + chr21" +``` + +--- + +### Task 8: Docs, skill, api.md, and roadmaps + +**Files:** +- Modify: `skills/genvarloader/SKILL.md`, `docs/source/{api,write,format,faq}.md`, `README.md`, `docs/roadmaps/rust-migration.md` +- Modify: `docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md` (mark delivered) — optional + +**Interfaces:** +- Consumes: the shipped feature. Produces: docs true against `main` per the repo's docs-audit + skill-maintenance gates. + +- [ ] **Step 1: Document `.svar2` as a `write` variant source** + +Update `docs/source/write.md` (and `SKILL.md`'s write section) to list `.svar2` directories as an accepted `variants=` source alongside VCF/PGEN/`.svar`, noting the write-time ranges cache and that `Dataset.open` accepts a `svar2=` override (parallel to `svar=`). Update `docs/source/format.md` with the `genotypes/svar2_*.npy` + `svar2_meta.json` cache layout and the `svar2_link` metadata key. + +- [ ] **Step 2: Sync `api.md` with `__all__`** + +If any new symbol was exported (e.g. a `svar2=` kwarg is not a new `__all__` symbol, but confirm no new public class leaked), run the sync check: + +```bash +pixi run -e dev python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. If a symbol was added to `__all__`, add its autodoc entry. + +- [ ] **Step 3: Update FAQ + README + rust-migration roadmap** + +`faq.md`: add "Can I use SVAR2 files with gvl?" → yes, same as `.svar`, with the search/gather cache. `README.md`: add `.svar2` to the supported variant sources if `.svar` is listed. `docs/roadmaps/rust-migration.md`: tick the SVAR2 dataset-wiring task, record the perf-verification result (Step 4), set the phase marker + PR link (per CLAUDE.md's rust-migration rule). + +- [ ] **Step 4: Perf verification (same-session before/after)** + +Per the spec + the `gvl-rust-perf-gate-shared-node-noise` and `gvl-profiling-perf-not-pyspy-native` memories: profile a warm SVAR2 `Dataset` read with `perf` on the Python process (paranoid=2, no `py-spy --native`) and confirm the DSO split flips from ~80% genoray `SearchTree::build` to gvl-kernel-bound, like SVAR1. Report as a **relative before/after within one allocation** (absolute wall-clock not comparable across allocations on shared Carter nodes). Record the numbers in the rust-migration roadmap. + +- [ ] **Step 5: Commit + finish the branch** + +```bash +rtk git add skills/ docs/ README.md +rtk git commit -m "docs(svar2): document .svar2 write source + dataset wiring" +``` + +Then use **superpowers:finishing-a-development-branch** to decide merge/PR. Before the PR: re-run `pixi run -e dev pytest tests -q` (full tree, not scoped) per CLAUDE.md's rename/shared-code gate. + +--- + +## Self-Review + +- **Spec coverage (Components B & C + format/parity/docs):** + - Component B: `.svar2` detection + dispatch + `_write_from_svar2` (Task 3); `_svar2_link.py` (Task 1); `metadata["ploidy"]` set (Task 3); reject unsupported variants (Task 3, Step 3d note). ✅ + - Component C: `Dataset.open` resolve+fingerprint + `svar2=` override (Task 5); `HapsSvar2` haplotype routing retiring `TODO(svar2-dataset-dispatch)` (Tasks 4/7); tracks via same cache (Task 6); `_svar2_source.py` refactor to cache+gather (Tasks 4/6/7). ✅ + - Cache format `.gvl/genotypes/` O(offsets) memmaps + `svar2_meta.json` + `Svar2Link` (Task 3). ✅ + - Parity & testing: byte-identical cached ≡ live ≡ decode on M6b matrix + chr21; SVAR1 additive-green; perf verification (Tasks 4–8). ✅ + - Format version: additive `svar2_link` tolerated by `_check_dataset_format_version`/`_migrate` (Task 2). ✅ + - Docs/roadmaps: skill, api.md, write/format/faq/README, rust-migration (Task 8). ✅ +- **Out of scope honored:** `variants`/`annotated` SVAR2 output modes raise `NotImplementedError` (Tasks 4/5); no `.svar2` on-disk format change (write only reads it). +- **Type consistency:** the cache bundle field names (`dense_range`, `region_starts`, `sample_cols`, `vk_snp_range`, `vk_indel_range`, `n_regions`/`n_samples`/`ploidy`) match exactly between `_write_from_svar2` (Task 3), `SparseVar2Source._query_cached` (Task 4), and the genoray `gather_ranges` bundle contract (genoray plan Task 4). `HapsSvar2.from_path`'s memmap names (`svar2_.npy`) match the files `_write_from_svar2` writes. +- **Known risk flagged in-plan:** Task 4 Step 3c bounds the first cut to deterministic/ragged reads and defers random-jitter fixed-length SVAR2 output to a follow-up (explicit `NotImplementedError`), keeping the crux integration testable against the live oracle rather than over-reaching. diff --git a/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md b/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md new file mode 100644 index 00000000..aff20cc1 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md @@ -0,0 +1,1092 @@ +# genoray Read-Bound Per-Class Gather + Query-Only Build — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a htslib-free query-only build of genoray plus a read-bound, per-class dense gather that reconstructs the same `BatchResult` payload as today's union path **without** building the contig-wide `DenseUnion` (eliminating the O(N_contig) per-read residual), so GenVarLoader can link `genoray_core` as a query-only path-dep and reconstruct SVAR2 entirely in Rust. + +**Architecture:** Two additive changes to the shipped `svar-2` search/gather split. (1) A `conversion` cargo feature (default-on) gates every htslib-touching module so `--no-default-features` compiles the read/query core alone. (2) `find_ranges` additionally emits per-class `dense_snp_range` / `dense_indel_range` (computed by a per-class `SearchTree` at search time), and a new `gather_ranges_readbound` slices each on-disk dense class window directly into a new split-dense `BatchResultSplit` — never calling `dense_union()`. The shipped `find_ranges` / `gather_ranges` / `read_ranges` / `overlap_batch` stay byte-unchanged as the parity oracle. + +**Tech Stack:** Rust 2024, PyO3 0.29, `numpy` 0.29, `svar2-codec` (workspace member), `rust-htslib` (made optional), `cargo test` with `proptest` + `tempfile`. + +**Repo:** `/carter/users/dlaub/projects/genoray` — branch `svar-2` (HEAD `7099f16`). Lib crate name is `genoray_core`. This is the **absolute** path GenVarLoader path-deps (there is no `../genoray` sibling checkout; the spec's `../genoray` is wrong). + +## Global Constraints + +- **Byte-identical parity contract.** For any `contig, regions, samples`, the read-bound path reconstructs the *same variants per hap* as the shipped union path and the `decode_hap` oracle — field-for-field. The split-dense `BatchResultSplit` merged with var_key equals `overlap_batch`'s union merged with var_key, per hap. +- **Additive.** `overlap_batch`, `find_ranges`, `gather_ranges`, `read_ranges`, `BatchResult`, `RangesBundle.dense_range`, and every existing Python dict key stay **byte-unchanged**. New code is new structs/functions/fields only. The full existing test suite (`tests/test_ranges_split.rs`, `tests/test_batch.rs`, `tests/test_decode_mat.rs`, `svar2-codec` proptests) stays green. +- **Query core is htslib-free.** After Task 1, `cargo build --no-default-features` and `cargo test --no-default-features` compile and pass without linking `rust-htslib`. The default (wheel) build is behavior-unchanged. +- **`rust-htslib` reach = `vcf_reader.rs` + `lib.rs:40-52`.** Only `vcf_reader.rs` uses htslib *types*; `lib.rs`'s `index_bcf_csi`/`index_vcf` are a second direct call site. Both must be gated, or `--no-default-features` fails to compile `lib.rs`. +- **`DenseView` + `carried()` live in `query.rs:120-136`**, not `dense.rs`. `dense.rs` holds only `DENSE_REGISTRY`/`DenseClass`/`DenseSpec`/`DenseMap`. +- **`decode_key` is `svar2_codec::decode_key`**, re-exported verbatim as `rvk::decode_key` (`rvk.rs:14`). Codec primitives used here: `rvk::snp_code_to_key`, `rvk::unpack_snp_key_at`, `rvk::deletion_len`. +- **Row/hap index conventions (unchanged):** `RangesBundle` per-hap row = `r * (n_samples*ploidy) + si*ploidy + p` where `si` is the *selected* sample slot and `sample_cols[si]` is the original sample index. `BatchResult` hap index = `(r*n_samples + s)*ploidy + p`, region-major. +- **Every Rust step:** `cargo test` compiles from source (no separate rebuild needed). Run `cargo test -p genoray_core` for the query core. Run `cargo fmt` + `cargo clippy --all-targets` before each commit; both must be clean. +- **Local-only.** No crates.io / PyPI publish in this plan. Task 6 builds a **local wheel** and confirms the crate builds for the downstream gvl path-dep; that wheel and path-dep MUST be the same commit. + +--- + +## File Structure + +- `Cargo.toml` — make `rust-htslib` optional; add `conversion` feature (default-on). *(Task 1)* +- `src/lib.rs` — `#[cfg(feature = "conversion")]` gates on htslib-touching modules + `index_bcf_csi`/`index_vcf`/`run_conversion_pipeline` + their `#[pymodule]` registrations. *(Task 1)* +- `src/query.rs` — add `dense_snp_overlap` / `dense_indel_overlap` methods, two new `RangesBundle` fields, `BatchResultSplit` struct, `gather_ranges_readbound` fn. *(Tasks 2, 3)* +- `src/py_query_ranges.rs` — add the two new range keys to `bundle_to_dict` / `bundle_from_dict`. *(Task 5)* +- `tests/test_readbound_gather.rs` — new parity + zero-union + per-class test file. *(Task 4)* +- `tests/common/mod.rs` — reused as-is (no change). +- `docs/roadmaps/*` (genoray-side roadmap) — mark the read-bound gather + conversion feature. *(Task 7)* + +--- + +## Task 1: `conversion` cargo feature (htslib-free query core) + +**Files:** +- Modify: `Cargo.toml` (`[dependencies]` `rust-htslib` line; `[features]` block) +- Modify: `src/lib.rs:5-32` (module decls), `src/lib.rs:40-52` (`index_bcf_csi`/`index_vcf`), `src/lib.rs:164-170` (`#[pymodule]`) + +**Interfaces:** +- Produces: a `conversion` feature such that `default = ["conversion", "extension-module"]`; `cargo build --no-default-features` compiles the query core (`query`, `search`, `spine`, `bits`, `nrvk`, `rvk`, `layout`, `dense`, `types`, `error`, `cost_model`, `py_query*`) without `rust-htslib`. + +- [ ] **Step 1: Write the failing build check** + +Add this test to a new file `tests/test_query_only_build.rs`: + +```rust +//! Compile-guard: the query core must build & link without the `conversion` +//! feature (no rust-htslib). If this file compiles under +//! `--no-default-features`, the gate is correct. +#[test] +fn query_core_symbols_are_reachable_without_conversion() { + // Referencing these paths forces the query core to be part of the + // no-default-features build graph. + use genoray_core::query::{ContigReader, find_ranges, gather_ranges}; + let _ = ContigReader::open; + let _ = find_ranges; + let _ = gather_ranges; +} +``` + +- [ ] **Step 2: Run it to verify current state** + +Run: `cargo test --no-default-features --test test_query_only_build 2>&1 | tail -30` +Expected: **FAIL to compile** — `rust-htslib` symbols in `vcf_reader.rs` / `lib.rs` are unconditionally in the build graph, and `pyo3/auto-initialize` (dev-dep) needs libpython. If it fails only on libpython linkage, add `--features pyo3/auto-initialize` is NOT wanted; instead confirm the failure mentions htslib/`vcf_reader`. Record the actual first error. + +- [ ] **Step 3: Make `rust-htslib` optional + add the feature** + +In `Cargo.toml`, change the `rust-htslib` dependency line to optional and add the feature. Replace: + +```toml +rust-htslib = { version = "1.0", default-features = false } +``` + +with: + +```toml +rust-htslib = { version = "1.0", default-features = false, optional = true } +``` + +and replace the `[features]` block: + +```toml +[features] +default = ["extension-module"] +extension-module = ["pyo3/extension-module"] +``` + +with: + +```toml +[features] +# `conversion` pulls in rust-htslib and gates the VCF→svar2 write/convert +# pipeline. Off => query-only core (what gvl links as a path-dep). +default = ["conversion", "extension-module"] +conversion = ["dep:rust-htslib"] +extension-module = ["pyo3/extension-module"] +``` + +- [ ] **Step 4: Gate the htslib-touching modules in `src/lib.rs`** + +At `src/lib.rs:5-32`, add `#[cfg(feature = "conversion")]` above each of these `pub mod` lines (leave the query-core modules ungated): `vcf_reader`, `writer`, `orchestrator`, `normalize`, `budget`, `executor`, `monitor`, `streams`, `merge`, `max_del`, `dense_merge`, `meta`, `py_convert`. Also gate `pub use orchestrator::process_chromosome;`. Example: + +```rust +#[cfg(feature = "conversion")] +pub mod vcf_reader; +#[cfg(feature = "conversion")] +pub mod writer; +// ... (repeat for the list above) +#[cfg(feature = "conversion")] +pub use orchestrator::process_chromosome; +``` + +Gate the two direct htslib call sites and the conversion pyfunctions at `src/lib.rs:40-52`: + +```rust +#[cfg(feature = "conversion")] +fn index_bcf_csi(/* ...existing signature... */) { /* ...unchanged body... */ } + +#[cfg(feature = "conversion")] +#[pyfunction] +fn index_vcf(/* ...existing... */) -> PyResult<()> { /* ...unchanged... */ } + +#[cfg(feature = "conversion")] +#[pyfunction] +fn run_conversion_pipeline(/* ...existing... */) -> PyResult<()> { /* ...unchanged... */ } +``` + +And gate their registrations in the `#[pymodule]` at `src/lib.rs:164-170`: + +```rust +#[pymodule] +fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { + #[cfg(feature = "conversion")] + m.add_function(wrap_pyfunction!(run_conversion_pipeline, m)?)?; + #[cfg(feature = "conversion")] + m.add_function(wrap_pyfunction!(index_vcf, m)?)?; + m.add_class::()?; + Ok(()) +} +``` + +> If the compiler reports another module transitively pulling htslib (e.g. a query-core module that `use`s a now-gated module), gate the *offending `use`*, not the query-core module — the query core must stay ungated. Record any module you had to additionally gate in the commit message. + +- [ ] **Step 5: Verify the query-only build compiles and passes** + +Run: `cargo build --no-default-features 2>&1 | tail -20` +Expected: compiles clean (no `rust-htslib`). +Run: `cargo test --no-default-features --test test_query_only_build 2>&1 | tail -20` +Expected: **PASS**. + +- [ ] **Step 6: Verify the default (wheel) build is unchanged** + +Run: `cargo build 2>&1 | tail -5 && cargo test 2>&1 | tail -30` +Expected: full suite green (default features include `conversion`, so nothing else changed). + +- [ ] **Step 7: fmt + clippy + commit** + +Run: `cargo fmt && cargo clippy --all-targets --no-default-features 2>&1 | tail -20 && cargo clippy --all-targets 2>&1 | tail -20` +Expected: no warnings. + +```bash +cd /carter/users/dlaub/projects/genoray +git add Cargo.toml src/lib.rs tests/test_query_only_build.rs +git commit -m "feat(query): conversion feature gates htslib; query core builds --no-default-features + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: Per-class dense overlap + `find_ranges` emits `dense_snp_range` / `dense_indel_range` + +**Files:** +- Modify: `src/query.rs` — add `dense_snp_overlap` / `dense_indel_overlap` methods to `impl ContigReader` (near `vk_snp_overlap` at `:608-651`); add two fields to `RangesBundle` (`:590-606`); populate them in `find_ranges` (`:657-701`). +- Test: `tests/test_readbound_gather.rs` (new; created here, extended in Task 4). + +**Interfaces:** +- Consumes: `DenseView::positions()` and `DenseView::keys` on `reader.dense_snp` / `reader.dense_indel` (both `Option`); `reader.dense_indel_max_del: u32`; `rvk::deletion_len`, `rvk::unpack_snp_key_at`; `SearchTree::new`, `overlap_range` (already imported in `query.rs`). +- Produces: + - `impl ContigReader { fn dense_snp_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize); fn dense_indel_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize); }` — absolute `[s,e)` into that class's on-disk dense positions/keys table; `(0,0)` when the class table is absent/empty. + - `RangesBundle` gains `pub dense_snp_range: Vec<(usize, usize)>` and `pub dense_indel_range: Vec<(usize, usize)>`, each length `n_regions` (per-region, sample-independent — dense is cohort-shared). + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_readbound_gather.rs` with the synth harness copied from `test_ranges_split.rs:55-79` and this first test: + +```rust +//! Read-bound per-class gather: find_ranges emits per-class dense ranges and +//! gather_ranges_readbound replays them into BatchResultSplit without building +//! the contig-wide DenseUnion. +mod common; + +use common::{SynthRecord, build_contig}; +use genoray_core::query::{ + ContigReader, find_ranges, gather_ranges, gather_ranges_readbound, overlap_batch, +}; +use genoray_core::search; +use tempfile::tempdir; + +fn synth_reader(out: &std::path::Path) -> ContigReader { + let samples = ["S0", "S1"]; + let records = vec![ + SynthRecord { pos: 100, ref_allele: b"A", alts: vec![&b"C"[..]], gt: vec![1, 0, 0, 0] }, + SynthRecord { pos: 200, ref_allele: b"A", alts: vec![&b"AT"[..]], gt: vec![0, 1, 1, 1] }, + SynthRecord { pos: 300, ref_allele: b"AT", alts: vec![&b"A"[..]], gt: vec![1, 1, 0, 1] }, + ]; + build_contig(out, "chr1", &samples, 2, &records); + ContigReader::open(out.to_str().unwrap(), "chr1", 2, 2).unwrap() +} + +#[test] +fn test_find_ranges_emits_per_class_dense_ranges() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let rb = find_ranges(&reader, ®ions, None); + // Both per-class range vectors are per-region (dense is cohort-shared). + assert_eq!(rb.dense_snp_range.len(), regions.len()); + assert_eq!(rb.dense_indel_range.len(), regions.len()); + // Each per-class window is a subset of that class's table; ranges are valid. + for &(s, e) in rb.dense_snp_range.iter().chain(rb.dense_indel_range.iter()) { + assert!(s <= e); + } + // Region 0 spans the whole contig: it must see the one dense SNP (pos 100 is + // var_key here, but the SNP class table is nonempty iff any SNP is dense) and + // the dense indels. The union window must equal snp∪indel counts. + let (us0, ue0) = rb.dense_range[0]; + let snp0 = rb.dense_snp_range[0].1 - rb.dense_snp_range[0].0; + let indel0 = rb.dense_indel_range[0].1 - rb.dense_indel_range[0].0; + assert_eq!(ue0 - us0, snp0 + indel0, + "union window size must equal sum of per-class window sizes"); +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test --test test_readbound_gather test_find_ranges_emits_per_class_dense_ranges 2>&1 | tail -20` +Expected: **FAIL to compile** — `gather_ranges_readbound` unresolved (added Task 3) and `rb.dense_snp_range` unknown field. + +> To iterate on Task 2 alone before Task 3 exists, temporarily comment the `gather_ranges_readbound` import; restore it in Task 3. + +- [ ] **Step 3: Add the two `RangesBundle` fields** + +In `src/query.rs`, extend the `RangesBundle` struct (`:590-606`) — append after `vk_indel_range`: + +```rust + /// `[s, e)` into `dense/snp`'s on-disk positions/keys, per region (dense is + /// cohort-shared, so one window per region, not per hap). Read-bound path. + pub dense_snp_range: Vec<(usize, usize)>, + /// `[s, e)` into `dense/indel`'s on-disk positions/keys, per region. + pub dense_indel_range: Vec<(usize, usize)>, +``` + +- [ ] **Step 4: Add the per-class overlap methods** + +In `src/query.rs`, inside `impl ContigReader` (right after `vk_indel_overlap` ends at `:651`), add: + +```rust + /// Absolute `[s, e)` into `dense/snp`'s positions/keys for one region. + /// SNP v_end = pos + 1 (max_region_length = 0). `(0, 0)` if no snp table. + fn dense_snp_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize) { + let d = match &self.dense_snp { + Some(d) => d, + None => return (0, 0), + }; + let positions = d.positions(); + if positions.is_empty() { + return (0, 0); + } + let v_ends: Vec = positions.iter().map(|&p| p + 1).collect(); + let tree = SearchTree::new(positions); + overlap_range(&tree, &v_ends, 0, q_start, q_end) + } + + /// Absolute `[s, e)` into `dense/indel`'s positions/keys for one region. + /// Indel v_end = pos + 1 + deletion_len(key); per-contig dense max_del bound. + fn dense_indel_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize) { + let d = match &self.dense_indel { + Some(d) => d, + None => return (0, 0), + }; + let positions = d.positions(); + if positions.is_empty() { + return (0, 0); + } + let keys = as_u32(&d.keys); + debug_assert_eq!(positions.len(), keys.len()); + let v_ends: Vec = positions + .iter() + .zip(keys.iter()) + .map(|(&pos, &key)| pos + 1 + rvk::deletion_len(key)) + .collect(); + let tree = SearchTree::new(positions); + overlap_range(&tree, &v_ends, self.dense_indel_max_del, q_start, q_end) + } +``` + +- [ ] **Step 5: Populate the fields in `find_ranges`** + +In `find_ranges` (`:657-701`), after the existing `dense_range` / `region_starts` computation (`:672-677`), add: + +```rust + let dense_snp_range: Vec<(usize, usize)> = regions + .iter() + .map(|&(qs, qe)| reader.dense_snp_overlap(qs, qe)) + .collect(); + let dense_indel_range: Vec<(usize, usize)> = regions + .iter() + .map(|&(qs, qe)| reader.dense_indel_overlap(qs, qe)) + .collect(); +``` + +and add the two fields to the returned `RangesBundle { ... }` literal (`:691-700`): + +```rust + dense_snp_range, + dense_indel_range, +``` + +- [ ] **Step 6: Fix the other `RangesBundle` construction site** + +`find_ranges` is the only constructor, but `cargo build` will confirm. Run: `cargo build 2>&1 | tail -20`. If any other `RangesBundle { ... }` literal errors on the missing fields, add the two fields there too. + +- [ ] **Step 7: Run the per-class range test (isolate it)** + +Temporarily comment the `gather_ranges_readbound` import + the not-yet-written test bodies so only `test_find_ranges_emits_per_class_dense_ranges` compiles. Run: +`cargo test --test test_readbound_gather test_find_ranges_emits_per_class_dense_ranges 2>&1 | tail -20` +Expected: **PASS**. + +- [ ] **Step 8: Confirm the shipped path is byte-unchanged** + +Run: `cargo test --test test_ranges_split 2>&1 | tail -20` +Expected: all existing split tests green (we only *added* fields + methods). + +- [ ] **Step 9: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs tests/test_readbound_gather.rs +git commit -m "feat(query): find_ranges emits per-class dense_snp_range/dense_indel_range + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: `BatchResultSplit` + `gather_ranges_readbound` (no `dense_union`) + +**Files:** +- Modify: `src/query.rs` — add `BatchResultSplit` struct (near `BatchResult` at `:488-504`); add `pub fn gather_ranges_readbound` (after `gather_ranges` at `:711-811`). +- Test: `tests/test_readbound_gather.rs` (Task 4 asserts parity). + +**Interfaces:** +- Consumes: `RangesBundle` (now with `dense_snp_range` / `dense_indel_range` from Task 2); `reader.vk_snp` / `reader.vk_indel` packed positions/keys; `reader.dense_snp` / `reader.dense_indel` (`DenseView`, with `.positions()`, `.keys`, `.carried(hap, col)`); `spine::merge_keys`; `bits::set_bit`; `rvk::{snp_code_to_key, unpack_snp_key_at, deletion_len}`; `as_bytes`, `as_u32`, `KeyRef`. +- Produces: + - `pub fn gather_ranges_readbound(reader: &ContigReader, rb: &RangesBundle) -> BatchResultSplit` — cartesian R×S'; the parity-test vehicle. Builds **zero** `SearchTree`, never calls `reader.dense_union()`. + - `pub fn gather_haps_readbound(reader, region_starts, orig_samples, vk_snp_range, vk_indel_range, dense_snp_range, dense_indel_range, ploidy) -> BatchResultSplit` — **flat per-query** (one `(region, sample)` per query row); the primitive gvl links and calls (Plan 2, Task 4). `n_samples = 1`, hap index `q*ploidy + p`. +- `BatchResultSplit` fields (var_key merged per hap, dense split by class): + ```rust + pub struct BatchResultSplit { + pub n_regions: usize, pub n_samples: usize, pub ploidy: usize, + pub vk: Vec, pub vk_off: Vec, + pub dense_snp: Vec, pub dense_snp_range: Vec<(usize, usize)>, + pub dense_snp_present: Vec, pub dense_snp_present_off: Vec, + pub dense_indel: Vec, pub dense_indel_range: Vec<(usize, usize)>, + pub dense_indel_present: Vec, pub dense_indel_present_off: Vec, + } + ``` + Per-hap presence bitmask is over that class's per-region window `[ds..de)`, LSB-first; `*_present_off` (len H+1) holds **bit** offsets. `H = n_regions * n_samples * ploidy`, hap index `(r*n_samples+s)*ploidy+p` over the *selected* samples. + +- [ ] **Step 1: Add the `BatchResultSplit` struct** + +In `src/query.rs`, after the `BatchResult` struct (`:504`), add: + +```rust +/// Read-bound analog of `BatchResult`: the var_key channel merged per hap (as +/// today), but the dense channel **split per class** so no contig-wide +/// `DenseUnion` is built. gvl merges `var_key ⋈ dense_snp ⋈ dense_indel` by +/// position downstream. `H = n_regions * n_samples * ploidy`, hap index +/// `(r*n_samples + s)*ploidy + p` over the *selected* samples. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BatchResultSplit { + pub n_regions: usize, + pub n_samples: usize, + pub ploidy: usize, + /// Flat merged var_key channel (snp+indel per hap); `vk_off` (len H+1) slices it. + pub vk: Vec, + pub vk_off: Vec, + /// Per-region `dense/snp` windows (uniform keys), concatenated. + pub dense_snp: Vec, + /// `[s, e)` into `dense_snp` per region (len n_regions). + pub dense_snp_range: Vec<(usize, usize)>, + /// Per-hap presence bitmask over that region's `dense_snp[s..e]`, LSB-first; + /// `dense_snp_present_off` (len H+1) holds BIT offsets. + pub dense_snp_present: Vec, + pub dense_snp_present_off: Vec, + /// Per-region `dense/indel` windows (uniform u32 keys), concatenated. + pub dense_indel: Vec, + pub dense_indel_range: Vec<(usize, usize)>, + pub dense_indel_present: Vec, + pub dense_indel_present_off: Vec, +} +``` + +- [ ] **Step 2: Write `gather_ranges_readbound`** + +In `src/query.rs`, after `gather_ranges` (`:811`), add. This mirrors `gather_ranges`'s var_key gather verbatim, and replaces the single union presence loop with two per-class window loops that read positions/keys straight from each on-disk dense table (no `dense_union()`): + +```rust +/// Tree-free, union-free gather: replay a `RangesBundle` into a split-dense +/// `BatchResultSplit`. Builds NO `SearchTree` and never calls `dense_union()` — +/// each region's dense windows come from the per-class `dense_snp_range` / +/// `dense_indel_range` computed in `find_ranges`. The var_key channel is +/// identical to `gather_ranges`; only the dense side is split per class. +pub fn gather_ranges_readbound(reader: &ContigReader, rb: &RangesBundle) -> BatchResultSplit { + let ploidy = rb.ploidy; + let n_samples = rb.n_samples; + let n_regions = rb.n_regions; + let hpr = n_samples * ploidy; + + let snp_positions = reader.vk_snp.positions(); + let snp_keys = as_bytes(&reader.vk_snp.keys); + let indel_positions = reader.vk_indel.positions(); + let indel_keys = as_u32(&reader.vk_indel.keys); + + // Dense class tables (may be absent). + let d_snp = reader.dense_snp.as_ref(); + let d_indel = reader.dense_indel.as_ref(); + let d_snp_pos: &[u32] = d_snp.map(|d| d.positions()).unwrap_or(&[]); + let d_indel_pos: &[u32] = d_indel.map(|d| d.positions()).unwrap_or(&[]); + + // --- dense channel windows (per region), decoded to uniform keys once --- + let mut dense_snp: Vec = Vec::new(); + let mut dense_snp_range: Vec<(usize, usize)> = Vec::with_capacity(n_regions); + let mut dense_indel: Vec = Vec::new(); + let mut dense_indel_range: Vec<(usize, usize)> = Vec::with_capacity(n_regions); + for r in 0..n_regions { + let (ss, se) = rb.dense_snp_range[r]; + let base = dense_snp.len(); + if let Some(d) = d_snp { + let keys = as_bytes(&d.keys); + for j in ss..se { + dense_snp.push(KeyRef { + position: d_snp_pos[j], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, j)), + }); + } + } + dense_snp_range.push((base, dense_snp.len())); + + let (is_, ie_) = rb.dense_indel_range[r]; + let base = dense_indel.len(); + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for j in is_..ie_ { + dense_indel.push(KeyRef { + position: d_indel_pos[j], + key: keys[j], + }); + } + } + dense_indel_range.push((base, dense_indel.len())); + } + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_snp_present: Vec = Vec::new(); + let mut dense_snp_present_off: Vec = vec![0]; + let mut dense_indel_present: Vec = Vec::new(); + let mut dense_indel_present_off: Vec = vec![0]; + + for r in 0..n_regions { + let qs = rb.region_starts[r]; + let (ss, se) = rb.dense_snp_range[r]; + let (is_r, ie_r) = rb.dense_indel_range[r]; + for si in 0..n_samples { + let orig_s = rb.sample_cols[si]; + for p in 0..ploidy { + let col = orig_s * ploidy + p; + let hap = col; + let row = r * hpr + si * ploidy + p; + + // --- var_key gather (identical to gather_ranges) --- + let (vs, ve) = rb.vk_snp_range[row]; + let mut snp_run: Vec = Vec::new(); + for (j, &pos) in snp_positions.iter().enumerate().take(ve).skip(vs) { + if qs < pos + 1 { + snp_run.push(KeyRef { + position: pos, + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(snp_keys, j)), + }); + } + } + let (vis, vie) = rb.vk_indel_range[row]; + let mut indel_run: Vec = Vec::new(); + for j in vis..vie { + let pos = indel_positions[j]; + let v_end = pos + 1 + rvk::deletion_len(indel_keys[j]); + if qs < v_end { + indel_run.push(KeyRef { position: pos, key: indel_keys[j] }); + } + } + let merged = spine::merge_keys(vec![snp_run, indel_run]); + vk.extend_from_slice(&merged); + vk_off.push(vk.len()); + + // --- dense/snp presence bits over [ss..se) --- + let nbits = se - ss; + let bit_base = *dense_snp_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_snp_present.len() < need { + dense_snp_present.resize(need, 0); + } + if let Some(d) = d_snp { + for (k, j) in (ss..se).enumerate() { + // snp v_end = pos + 1; left-overlap re-check qs < v_end. + if d.carried(hap, j) && qs < d_snp_pos[j] + 1 { + bits::set_bit(&mut dense_snp_present, bit_base + k); + } + } + } + dense_snp_present_off.push(bit_base + nbits); + + // --- dense/indel presence bits over [is_r..ie_r) --- + let nbits = ie_r - is_r; + let bit_base = *dense_indel_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_indel_present.len() < need { + dense_indel_present.resize(need, 0); + } + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for (k, j) in (is_r..ie_r).enumerate() { + let v_end = d_indel_pos[j] + 1 + rvk::deletion_len(keys[j]); + if d.carried(hap, j) && qs < v_end { + bits::set_bit(&mut dense_indel_present, bit_base + k); + } + } + } + dense_indel_present_off.push(bit_base + nbits); + } + } + } + + BatchResultSplit { + n_regions, + n_samples, + ploidy, + vk, + vk_off, + dense_snp, + dense_snp_range, + dense_snp_present, + dense_snp_present_off, + dense_indel, + dense_indel_range, + dense_indel_present, + dense_indel_present_off, + } +} +``` + +> **Presence-bit indexing note:** `d.carried(hap, col)` addresses the *global* per-class dense column, and here `j` (the absolute on-disk row inside `[ss..se)`) **is** that global column, because `dense_snp_range` / `dense_indel_range` are absolute indices into the class table. This is the read-bound simplification: the union path had to remap through `dense.src[j] = (is_indel, dcol)`; per-class, `j` is already `dcol`. + +- [ ] **Step 3: Add the flat per-query gather `gather_haps_readbound` (gvl's read primitive)** + +`gather_ranges_readbound` is cartesian R×S' and is the parity-test vehicle (easy to compare to `overlap_batch`). But gvl reads an **arbitrary set of `(region, sample)` pairs** — one query row each, exactly like SVAR1's `geno_offset_idx` — so it needs a *flat per-query* primitive where each query carries its own original sample (dense carriage needs `hap = orig_sample*ploidy + p`). Add, after `gather_ranges_readbound`: + +```rust +/// Flat per-query read-bound gather for gvl's arbitrary-(region,sample) reads. +/// Each of `n_q = region_starts.len()` queries is one (region, sample) pair +/// reconstructing `ploidy` haps. Range arrays are per-query (`dense_*_range`, +/// length n_q) or per-(query,ploid) (`vk_*_range`, length n_q*ploidy, row = +/// q*ploidy + p). Builds zero SearchTrees and never calls `dense_union()`. +/// Returns a `BatchResultSplit` with `n_samples = 1`, hap index `q*ploidy + p`. +pub fn gather_haps_readbound( + reader: &ContigReader, + region_starts: &[u32], + orig_samples: &[usize], + vk_snp_range: &[(usize, usize)], + vk_indel_range: &[(usize, usize)], + dense_snp_range: &[(usize, usize)], + dense_indel_range: &[(usize, usize)], + ploidy: usize, +) -> BatchResultSplit { + let n_q = region_starts.len(); + assert_eq!(orig_samples.len(), n_q); + assert_eq!(dense_snp_range.len(), n_q); + assert_eq!(dense_indel_range.len(), n_q); + assert_eq!(vk_snp_range.len(), n_q * ploidy); + assert_eq!(vk_indel_range.len(), n_q * ploidy); + + let snp_positions = reader.vk_snp.positions(); + let snp_keys = as_bytes(&reader.vk_snp.keys); + let indel_positions = reader.vk_indel.positions(); + let indel_keys = as_u32(&reader.vk_indel.keys); + let d_snp = reader.dense_snp.as_ref(); + let d_indel = reader.dense_indel.as_ref(); + let d_snp_pos: &[u32] = d_snp.map(|d| d.positions()).unwrap_or(&[]); + let d_indel_pos: &[u32] = d_indel.map(|d| d.positions()).unwrap_or(&[]); + + // Dense windows per query (uniform keys), decoded once. + let mut dense_snp: Vec = Vec::new(); + let mut dense_snp_range_out: Vec<(usize, usize)> = Vec::with_capacity(n_q); + let mut dense_indel: Vec = Vec::new(); + let mut dense_indel_range_out: Vec<(usize, usize)> = Vec::with_capacity(n_q); + for q in 0..n_q { + let (ss, se) = dense_snp_range[q]; + let base = dense_snp.len(); + if let Some(d) = d_snp { + let keys = as_bytes(&d.keys); + for j in ss..se { + dense_snp.push(KeyRef { + position: d_snp_pos[j], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, j)), + }); + } + } + dense_snp_range_out.push((base, dense_snp.len())); + let (is_, ie_) = dense_indel_range[q]; + let base = dense_indel.len(); + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for j in is_..ie_ { + dense_indel.push(KeyRef { position: d_indel_pos[j], key: keys[j] }); + } + } + dense_indel_range_out.push((base, dense_indel.len())); + } + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_snp_present: Vec = Vec::new(); + let mut dense_snp_present_off: Vec = vec![0]; + let mut dense_indel_present: Vec = Vec::new(); + let mut dense_indel_present_off: Vec = vec![0]; + + for q in 0..n_q { + let qs = region_starts[q]; + let orig_s = orig_samples[q]; + let (ss, se) = dense_snp_range[q]; + let (is_r, ie_r) = dense_indel_range[q]; + for p in 0..ploidy { + let hap = orig_s * ploidy + p; + let row = q * ploidy + p; + + // var_key gather. + let (vs, ve) = vk_snp_range[row]; + let mut snp_run: Vec = Vec::new(); + for (j, &pos) in snp_positions.iter().enumerate().take(ve).skip(vs) { + if qs < pos + 1 { + snp_run.push(KeyRef { + position: pos, + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(snp_keys, j)), + }); + } + } + let (vis, vie) = vk_indel_range[row]; + let mut indel_run: Vec = Vec::new(); + for j in vis..vie { + let pos = indel_positions[j]; + let v_end = pos + 1 + rvk::deletion_len(indel_keys[j]); + if qs < v_end { + indel_run.push(KeyRef { position: pos, key: indel_keys[j] }); + } + } + vk.extend_from_slice(&spine::merge_keys(vec![snp_run, indel_run])); + vk_off.push(vk.len()); + + // dense/snp presence over [ss..se). + let nbits = se - ss; + let bit_base = *dense_snp_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_snp_present.len() < need { dense_snp_present.resize(need, 0); } + if let Some(d) = d_snp { + for (k, j) in (ss..se).enumerate() { + if d.carried(hap, j) && qs < d_snp_pos[j] + 1 { + bits::set_bit(&mut dense_snp_present, bit_base + k); + } + } + } + dense_snp_present_off.push(bit_base + nbits); + + // dense/indel presence over [is_r..ie_r). + let nbits = ie_r - is_r; + let bit_base = *dense_indel_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_indel_present.len() < need { dense_indel_present.resize(need, 0); } + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for (k, j) in (is_r..ie_r).enumerate() { + let v_end = d_indel_pos[j] + 1 + rvk::deletion_len(keys[j]); + if d.carried(hap, j) && qs < v_end { + bits::set_bit(&mut dense_indel_present, bit_base + k); + } + } + } + dense_indel_present_off.push(bit_base + nbits); + } + } + + BatchResultSplit { + n_regions: n_q, + n_samples: 1, + ploidy, + vk, + vk_off, + dense_snp, + dense_snp_range: dense_snp_range_out, + dense_snp_present, + dense_snp_present_off, + dense_indel, + dense_indel_range: dense_indel_range_out, + dense_indel_present, + dense_indel_present_off, + } +} +``` + +- [ ] **Step 4: Build** + +Run: `cargo build 2>&1 | tail -20` +Expected: compiles (unused-warnings fine until Task 4 references both functions). + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs +git commit -m "feat(query): gather_ranges_readbound + gather_haps_readbound + BatchResultSplit + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Parity + zero-union + per-class tests + +**Files:** +- Modify: `tests/test_readbound_gather.rs` (add the parity/zero-union tests). + +**Interfaces:** +- Consumes: `gather_ranges_readbound`, `find_ranges`, `overlap_batch`, `read_ranges`, `BatchResult::decode_hap`, `search::search_tree_build_count`. + +**Parity strategy (byte-identical contract).** `BatchResultSplit` has a different *shape* than `BatchResult` (dense split, not unioned), so we assert parity at the **decoded-variants** level: for every `(r, s, p)`, the set of merged `(position, key)` from the read-bound result equals the set from `overlap_batch`'s union path. We build a local `readbound_decode_hap` helper that merges `vk ⋈ dense_snp ⋈ dense_indel` (mirroring how gvl will), then compare against `BatchResult::decode_hap` (the shipped oracle). + +- [ ] **Step 1: Add the read-bound decode helper + parity test** + +Append to `tests/test_readbound_gather.rs`: + +```rust +use genoray_core::query::{BatchResultSplit, HapCalls, decode_keyref_pub}; // see note below + +/// Merge vk ⋈ dense_snp ⋈ dense_indel for one hap and decode — the gvl-side +/// reconstruction, expressed as a test oracle. +fn readbound_decode_hap( + br: &BatchResultSplit, + reader: &ContigReader, + r: usize, + s: usize, + p: usize, +) -> Vec<(u32, i32)> { + use genoray_core::query::KeyRef; + let h = (r * br.n_samples + s) * br.ploidy + p; + let mut merged: Vec = br.vk[br.vk_off[h]..br.vk_off[h + 1]].to_vec(); + + let (ss, se) = br.dense_snp_range[r]; + let bit0 = br.dense_snp_present_off[h]; + for (k, j) in (ss..se).enumerate() { + if genoray_core::bits_get_bit(&br.dense_snp_present, bit0 + k) { + merged.push(br.dense_snp[j]); + } + } + let (is_, ie_) = br.dense_indel_range[r]; + let bit0 = br.dense_indel_present_off[h]; + for (k, j) in (is_..ie_).enumerate() { + if genoray_core::bits_get_bit(&br.dense_indel_present, bit0 + k) { + merged.push(br.dense_indel[j]); + } + } + // Stable position sort (var_key already ahead of dense within its own run). + merged.sort_by_key(|kr| kr.position); + merged.into_iter().map(|kr| (kr.position, kr.key as i32)).collect() +} + +#[test] +fn test_readbound_reconstructs_union_per_hap() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32), (150u32, 250u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let rb = find_ranges(&reader, ®ions, None); + let got = gather_ranges_readbound(&reader, &rb); + + assert_eq!(got.n_regions, oracle.n_regions); + assert_eq!(got.n_samples, oracle.n_samples); + assert_eq!(got.ploidy, oracle.ploidy); + + for r in 0..oracle.n_regions { + for s in 0..oracle.n_samples { + for p in 0..oracle.ploidy { + // Oracle: decode via the shipped union decode_hap, keep (pos, key). + let hc: HapCalls = oracle.decode_hap(&reader, r, s, p); + // decode_hap returns decoded alts, not raw keys — compare on the + // (position, ilen) projection that survives decode instead. + let want: Vec<(u32, i32)> = + hc.positions.iter().zip(hc.ilens.iter()).map(|(&a, &b)| (a, b)).collect(); + let got_keys = readbound_decode_hap(&got, &reader, r, s, p); + // Decode the read-bound raw keys the same way to get ilens. + let got_dec: Vec<(u32, i32)> = got_keys + .iter() + .map(|&(pos, key)| (pos, decode_keyref_pub(pos, key as u32, &reader))) + .collect(); + assert_eq!(got_dec, want, "hap (r={r}, s={s}, p={p})"); + } + } + } +} +``` + +> **Helper exports needed.** This test references three items that must be `pub` in genoray. In `src/query.rs`: make `decode_keyref` reachable via a thin public wrapper `pub fn decode_keyref_pub(position: u32, key: u32, reader: &ContigReader) -> i32` that builds a `KeyRef { position, key }`, calls the existing `decode_keyref(kr, reader.lut.as_ref())`, and returns `.ilen`. In `src/lib.rs` add `pub fn bits_get_bit(bytes: &[u8], i: usize) -> bool { bits::get_bit(bytes, i) }` (a re-export shim; `bits` is already `pub mod`). Also `pub use query::{BatchResultSplit, KeyRef, HapCalls};` if not already public — `HapCalls` (`:435`) and `KeyRef` are already `pub`. Add these shims in the same Task 3/Task 4 commit. + +- [ ] **Step 2: Run the parity test** + +Run: `cargo test --test test_readbound_gather test_readbound_reconstructs_union_per_hap 2>&1 | tail -30` +Expected: **PASS**. If a hap mismatches, the failure prints `(r, s, p)` — debug the per-class window vs. union window for that region (usually a `qs < v_end` re-check discrepancy). + +- [ ] **Step 3: Add the zero-union / zero-tree assertion** + +Append: + +```rust +#[test] +fn test_readbound_gather_builds_no_search_tree() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let rb = find_ranges(&reader, ®ions, None); + let before = search::search_tree_build_count(); + let _ = gather_ranges_readbound(&reader, &rb); + assert_eq!( + search::search_tree_build_count(), + before, + "gather_ranges_readbound must build zero SearchTrees (no dense_union)" + ); +} +``` + +- [ ] **Step 4: Add a sample-subset parity test (mirrors the `read_ranges` subset oracle)** + +Append: + +```rust +#[test] +fn test_readbound_subset_matches_full_selected_haps() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); // 2 samples, ploidy 2 + let regions = vec![(0u32, 1_000_000u32)]; + + let full = gather_ranges_readbound(&reader, &find_ranges(&reader, ®ions, None)); + // Select only sample 1. + let sub = gather_ranges_readbound(&reader, &find_ranges(&reader, ®ions, Some(&[1]))); + assert_eq!(sub.n_samples, 1); + for p in 0..reader_ploidy(&reader) { + let a = readbound_decode_hap(&sub, &reader, 0, 0, p); // selected slot 0 == orig sample 1 + let b = readbound_decode_hap(&full, &reader, 0, 1, p); // orig sample 1 + assert_eq!(a, b, "subset ploid {p}"); + } +} + +fn reader_ploidy(_r: &ContigReader) -> usize { 2 } +``` + +- [ ] **Step 5: Add the flat-gather parity test (flat ≡ cartesian for full cohort)** + +Append — proves `gather_haps_readbound` (gvl's primitive) agrees with the cartesian `gather_ranges_readbound` when the flat queries enumerate the full R×S' cohort: + +```rust +#[test] +fn test_flat_gather_matches_cartesian_full_cohort() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); // 2 samples, ploidy 2 + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + let ploidy = 2usize; + + let rb = find_ranges(&reader, ®ions, None); + let cart = gather_ranges_readbound(&reader, &rb); + + // Enumerate flat queries in the SAME order cart lays out haps: + // region-major, samples 0..S, so query q = r*S + s, orig sample = s. + let s_n = rb.n_samples; + let mut region_starts = Vec::new(); + let mut orig_samples = Vec::new(); + let mut vk_snp_range = Vec::new(); + let mut vk_indel_range = Vec::new(); + let mut dsr = Vec::new(); + let mut dir_ = Vec::new(); + for r in 0..regions.len() { + for s in 0..s_n { + region_starts.push(rb.region_starts[r]); + orig_samples.push(rb.sample_cols[s]); + dsr.push(rb.dense_snp_range[r]); + dir_.push(rb.dense_indel_range[r]); + for p in 0..ploidy { + let row = r * (s_n * ploidy) + s * ploidy + p; + vk_snp_range.push(rb.vk_snp_range[row]); + vk_indel_range.push(rb.vk_indel_range[row]); + } + } + } + let flat = gather_haps_readbound( + &reader, ®ion_starts, &orig_samples, + &vk_snp_range, &vk_indel_range, &dsr, &dir_, ploidy, + ); + + // Compare decoded per-hap. cart hap (r,s,p) == flat query q=r*S+s, ploid p. + for r in 0..regions.len() { + for s in 0..s_n { + for p in 0..ploidy { + let a = readbound_decode_hap(&cart, &reader, r, s, p); + let b = readbound_decode_hap(&flat, &reader, r * s_n + s, 0, p); + assert_eq!(a, b, "flat vs cartesian (r={r}, s={s}, p={p})"); + } + } + } +} +``` + +- [ ] **Step 6: Run the full new test file** + +Run: `cargo test --test test_readbound_gather 2>&1 | tail -30` +Expected: all five tests PASS. + +- [ ] **Step 7: Run the full suite (additive guarantee)** + +Run: `cargo test 2>&1 | tail -40` +Expected: entire genoray suite green — the shipped union path and every existing test unchanged. + +- [ ] **Step 8: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs src/lib.rs tests/test_readbound_gather.rs +git commit -m "test(query): read-bound gather parity vs union+decode, zero-tree, subset + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Python surface — `find_ranges` dict exposes the per-class ranges + +**Why:** gvl's **write** path calls genoray's Python `find_ranges(...)` and streams the resulting arrays into the cache (Plan 2, Task 2). The read path is pure-Rust and needs no Python surface, but the write cache needs `dense_snp_range` / `dense_indel_range` in the `find_ranges` dict. + +**Files:** +- Modify: `src/py_query_ranges.rs` — `bundle_to_dict` (`:~28`) and `bundle_from_dict` (`:~120`). + +**Interfaces:** +- Produces: the Python `find_ranges(...)` / `read_ranges(...)` dict gains keys `dense_snp_range` and `dense_indel_range`, each an `(R, 2)` `int64` numpy array. Existing keys byte-unchanged. + +- [ ] **Step 1: Write the failing Python parity test** + +Create `tests/test_py_ranges_readbound.py` (run via the built wheel — this is a Python test, deferred to after Task 6's wheel build; write it now, run it in Task 6): + +```python +import numpy as np +# genoray._core.PyContigReader is constructed the same way the existing +# py_query_ranges tests do; reuse that harness path if one exists in genoray's +# python test suite. Placeholder assertion of the new keys: +def _assert_keys(d): + assert "dense_snp_range" in d and "dense_indel_range" in d + for k in ("dense_snp_range", "dense_indel_range"): + a = np.asarray(d[k]) + assert a.ndim == 2 and a.shape[1] == 2 and a.dtype == np.int64 +``` + +- [ ] **Step 2: Add the keys to `bundle_to_dict`** + +In `src/py_query_ranges.rs`, in `bundle_to_dict`, next to where `dense_range` is inserted, add (mirror the exact `(R,2)` i64 conversion used for `dense_range`): + +```rust + let snp = PyArray2::from_vec2( + py, + &rb.dense_snp_range.iter().map(|&(s, e)| vec![s as i64, e as i64]).collect::>(), + )?; + dict.set_item("dense_snp_range", snp)?; + let indel = PyArray2::from_vec2( + py, + &rb.dense_indel_range.iter().map(|&(s, e)| vec![s as i64, e as i64]).collect::>(), + )?; + dict.set_item("dense_indel_range", indel)?; +``` + +> Match the *existing* `dense_range` serialization idiom in this file exactly (it may use `PyArray2::from_owned_array` over an `Array2`); if so, build `Array2::from_shape_vec((R, 2), flat)` the same way rather than `from_vec2`. Read the `dense_range` block first and copy its shape/dtype path. + +- [ ] **Step 3: Add the keys to `bundle_from_dict`** + +If `bundle_from_dict` round-trips (used by the Rust dict-parity test), parse the two new keys back into `Vec<(usize,usize)>` mirroring `dense_range`'s parse. If `bundle_from_dict` is only used for the union oracle and ignores unknown keys, guard the parse with `if let Some(...) = dict.get_item(...)` so old dicts still load. + +- [ ] **Step 4: Build + confirm the Rust dict-parity test still passes** + +Run: `cargo test --test test_ranges_split 2>&1 | tail -20` +Expected: `assert_payload_dicts_eq` still green (it checks only the original key set; new keys are additive). + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/py_query_ranges.rs tests/test_py_ranges_readbound.py +git commit -m "feat(py): find_ranges dict exposes dense_snp_range/dense_indel_range + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Build the local wheel + confirm downstream crate build + +**Files:** none (build/verification only). + +- [ ] **Step 1: Build the wheel with default features (conversion on)** + +Run (in the genoray pixi/venv used to build the wheel): +`cd /carter/users/dlaub/projects/genoray && pixi run -e dev maturin develop --release 2>&1 | tail -20` *(use genoray's actual build task; if it uses `maturin build`, produce the wheel and note its path)* +Expected: wheel builds; `python -c "import genoray"` works. + +- [ ] **Step 2: Run the Python range test from Task 5** + +Run: `pytest tests/test_py_ranges_readbound.py -q 2>&1 | tail -20` +Expected: PASS (new dict keys present, correct shape/dtype). + +- [ ] **Step 3: Confirm the query-only crate builds for the downstream path-dep** + +Run: `cargo build --no-default-features --release 2>&1 | tail -10` +Expected: clean — this is exactly what gvl's `genoray_core = { path = ..., default-features = false }` compiles (Plan 2, Task 3). + +- [ ] **Step 4: Record the exact HEAD commit for the sync contract** + +Run: `git rev-parse HEAD` +Record the commit hash in the PR description and in Plan 2's Task 3 (the gvl path-dep and this wheel MUST be this commit). + +- [ ] **Step 5: Commit any build-config changes** (only if `pixi.toml` / CI touched; otherwise skip). + +--- + +## Task 7: genoray docs / roadmap + +**Files:** +- Modify: genoray's migration/roadmap doc (search `docs/` for the search/gather-split roadmap entry). + +- [ ] **Step 1: Mark the read-bound gather + conversion feature** + +Add a roadmap entry: `conversion` query-only build ✅; per-class `find_ranges` ranges + `gather_ranges_readbound` + `BatchResultSplit` ✅ (parity vs union & `decode_hap`, zero-tree control); note it is additive to the shipped split. Link this plan. + +- [ ] **Step 2: Commit** + +```bash +git add docs/ +git commit -m "docs: read-bound per-class gather + conversion feature roadmap + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Self-Review Notes (traceability to the spec) + +- **Spec A1 (conversion feature)** → Task 1. Correction applied: gate `lib.rs:40-52` (`index_bcf_csi`/`index_vcf`) too, not only `vcf_reader.rs`. +- **Spec A2 (per-class ranges + read-bound gather + split-dense BatchResult)** → Tasks 2 (ranges), 3 (`gather_ranges_readbound` + `BatchResultSplit`), 5 (Python surface for the write cache). +- **Spec "parity vs union & decode"** → Task 4 (decoded-per-hap parity, zero-tree control, subset oracle). +- **Spec "shipped union path byte-unchanged"** → every task re-runs `test_ranges_split` / full suite; new code is additive structs/fields/functions only. +- **Spec "build local wheel + crate"** → Task 6. +- **Open question (channel factoring)** → resolved as: var_key merged per hap (unchanged) + dense split per class in `BatchResultSplit`. gvl consumes this exact shape (Plan 2, Task 4). diff --git a/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md b/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md new file mode 100644 index 00000000..89f2eda7 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md @@ -0,0 +1,981 @@ +# SVAR2 gvl Read-Bound Dataset Wiring — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the `.svar2` variant source into `gvl.write()` (a write-time 6-array offsets cache, dataset samples only) and `gvl.Dataset.__getitem__` (one all-Rust FFI call per read that gathers off the cache with **no interval-search-tree rebuild and no dense-union rebuild**), so all four output modes (haplotypes, tracks, variants, variant-windows) reconstruct in Rust, matching the SVAR1 read path. + +**Architecture:** Mirror the SVAR1 write path (`_write_from_svar` → `offsets.npy` + `svar_meta.json` + `SvarLink`). At write, per contig call genoray's Python `find_ranges(samples=dataset)` (Plan 1) and stream the 6 arrays into memmaps under `genotypes/svar2_ranges/`. At read, gvl's Rust links `genoray_core` (query-only path-dep), opens a `ContigReader` per contig **once** at `Dataset.open` (an `Svar2Store` pyclass), and per read builds a `RangesBundle`-equivalent from the cached memmap slice → calls genoray's flat `gather_haps_readbound` → gets a split-dense `BatchResultSplit` → merges `var_key ⋈ dense_snp ⋈ dense_indel` and reconstructs in Rust, decoding keys inline via `svar2-codec`. The SVAR1 path is byte-unchanged. + +**Tech Stack:** Python 3.10+ (pydantic, polars, numpy), Rust 2024 (PyO3 0.28 abi3-py310, `numpy` 0.28, `svar2-codec` path-dep, **new** `genoray_core` path-dep), `genoray` (local wheel), `pixi -e dev`, `maturin develop --release`. + +**Depends on:** the genoray PR in `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` — MUST be merged and the local wheel + crate built from the **same commit** first. Record that commit; the gvl path-dep and the genoray Python wheel must match it (the `RangesBundle`/`BatchResultSplit` field layout is the contract). + +## Global Constraints + +- **Byte-identical parity contract.** For any `contig, regions, samples` and every output mode: read-bound reconstruct ≡ the existing union-based `SparseVar2Source.reconstruct`/`realign_tracks` (which use genoray `overlap_batch`) ≡ genoray `decode` oracle ≡ SVAR1 output for an equivalent dataset — byte-for-byte / field-for-field. +- **Additive.** The SVAR1 gvl path (`_write_from_svar`, `Haps` SVAR1 branch, `reconstruct_haplotypes_fused`, etc.) stays byte-unchanged; full SVAR1 regression green (`pixi run -e dev pytest tests -q` + `cargo test`). The existing standalone `SparseVar2Source` (union path) stays as the parity oracle until Task 8 retires only its live dispatch. +- **Write caches only the dataset's samples `S'`.** `gvl.write` already selects the sample set; the cache is sized to `S'`, not the full `.svar2` cohort — exactly like `_write_from_svar`. +- **Rebuild Rust before Python tests.** After any `src/` edit: `pixi run -e dev maturin develop --release` BEFORE `pixi run -e dev pytest` — otherwise pytest imports the stale `.so`. `cargo test` compiles from source and is unaffected. +- **Full-tree before pushing shared-code changes.** `_write.py`, `_haps.py`, `_open.py`, `_reconstruct.py`, `_impl.py` are shared; run `pixi run -e dev pytest tests -q` (dataset **and** unit), not a scoped subset, before pushing. +- **Lint gate.** `pixi run -e dev ruff check python/ tests/` + `pixi run -e dev ruff format python/ tests/` + `pixi run -e dev typecheck`; `cargo fmt` + `cargo clippy --all-targets` for Rust. +- **Docs/skill gates.** `.svar2` becomes a public `write` variant source ⇒ update `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, `README.md`; keep `api.md` in sync with any new `__all__` symbol (Task 9). +- **Reject unsupported variants** (symbolic/breakend) exactly as SVAR1 does (`_reject_unsupported_variants`). +- **genoray repo path is absolute:** `/carter/users/dlaub/projects/genoray` (there is no `../genoray`). gvl already path-deps `svar2-codec = { path = "/carter/users/dlaub/projects/genoray/svar2-codec" }`. + +--- + +## File Structure + +- `python/genvarloader/_dataset/_svar2_link.py` — **new**; `Svar2Link`/`Svar2Fingerprint`/`_resolve_svar2`/`_verify_svar2_fingerprint`. Models `_svar_link.py`. *(Task 1)* +- `python/genvarloader/_dataset/_write.py` — add `.svar2` coercion arm (`:~225`), `SparseVar2` dispatch arm (`:~325`), `_write_from_svar2`, `Metadata.svar2_link` field (`:86-98`). *(Tasks 1, 2)* +- `Cargo.toml` — add `genoray_core = { path = "/carter/users/dlaub/projects/genoray", default-features = false }`. *(Task 3)* +- `src/svar2/store.rs` — **new**; `Svar2Store` pyclass wrapping `HashMap`. *(Task 3)* +- `src/svar2/mod.rs` — add `merge_hap3` (3-source merge). *(Task 4)* +- `src/ffi/mod.rs` — `reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, `decode_variants_from_svar2_readbound`. *(Tasks 4, 5, 6)* +- `src/reconstruct/mod.rs` — internal `reconstruct_haplotypes_from_split` (consumes `BatchResultSplit`). *(Task 4)* +- `src/tracks/mod.rs` — internal `shift_and_realign_tracks_from_split`. *(Task 5)* +- `src/lib.rs` — register the new pyclass + pyfunctions. *(Tasks 3–6)* +- `python/genvarloader/_dataset/_haps.py` — `Haps` source discriminant (`svar` vs `svar2`); route `_reconstruct_haplotypes` / variants; open `Svar2Store`. *(Tasks 4, 6, 7)* +- `python/genvarloader/_dataset/_open.py` — thread `svar2_link`/`svar2` override through `_build_seqs`. *(Task 7)* +- `python/genvarloader/_dataset/_impl.py` — `Dataset.open(svar2=...)` override param. *(Task 7)* +- `python/genvarloader/_dataset/_reconstruct.py` — `HapsTracks` routes to the svar2 track kernel when source is svar2. *(Task 5)* +- `python/genvarloader/_dataset/_svar2_source.py` — retire live `overlap_batch` dispatch; keep as parity oracle only. *(Task 8)* + +--- + +## Task 1: `_svar2_link.py` + `Metadata.svar2_link` field + +**Files:** +- Create: `python/genvarloader/_dataset/_svar2_link.py` +- Modify: `python/genvarloader/_dataset/_write.py:86-98` (`Metadata`) +- Test: `tests/unit/dataset/test_svar2_link.py` (new) + +**Interfaces:** +- Produces: + - `class Svar2Fingerprint(BaseModel)`: `n_variants: int`, `store_bytes: int`. + - `class Svar2Link(BaseModel)`: `relative_path: str`, `absolute_path: str`, `fingerprint: Svar2Fingerprint`. + - `def _resolve_svar2(gvl_path: Path, link: Svar2Link | None, override: Path | str | None) -> Path` — override → link.relative → link.absolute → sibling `*.svar2`. + - `def _verify_svar2_fingerprint(svar2_path: Path, link: Svar2Link | None) -> None` — no-op if `link is None`; else compare `n_variants` (from the `.svar2` index) + a canonical store-file byte count; raise `ValueError` on mismatch. + - `Metadata` gains `svar2_link: Svar2Link | None = None`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/unit/dataset/test_svar2_link.py`: + +```python +from pathlib import Path +import pytest +from genvarloader._dataset._svar2_link import ( + Svar2Link, Svar2Fingerprint, _resolve_svar2, _verify_svar2_fingerprint, +) + + +def test_resolve_prefers_override(tmp_path: Path): + real = tmp_path / "cohort.svar2" + real.mkdir() + link = Svar2Link(relative_path="nope.svar2", absolute_path="/nope.svar2", + fingerprint=Svar2Fingerprint(n_variants=1, store_bytes=1)) + assert _resolve_svar2(tmp_path, link, real) == real + + +def test_resolve_missing_override_raises(tmp_path: Path): + with pytest.raises(FileNotFoundError): + _resolve_svar2(tmp_path, None, tmp_path / "absent.svar2") + + +def test_verify_none_link_is_noop(tmp_path: Path): + _verify_svar2_fingerprint(tmp_path, None) # must not raise +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: FAIL — `ModuleNotFoundError: _svar2_link`. + +- [ ] **Step 3: Write `_svar2_link.py`** + +Create the file. Model it on `_svar_link.py` (`_resolve_svar`/`_verify_fingerprint`), changing the fingerprint to the `.svar2` store's stable identity. The `.svar2` index is at `/index.arrow` (confirm the exact filename by inspecting a real `.svar2`; genoray's `SparseVar2` exposes `.index` — the n_variants source); the canonical byte count is the summed size of the on-disk dense/var_key store files: + +```python +"""Resolution and integrity for the GVL dataset → .svar2 back-reference. + +Mirrors _svar_link.py; the fingerprint keys on the .svar2 store's stable identity +(variant count + a canonical store-file byte count) rather than SVAR1's +variant_idxs.npy, which .svar2 does not have. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +def _svar2_store_bytes(svar2_path: Path) -> int: + """Canonical, stable byte count of the .svar2 on-disk stores. Sum the sizes of + the packed dense + var_key key files across contigs, sorted for determinism.""" + total = 0 + for p in sorted(svar2_path.rglob("*")): + if p.is_file() and p.suffix in {".bin", ".npy"} and "keys" in p.name: + total += p.stat().st_size + return total + + +def _svar2_n_variants(svar2_path: Path) -> int: + import polars as pl + # .svar2 index; confirm filename against a real store (SparseVar2().index). + return pl.scan_ipc(svar2_path / "index.arrow").select(pl.len()).collect().item() + + +def _resolve_svar2( + gvl_path: Path, link: "Svar2Link | None", override: "Path | str | None" +) -> Path: + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_svar2_fingerprint(svar2_path: Path, link: "Svar2Link | None") -> None: + if link is None: + return + n_obs = _svar2_n_variants(svar2_path) + bytes_obs = _svar2_store_bytes(svar2_path) + exp = link.fingerprint + mismatches: list[str] = [] + if n_obs != exp.n_variants: + mismatches.append(f"n_variants: expected {exp.n_variants}, observed {n_obs}") + if bytes_obs != exp.store_bytes: + mismatches.append(f"store_bytes: expected {exp.store_bytes}, observed {bytes_obs}") + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) + + +def make_svar2_link(gvl_path: Path, svar2_path: Path) -> Svar2Link: + svar2_resolved = svar2_path.resolve() + return Svar2Link( + relative_path=os.path.relpath(svar2_resolved, start=gvl_path).replace(os.sep, "/"), + absolute_path=str(svar2_resolved), + fingerprint=Svar2Fingerprint( + n_variants=_svar2_n_variants(svar2_resolved), + store_bytes=_svar2_store_bytes(svar2_resolved), + ), + ) +``` + +> **Confirm before finalizing:** open a real `.svar2` (the MVP fixture) and verify (a) the index filename used by `_svar2_n_variants` and (b) that `_svar2_store_bytes`'s glob captures ≥1 stable file. Adjust the patterns to the actual layout; the contract is only that the count is deterministic and changes iff the store changes. + +- [ ] **Step 4: Add the `Metadata` field** + +In `python/genvarloader/_dataset/_write.py`, add the import near the `SvarLink` import (`:40`): + +```python +from ._svar2_link import Svar2Link +``` + +and add to `Metadata` (`:86-98`), after `svar_link`: + +```python + svar2_link: Svar2Link | None = None +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: PASS. +Run: `python -c "from genvarloader._dataset._write import Metadata; print(Metadata.model_fields['svar2_link'])"` +Expected: prints the optional field (default None) — confirms backward/forward compat (no format bump needed; `_check_dataset_format_version` gates only on major, and old datasets fill the default). + +- [ ] **Step 6: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_svar2_link.py python/genvarloader/_dataset/_write.py tests/unit/dataset/test_svar2_link.py +git commit -m "feat(dataset): Svar2Link resolution/fingerprint + Metadata.svar2_link + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: `_write_from_svar2` + write dispatch (the 6-array cache) + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` — coercion arm (`:216-234`), dispatch arm (`:285-333`), add `_write_from_svar2`. +- Test: `tests/dataset/test_write_svar2.py` (new). + +**Interfaces:** +- Consumes: genoray `SparseVar2` with `.ploidy`, `.index`, and a per-contig batched `find_ranges(contig, starts, ends, samples=...) -> dict` returning the 6 arrays: `vk_snp_range (R,S',P,2)`, `vk_indel_range (R,S',P,2)`, `dense_snp_range (R,2)`, `dense_indel_range (R,2)`, `region_starts (R,)`, `sample_cols (S',)` (Plan 1 Task 5 added `dense_snp_range`/`dense_indel_range` to the dict). +- Produces: `def _write_from_svar2(path: Path, bed: pl.DataFrame, svar2: "SparseVar2", samples: list[str], extend_to_length: bool) -> tuple[pl.DataFrame, Svar2Link]`. Writes memmaps under `path/genotypes/svar2_ranges/` + `svar2_meta.json`; returns the extended bed (max_ends folded in) and the `Svar2Link`. + +> **genoray dependency check (do first):** confirm `SparseVar2.find_ranges(contig, starts, ends, samples=...)` exists in the local genoray wheel and returns the 6-key dict. If genoray only exposes `find_ranges` on `PyContigReader` (per-contig, no batched Python entry on `SparseVar2`), add a thin `SparseVar2.find_ranges` wrapper in genoray that dispatches to the contig's `PyContigReader.find_ranges` — a small genoray-side addition; fold it into the genoray PR (Plan 1 Task 5) rather than marshalling in gvl. + +- [ ] **Step 1: Write the failing test** + +Create `tests/dataset/test_write_svar2.py`. Reuse the `.svar2` fixture the current `SparseVar2Source` tests use (search `tests/` for an existing `.svar2` path / `SparseVar2(` construction; if none, build one from the MVP `svar2_mvp` store or a synthetic genoray fixture). Skeleton: + +```python +import json +from pathlib import Path +import numpy as np +import polars as pl +import pytest + +import genvarloader as gvl +from genvarloader._dataset._svar2_link import Svar2Link + +SVAR2_FIXTURE = ... # Path to a small .svar2 store (reuse existing test fixture) + + +def test_write_svar2_emits_cache(tmp_path: Path): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + bed = pl.DataFrame({ + "chrom": ["chr1", "chr1"], + "chromStart": [0, 250], + "chromEnd": [1000, 400], + }) + out = tmp_path / "ds.gvl" + gvl.write(out, bed, variants=svar2, samples=None, overwrite=True) + + rd = out / "genotypes" / "svar2_ranges" + meta = json.loads((rd / "svar2_meta.json").read_text()) + assert set(meta) >= { + "vk_snp_range", "vk_indel_range", "dense_snp_range", + "dense_indel_range", "region_starts", "sample_cols", + } + # metadata.json carries the link + ploidy. + md = json.loads((out / "metadata.json").read_text()) + assert md["svar2_link"] is not None + assert md["ploidy"] == svar2.ploidy + Svar2Link.model_validate(md["svar2_link"]) # shape check +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: FAIL — `gvl.write` raises "unrecognized file extension" / doesn't dispatch `SparseVar2`. + +- [ ] **Step 3: Add the `.svar2` coercion arm** + +In `_write.py`, add the import near the `SparseVar` import (`:19`): change `from genoray import PGEN, VCF, Reader, SparseVar` to also import `SparseVar2`: + +```python +from genoray import PGEN, VCF, Reader, SparseVar, SparseVar2 +``` + +In the coercion block (`:216-234`), add an arm before the `else` that raises: + +```python + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) +``` + +- [ ] **Step 4: Add the dispatch arm** + +In the genotype-writing dispatch (`:325-330`), after the `SparseVar` branch, add: + +```python + elif isinstance(variants, SparseVar2): + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link +``` + +(`metadata["ploidy"] = variants.ploidy` at `:330` already runs for all sources, including `SparseVar2`.) + +- [ ] **Step 5: Write `_write_from_svar2`** + +Add the function near `_write_from_svar` (`:961`). It streams the 6 arrays per contig into memmaps and computes `max_ends` for the bed (mirroring SVAR1's end-extension). Because the read-bound gather needs per-`(region, selected-sample, ploid)` var_key ranges and per-region dense ranges, the memmaps are shaped `(R, S', P, 2)` / `(R, 2)` / `(R,)` / `(S',)`: + +```python +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: "SparseVar2", + samples: list[str], + extend_to_length: bool, +) -> "tuple[pl.DataFrame, Svar2Link]": + _reject_unsupported_variants(svar2.index, "SVAR2") + + out_dir = path / "genotypes" / "svar2_ranges" + out_dir.mkdir(parents=True, exist_ok=True) + + R, S, P = bed.height, len(samples), svar2.ploidy + vk_snp = np.memmap(out_dir / "vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap(out_dir / "vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + dense_snp = np.memmap(out_dir / "dense_snp_range.npy", np.int64, "w+", shape=(R, 2)) + dense_indel = np.memmap(out_dir / "dense_indel_range.npy", np.int64, "w+", shape=(R, 2)) + region_starts = np.memmap(out_dir / "region_starts.npy", np.int64, "w+", shape=(R,)) + # sample_cols: selected slot -> original sample index (same for every contig). + sample_cols_full = np.asarray( + [svar2.available_samples.index(s) for s in samples], np.int64 + ) + np.save(out_dir / "sample_cols.npy", sample_cols_full) + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "vk_snp_range": {"shape": [R, S, P, 2], "dtype": " **`max_ends` NOTE.** SVAR1 derives `max_ends` from the per-region max `v_idx` and `v_ends[v_idx]`. For SVAR2 the equivalent is the farthest `v_end` among the region's overlapping variants (var_key + dense) for the dataset's samples. If genoray exposes a helper (`_region_max_ends` / a field on the `find_ranges` dict), use it. If not, add `max_end_range` per region to genoray's `find_ranges` dict (small genoray addition, fold into Plan 1 Task 5) rather than recomputing in Python. Confirm the exact source before implementing; do **not** guess a formula — measure it against `_write_from_svar`'s `max_ends` on a shared fixture (Step 7). + +- [ ] **Step 6: Rebuild not needed (Python-only) — run the write test** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS (cache + meta + link written). + +- [ ] **Step 7: Add a max_ends parity assertion** + +Add to the test file: write the *same* regions from an equivalent `.svar` (SVAR1) and `.svar2` of the same cohort (the MVP fixtures are matched); assert the two extended `chromEnd` columns are equal. This pins the end-extension semantics. + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS. + +- [ ] **Step 8: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "feat(write): _write_from_svar2 6-array ranges cache + dispatch + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: gvl links `genoray_core`; `Svar2Store` pyclass + +**Files:** +- Modify: `Cargo.toml` (`[dependencies]`) +- Create: `src/svar2/store.rs` +- Modify: `src/svar2/mod.rs` (`pub mod store;`), `src/lib.rs` (register pyclass) +- Test: `tests/unit/dataset/test_svar2_store.py` (new) + +**Interfaces:** +- Produces: `Svar2Store` pyclass. Python: `Svar2Store(store_path: str, contigs: list[str], n_samples: int, ploidy: int)` opens one `genoray_core::query::ContigReader` per contig, held for the store's lifetime (the SVAR2 analog of SVAR1's once-built `_HapsFfiStatic`). Rust-internal: `fn reader(&self, contig: &str) -> &ContigReader`. + +- [ ] **Step 1: Add the `genoray_core` path-dep** + +In `Cargo.toml` `[dependencies]`, after the `svar2-codec` line, add: + +```toml +genoray_core = { path = "/carter/users/dlaub/projects/genoray", default-features = false } +``` + +`default-features = false` drops `conversion` (htslib) and `extension-module`, yielding the query-only core (Plan 1 Task 1). Confirm the genoray checkout is at the commit recorded in Plan 1 Task 6. + +- [ ] **Step 2: Write the failing test** + +Create `tests/unit/dataset/test_svar2_store.py`: + +```python +import pytest +from genvarloader.genvarloader import Svar2Store # compiled ext + +SVAR2_FIXTURE = ... # same fixture as Task 2 + + +def test_store_opens_contigs(): + store = Svar2Store(str(SVAR2_FIXTURE), ["chr1"], n_samples=2, ploidy=2) + assert store.contigs() == ["chr1"] +``` + +- [ ] **Step 3: Write `src/svar2/store.rs`** + +```rust +use std::collections::HashMap; + +use genoray_core::query::ContigReader; +use pyo3::exceptions::PyIOError; +use pyo3::prelude::*; + +/// Opened once at Dataset.open; holds one query-only ContigReader per contig for +/// the store's lifetime (SVAR2 analog of SVAR1's cached _HapsFfiStatic). +#[pyclass] +pub struct Svar2Store { + readers: HashMap, +} + +impl Svar2Store { + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } +} + +#[pymethods] +impl Svar2Store { + #[new] + fn new(store_path: &str, contigs: Vec, n_samples: usize, ploidy: usize) -> PyResult { + let mut readers = HashMap::with_capacity(contigs.len()); + for c in contigs { + let r = ContigReader::open(store_path, &c, n_samples, ploidy) + .map_err(|e| PyIOError::new_err(format!("open contig {c}: {e}")))?; + readers.insert(c, r); + } + Ok(Self { readers }) + } + + fn contigs(&self) -> Vec { + let mut v: Vec = self.readers.keys().cloned().collect(); + v.sort(); + v + } +} +``` + +Add `pub mod store;` to `src/svar2/mod.rs`, and in `src/lib.rs`'s `#[pymodule]` add `m.add_class::()?;`. + +- [ ] **Step 4: Build + run** + +Run: `cargo build 2>&1 | tail -20` (first build compiles genoray_core query-only; may take a bit). +Then: `pixi run -e dev maturin develop --release 2>&1 | tail -20` +Then: `pixi run -e dev pytest tests/unit/dataset/test_svar2_store.py -q` +Expected: PASS. + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add Cargo.toml Cargo.lock src/svar2/store.rs src/svar2/mod.rs src/lib.rs tests/unit/dataset/test_svar2_store.py +git commit -m "feat(rust): link genoray_core (query-only) + Svar2Store pyclass + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Read-bound haplotype kernel (all-Rust, one FFI call) + +**Files:** +- Modify: `src/svar2/mod.rs` — add `merge_hap3`. +- Modify: `src/reconstruct/mod.rs` — add `reconstruct_haplotypes_from_split`. +- Modify: `src/ffi/mod.rs` — add `reconstruct_haplotypes_from_svar2_readbound`. +- Modify: `src/lib.rs` — register it. +- Test: `tests/dataset/test_svar2_readbound_haps.py` (parity vs union oracle), `src/svar2/mod.rs` `#[cfg(test)]` (merge_hap3 unit). + +**Interfaces:** +- Consumes: `genoray_core::query::{gather_haps_readbound, BatchResultSplit, KeyRef}`; `ContigReader::lut_arrays()` (in-Rust LUT, no Python marshalling); existing `svar2::decode_alt`, `reconstruct::reconstruct_haplotype_core`. +- Produces the FFI pyfunction: + ``` + reconstruct_haplotypes_from_svar2_readbound( + store: &Svar2Store, contig: &str, + region_starts: (n_q,) u32, orig_samples: (n_q,) i64, + vk_snp_range: (n_q*P, 2) i64, vk_indel_range: (n_q*P, 2) i64, + dense_snp_range: (n_q, 2) i64, dense_indel_range: (n_q, 2) i64, + region_bounds: (n_q, 2) i32, # [start, end) per query, post-jitter + shifts: (n_q, P) i32, + ref_: (n_ref,) u8, ref_offsets: (n_contig+1,) i64, + pad_char: u8, output_length: i64, parallel: bool, + ) -> (out_data u8, out_offsets i64) + ``` + `n_q` = number of `(region, sample)` query rows; ploidy `P` inferred from `shifts.shape[1]`. + +- [ ] **Step 1: Write the `merge_hap3` unit test** + +In `src/svar2/mod.rs` `#[cfg(test)]`, add: + +```rust +#[test] +fn merge_hap3_is_position_sorted_stable() { + // vk at pos 5,20; dense_snp at 10; dense_indel at 10,30. + let vk = [(5u32, 100u32), (20, 200)]; + let dsnp = [(10u32, 300u32)]; + let dindel = [(10u32, 400u32), (30, 500)]; + let out = merge_hap3(&vk, &dsnp, &dindel); + let positions: Vec = out.iter().map(|&(p, _)| p).collect(); + assert_eq!(positions, vec![5, 10, 10, 20, 30]); + // On the pos-10 tie: vk-source first (none here), then dense_snp before dense_indel. + assert_eq!(out[1], (10, 300)); + assert_eq!(out[2], (10, 400)); +} +``` + +- [ ] **Step 2: Run it (fails to compile)** + +Run: `cargo test svar2::mod 2>&1 | tail -20` (or `cargo test merge_hap3`) +Expected: FAIL — `merge_hap3` undefined. + +- [ ] **Step 3: Write `merge_hap3`** + +In `src/svar2/mod.rs`, add. It mirrors the existing 2-source `merge_hap` (`:30-51`) but takes three already-position-ordered inputs and stable-sorts by position (vk pushed first, then dense_snp, then dense_indel — matching genoray's `merge_keys(vec![vk, dense_snp, dense_indel])` tie order): + +```rust +/// Merge one hap's var_key ⋈ dense_snp ⋈ dense_indel into one position-sorted +/// (pos, key) list. Stable: on a shared position, var_key precedes dense_snp +/// precedes dense_indel — the order genoray's decode oracle uses. +pub fn merge_hap3( + vk: &[(u32, u32)], + dense_snp: &[(u32, u32)], + dense_indel: &[(u32, u32)], +) -> Vec<(u32, u32)> { + let mut a: Vec<(u32, u32)> = Vec::with_capacity(vk.len() + dense_snp.len() + dense_indel.len()); + a.extend_from_slice(vk); + a.extend_from_slice(dense_snp); + a.extend_from_slice(dense_indel); + a.sort_by_key(|&(p, _)| p); // stable + a +} +``` + +- [ ] **Step 4: Run the unit test** + +Run: `cargo test merge_hap3 2>&1 | tail -20` +Expected: PASS. + +- [ ] **Step 5: Write the internal `reconstruct_haplotypes_from_split`** + +In `src/reconstruct/mod.rs`, add a function consuming a `BatchResultSplit`. It mirrors the existing `reconstruct_haplotypes_from_svar2` (`:611`) but sources per-hap merged keys from the split result via `merge_hap3` (extracting present dense entries per hap from the presence bitmasks), and decodes via `svar2::decode_alt` with LUT bytes from the reader. Signature: + +```rust +use genoray_core::query::BatchResultSplit; + +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_split( + mut out: ArrayViewMut1, + out_offsets: ArrayView1, + region_bounds: ArrayView2, // (n_q, 2) + shifts: ArrayView2, // (n_q, P) + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], + ref_: ArrayView1, + ref_offsets: ArrayView1, + pad_char: u8, + parallel: bool, +) { + // For each query q and ploid p (hap h = q*P + p): + // vk_h = br.vk[br.vk_off[h]..br.vk_off[h+1]] -> (pos, key) + // dsnp = present entries of br.dense_snp[br.dense_snp_range[q]] via br.dense_snp_present + // dind = present entries of br.dense_indel[br.dense_indel_range[q]] via br.dense_indel_present + // merged = svar2::merge_hap3(&vk_h, &dsnp, &dind) + // provide(i) decodes merged[i].key via svar2::decode_alt(key, lut_bytes, lut_off), + // patching an empty pure-DEL alt with the anchor ref[pos] byte (as the existing + // reconstruct_haplotypes_from_svar2 does at :690-710). + // reconstruct_haplotype_core(merged.len(), provide, shift, contig_ref, ref_start, + // out_view, pad_char, None, None, None) + // Parallel path: split_at_mut chain over out, one disjoint chunk per hap (mirror + // reconstruct_haplotypes_from_svar2's parallel arm at :740-879). + // ... +} +``` + +Extract "present dense entries for hap h in query q" with a small helper: + +```rust +fn present_dense( + dense: &[genoray_core::query::KeyRef], + range: (usize, usize), + present: &[u8], + bit0: usize, +) -> Vec<(u32, u32)> { + let (s, e) = range; + let mut out = Vec::new(); + for (k, j) in (s..e).enumerate() { + if genoray_core::bits_get_bit(present, bit0 + k) { + out.push((dense[j].position, dense[j].key)); + } + } + out +} +``` + +(`genoray_core::bits_get_bit` is the shim added in Plan 1 Task 4; `KeyRef` fields `position`/`key` are `pub`.) + +> **Sizing pass.** Before reconstruct, size outputs exactly like `reconstruct_haplotypes_from_svar2`: compute per-hap applied-ilen diffs (a `hap_diffs_split` analog of `svar2::hap_diffs_svar2` operating over the merged keys), prefix-sum to `out_offsets`, `uninit_output`. Reuse `svar2::hap_diffs_svar2`'s clipping logic; it only needs `(pos, ilen)` per merged key, which `decode_alt` yields as `v_diff`. + +- [ ] **Step 6: Write the FFI `reconstruct_haplotypes_from_svar2_readbound`** + +In `src/ffi/mod.rs`, mirror the fused SVAR1 entry (`:618`) / the existing `reconstruct_haplotypes_from_svar2` (`:768`). Body: look up `reader = store.reader(contig)`; convert the `(n,2)` range arrays to `Vec<(usize,usize)>` and `orig_samples` to `Vec`; call `genoray_core::query::gather_haps_readbound(reader, ®ion_starts, &orig_samples, &vk_snp_range, &vk_indel_range, &dense_snp_range, &dense_indel_range, ploidy)`; get `(lut_bytes, lut_off)` from `reader.lut_arrays()`; run the sizing pass; allocate; call `reconstruct_haplotypes_from_split`; return `(out_data, out_offsets)`. Register in `src/lib.rs`. + +- [ ] **Step 7: Write the parity test (vs the union oracle)** + +Create `tests/dataset/test_svar2_readbound_haps.py`. The oracle is the existing `SparseVar2Source.reconstruct` (genoray `overlap_batch`, union path) — the spec's byte-identical contract: + +```python +import numpy as np +from genvarloader._dataset._svar2_source import SparseVar2Source +from genvarloader._dataset._svar2_store_py import build_readbound_haps # thin py wrapper (Task 7) + +SVAR2_FIXTURE = ... # same fixture + + +def test_readbound_haps_match_union_oracle(): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + contig = "chr1" + regions = [(0, 1000), (250, 400), (150, 250)] + ref, ref_offsets, pad = _load_contig_ref(contig) # helper: contig bytes + + union = SparseVar2Source(svar2).reconstruct( + contig, regions, ref, ref_offsets, pad, shifts=None, output_length=-1 + ) + readbound = build_readbound_haps( # opens Svar2Store, slices no cache (direct find_ranges), + svar2, contig, regions, ref, ref_offsets, pad, shifts=None, output_length=-1 + ) + # Ragged equality: same offsets + same bytes. + assert np.array_equal(np.asarray(union.offsets), np.asarray(readbound.offsets)) + assert np.array_equal(union.data.view("u1"), readbound.data.view("u1")) +``` + +> `build_readbound_haps` is a thin Python test helper that, for the given regions, computes the flat per-query `(region_starts, orig_samples, vk/dense ranges)` via `svar2.find_ranges` (full cohort, R×S' flattened to n_q = R*S rows in region-major sample order), opens an `Svar2Store`, and calls `reconstruct_haplotypes_from_svar2_readbound`. It exercises the exact FFI the dataset read path uses (Task 7), decoupled from cache I/O. Put it in a small `python/genvarloader/_dataset/_svar2_store_py.py` alongside the pyclass usage. + +- [ ] **Step 8: Rebuild + run parity** + +Run: `pixi run -e dev maturin develop --release 2>&1 | tail -20` +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py -q` +Expected: PASS (byte-identical to the union oracle). + +- [ ] **Step 9: cargo test + fmt + clippy + commit** + +Run: `cargo test 2>&1 | tail -30` (merge_hap3 + any Rust svar2 tests green). + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add src/svar2/mod.rs src/reconstruct/mod.rs src/ffi/mod.rs src/lib.rs python/genvarloader/_dataset/_svar2_store_py.py tests/dataset/test_svar2_readbound_haps.py +git commit -m "feat(rust): read-bound svar2 haplotype kernel (gather_haps_readbound + merge_hap3) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Read-bound tracks kernel + +**Files:** +- Modify: `src/tracks/mod.rs` (add `shift_and_realign_tracks_from_split`), `src/ffi/mod.rs` (add `shift_and_realign_tracks_from_svar2_readbound`), `src/lib.rs`. +- Modify: `python/genvarloader/_dataset/_svar2_store_py.py` (a `build_readbound_tracks` helper). +- Test: `tests/dataset/test_svar2_readbound_tracks.py`. + +**Interfaces:** +- Produces `shift_and_realign_tracks_from_svar2_readbound(store, contig, , tracks, track_offsets, params, strategy_id, base_seed, parallel) -> (out_data f32, out_offsets i64)`. Tracks need only `ilen`/`deletion_len` per merged key (no allele bytes), so decoding is cheaper — but reuse the **same** `gather_haps_readbound` + `merge_hap3` merge so the two modes read identical variant sets. + +- [ ] **Step 1: Write the failing parity test** + +Create `tests/dataset/test_svar2_readbound_tracks.py`, oracle = `SparseVar2Source(svar2).realign_tracks(...)`: + +```python +def test_readbound_tracks_match_union_oracle(): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + contig, regions = "chr1", [(0, 1000), (250, 400)] + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + union = SparseVar2Source(svar2).realign_tracks( + contig, regions, tracks, toff, params, strat, seed, shifts=None) + rb = build_readbound_tracks( + svar2, contig, regions, tracks, toff, params, strat, seed, shifts=None) + import numpy as np + assert np.array_equal(np.asarray(union.offsets), np.asarray(rb.offsets)) + assert np.allclose(union.data, rb.data, equal_nan=True) +``` + +- [ ] **Step 2: Run to confirm it fails** — `pixi run -e dev pytest tests/dataset/test_svar2_readbound_tracks.py -q` → FAIL (`build_readbound_tracks`/FFI missing). + +- [ ] **Step 3: Implement the Rust track-from-split kernel + FFI**, mirroring `shift_and_realign_tracks_from_svar2` (`src/tracks/mod.rs:698`, `src/ffi/mod.rs:897`) but sourcing merged keys from `BatchResultSplit` via `merge_hap3` (ilen only). Register in `src/lib.rs`. + +- [ ] **Step 4: Rebuild + run** — `pixi run -e dev maturin develop --release` then the test → PASS. + +- [ ] **Step 5: fmt + clippy + lint + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/tracks/mod.rs src/ffi/mod.rs src/lib.rs python/genvarloader/_dataset/_svar2_store_py.py tests/dataset/test_svar2_readbound_tracks.py +git commit -m "feat(rust): read-bound svar2 track re-alignment kernel + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Read-bound variants / variant-windows kernel + +**Files:** +- Modify: `src/ffi/mod.rs` (add `decode_variants_from_svar2_readbound`), `src/lib.rs`. +- Modify: `python/genvarloader/_dataset/_svar2_store_py.py` (`build_readbound_variants`). +- Test: `tests/dataset/test_svar2_readbound_variants.py`. + +**Interfaces:** +- Produces `decode_variants_from_svar2_readbound(store, contig, ) -> RaggedVariants-backing arrays` (per-hap positions, ilens, ALT bytes + offsets), decoding each merged key via `svar2::decode_alt` (`Inline`/`PureDel`/`Lookup`, LUT from `reader.lut_arrays()`), mirroring genoray `decode_hap`. `variant-windows` reuses the same decode + the existing window-materialization gvl already applies for SVAR1 variants. + +- [ ] **Step 1: Write the failing parity test** — oracle: decode the same regions via genoray's `read_ranges(...).decode_hap` per hap (or the existing `SparseVar2Source` variants path if present). Assert per-hap `(positions, ilens, alts)` equal. + +- [ ] **Step 2: Run → FAIL.** + +- [ ] **Step 3: Implement `decode_variants_from_svar2_readbound`** — build `BatchResultSplit` via `gather_haps_readbound`, `merge_hap3` per hap, `decode_alt` each key into the `RaggedVariants` SoA. Register in `src/lib.rs`. + +- [ ] **Step 4: Rebuild + run → PASS.** + +- [ ] **Step 5: fmt + clippy + lint + commit** (message: `feat(rust): read-bound svar2 variants/variant-windows decode`). + +--- + +## Task 7: Dataset read dispatch wiring (Haps discriminant, open, __getitem__) + +**Files:** +- Modify: `python/genvarloader/_dataset/_haps.py` — `Haps.from_path` opens an `Svar2Store` when the dataset is svar2-backed; add a `source` discriminant; `_reconstruct_haplotypes` and the variants path branch on it; slice the cache per query. +- Modify: `python/genvarloader/_dataset/_open.py:143-173` (`_build_seqs`) — thread `svar2_link` + `svar2` override. +- Modify: `python/genvarloader/_dataset/_impl.py:124-199` (`Dataset.open`) — add keyword-only `svar2: str | Path | None = None`. +- Modify: `python/genvarloader/_dataset/_reconstruct.py:130-287` (`HapsTracks.__call__`) — route to the svar2 track kernel when source is svar2. +- Test: `tests/dataset/test_svar2_dataset.py` (end-to-end, all four modes). + +**Interfaces:** +- Consumes: `svar2_ranges/` cache (Task 2), `Svar2Store` (Task 3), the three read-bound kernels (Tasks 4–6), `_resolve_svar2`/`_verify_svar2_fingerprint` (Task 1). +- Produces: a dataset whose `Haps` carries `source: Literal["svar", "svar2"]`; `Dataset.open(path, svar2=)` resolves + fingerprints the `Svar2Link`; `dataset[region, sample]` issues **one** FFI call to the appropriate read-bound kernel — no interval search, no dense-union at read. + +**The cache-slice → FFI mapping (the hot loop).** For a query block of `n_q` rows, each row `q` is a `(region_idx r_q, sample_idx si_q)` pair with post-jitter bounds `[start_q, end_q)`: +- `region_starts[q] = start_q` (post-jitter). +- `orig_samples[q] = sample_cols[si_q]` (from the cache's `sample_cols.npy`). +- `vk_snp_range[q*P + p] = cache.vk_snp_range[r_q, si_q, p]`, likewise `vk_indel_range`. +- `dense_snp_range[q] = cache.dense_snp_range[r_q]`, `dense_indel_range[q] = cache.dense_indel_range[r_q]` (dense is per-region, sample-independent). +- Gather these with numpy fancy-indexing on the memmapped cache (sub-linear; no per-read search), pass to the FFI. + +- [ ] **Step 1: Write the end-to-end test (all four modes) — failing** + +Create `tests/dataset/test_svar2_dataset.py`. Build two datasets from matched MVP fixtures — one from `.svar` (SVAR1), one from `.svar2` — over the same bed/samples/reference, and assert equality per mode: + +```python +import numpy as np +import genvarloader as gvl + + +def _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref): + from genoray import SparseVar, SparseVar2 + d1 = tmp_path / "d1.gvl"; d2 = tmp_path / "d2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture), overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture), overwrite=True) + return gvl.Dataset.open(d1, reference=ref), gvl.Dataset.open(d2, reference=ref) + + +def test_svar2_haplotypes_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +def test_svar2_tracks_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref, bigwig): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_tracks(bigwig)[:, :] + b = ds2.with_tracks(bigwig)[:, :] + assert np.allclose(np.asarray(a), np.asarray(b), equal_nan=True) + + +def test_svar2_variants_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants")[:, :] + b = ds2.with_seqs("variants")[:, :] + assert a == b # RaggedVariants equality (positions/ilens/alts) +``` + +> These fixtures (matched `.svar`/`.svar2` of the same cohort, a shared bed, a reference, a bigwig) should be added to `tests/dataset/conftest.py`; reuse the MVP `svar2_mvp` chr21 stores or a small synthetic pair. If a matched SVAR1 store isn't available, use the union-path `SparseVar2Source` as the oracle instead of SVAR1 (weaker cross-format check but still the byte-identical contract). + +- [ ] **Step 2: Run → FAIL** (svar2 dataset opens but reconstructs via the SVAR1 path or errors — no dispatch yet). + +- [ ] **Step 3: Add the `Svar2Store` open + discriminant in `Haps.from_path`** + +In `_haps.py:363`, add a branch: when `path/genotypes/svar2_ranges/svar2_meta.json` exists (the svar2 discriminant, analogous to SVAR1's `svar_meta.json` at `:388`), resolve the `.svar2` via `_resolve_svar2(path, svar2_link, svar2_override)`, `_verify_svar2_fingerprint(...)`, memmap the six cache arrays, open `Svar2Store(str(svar2_path), contigs, n_samples=len(samples), ploidy=ploidy)`, and construct the `Haps` with `source="svar2"` (add the field to `Haps`), the store, and the cache arrays. Leave the existing SVAR1 branch as `source="svar"`. `_build_seqs` already forwards `svar_link`/`svar_override`; add `svar2_link`/`svar2_override` params to `Haps.from_path` and forward them from `_build_seqs`. + +- [ ] **Step 4: Branch `_reconstruct_haplotypes` on `source`** + +In `_haps.py:809`, at the top of `_reconstruct_haplotypes`, if `self.source == "svar2"`, build the flat per-query FFI inputs from the cache (the mapping above) and call `reconstruct_haplotypes_from_svar2_readbound(self.store, contig, ...)`, returning the `Ragged`. Else fall through to the unchanged SVAR1 `reconstruct_haplotypes_fused` path. Do the same source-branch for the variants path (Task 6 kernel) and, in `_reconstruct.py`'s `HapsTracks.__call__`, for the track kernel (Task 5). + +> **Splice / annotated / RC / AF-keep.** This plan wires the four Phase-1 modes for the **unspliced, no-keep, no-in-kernel-RC** path (matching the current svar2 kernels' "first cut minimal"). If the test bed exercises jitter only (no splice, no `min_af`/`max_af`, no annotated), that's covered. Guard the svar2 branch to `raise NotImplementedError` for splice plans / `keep` masks / annotated kind / in-kernel `to_rc` until those `_from_svar2` kernels exist (out of scope here — see the spec's "Annotated out of scope" and the SVAR2 kernel gaps). Add explicit `raise` guards so a user hitting an unsupported combo gets a clear error, not silently-wrong output. + +- [ ] **Step 5: Add `Dataset.open(svar2=...)`** + +In `_impl.py:124`, add `svar2: str | Path | None = None` (keyword-only, next to `svar`), thread it into `OpenRequest(..., svar2=svar2)`, and in `_open.py` forward `self.svar2` to `_build_seqs` → `Haps.from_path(svar2_override=...)`. Mirror the two `@overload`s if they enumerate kwargs. + +- [ ] **Step 6: Rebuild + run the end-to-end tests** + +Run: `pixi run -e dev maturin develop --release` +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: all four modes PASS (byte-identical to SVAR1 / union oracle). + +- [ ] **Step 7: Full SVAR1 regression (additive guarantee)** + +Run: `pixi run -e dev pytest tests -q` +Expected: entire tree green — SVAR1 path byte-unchanged, unit + dataset both covered. + +- [ ] **Step 8: Lint + fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_haps.py python/genvarloader/_dataset/_open.py python/genvarloader/_dataset/_impl.py python/genvarloader/_dataset/_reconstruct.py tests/dataset/test_svar2_dataset.py tests/dataset/conftest.py +git commit -m "feat(dataset): wire svar2 read dispatch (Svar2Store, source discriminant, svar2= override) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 8: Retire the live `overlap_batch` dispatch in `_svar2_source.py` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_source.py`. + +**Interfaces:** `SparseVar2Source` stays as a **parity oracle** (used by Tasks 4–6 tests) but is no longer on any live read path. Remove the `TODO(svar2-dataset-dispatch)` marker since dispatch now lives in `Haps` (Task 7). + +- [ ] **Step 1: Update the module docstring** — replace the `TODO(svar2-dataset-dispatch)` paragraph with a note that live dispatch is wired in `Haps` (read-bound, `_haps.py`) and this adapter is retained only as the union-path parity oracle for tests. + +- [ ] **Step 2: Confirm nothing imports it on a live path** + +Run: `pixi run -e dev python -c "import genvarloader"` and search: `rtk grep "SparseVar2Source" python/` — expect references only in `_svar2_source.py` and tests, not in `_haps.py`/`_impl.py`/`_open.py`. + +- [ ] **Step 3: Run the oracle tests + commit** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_tracks.py -q` +Expected: PASS (oracle still callable). + +```bash +git add python/genvarloader/_dataset/_svar2_source.py +git commit -m "refactor(dataset): retire svar2 live overlap_batch dispatch (oracle-only) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 9: Docs / roadmap / skill / api.md audit + +**Files:** +- Modify: `skills/genvarloader/SKILL.md`, `docs/source/{write.md,format.md,faq.md,api.md}`, `README.md`, `docs/roadmaps/rust-migration.md`. + +- [ ] **Step 1: Document `.svar2` as a `write` variant source** — in `SKILL.md` and `docs/source/write.md`: `.svar2` accepted alongside `.svar`/VCF/PGEN; note it produces a read-bound ranges cache; `Dataset.open(..., svar2=)` mirrors `svar=`. In `format.md`: document the `genotypes/svar2_ranges/` layout (the six arrays + `svar2_meta.json`) and the `metadata.json` `svar2_link` field. In `faq.md`/`README.md`: `.svar2`'s on-disk size advantage; the read path builds no interval tree / no dense union. + +- [ ] **Step 2: api.md ↔ `__all__` sync** — if any new public symbol was added to `python/genvarloader/__init__.py` `__all__` (e.g. a `migrate_svar2_link` analog), add its autodoc entry. Run the gate: + +```bash +pixi run -e dev python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. + +- [ ] **Step 3: Roadmap** — in `docs/roadmaps/rust-migration.md`, tick the read-bound SVAR2 wiring; record the parity results (all four modes byte-identical to SVAR1/union oracle) and set the phase marker + link this plan and the genoray plan. Note the byte-identical parity contract is satisfied. + +- [ ] **Step 4: Commit** + +```bash +git add skills/ docs/ README.md +git commit -m "docs: .svar2 as a write variant source + read-bound wiring (skill, format, faq, roadmap) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 10: Relocate + re-run the MVP benchmark + +**Files:** none in-repo (benchmark scripts live in the relocated `svar2_mvp`). + +- [ ] **Step 1: Relocate the MVP tree** + +```bash +mv /carter/users/dlaub/repos/for_loukik/svar2_mvp /carter/users/dlaub/projects/svar2_mvp +``` +Then update any absolute paths in `svar2_mvp/build_source.sh` and the benchmark driver (`rtk grep "for_loukik" /carter/users/dlaub/projects/svar2_mvp`). + +- [ ] **Step 2: Re-run the SVAR1-vs-SVAR2 `Dataset.__getitem__` benchmark** on chr21 germline (3202) + somatic (16007) after wiring — latency (same-session before/after within one allocation) + store size. Follow the profiling memory: profile the Python process with `perf` (paranoid=2, no sudo), not `py-spy --native`. Report the perf DSO split. + +- [ ] **Step 3: Verify success criteria** — the warm SVAR2 read shows **neither** `SearchTree::build` **nor** a dense-union rebuild (the DSO split flips from ~80% genoray to gvl-kernel-bound, like SVAR1), and SVAR2's store-size advantage holds. Record numbers as a **relative** same-session before/after (absolute wall-clock is not comparable across allocations on shared Carter nodes — per the perf-gate memory). + +- [ ] **Step 4: Record results** in the roadmap checkpoint (Task 9's roadmap file) and commit any driver-path edits that live inside the repo (if the benchmark driver is tracked). + +--- + +## Self-Review Notes (traceability to the spec) + +- **Spec Component B (write)** → Tasks 1 (`_svar2_link.py` + `Metadata.svar2_link`), 2 (`_write_from_svar2` + coercion + dispatch + 6-array cache). Reject unsupported variants via `_reject_unsupported_variants` (Task 2 Step 5). +- **Spec Component C (read, all-Rust)** → Task 3 (`genoray_core` path-dep + `Svar2Store` opened once = the SVAR1 `ffi_static` analog), Task 4 (one FFI call, `gather_haps_readbound` + `merge_hap3`, LUT via `reader.lut_arrays()` — no numpy round-trip, no Python `gather_ranges`), Task 7 (dispatch discriminant retiring `TODO(svar2-dataset-dispatch)`). +- **Spec "all four output modes, all Rust"** → Task 4 (haplotypes), 5 (tracks), 6 (variants + variant-windows). Annotated explicitly out of scope (guarded `NotImplementedError`, Task 7 Step 4). +- **Spec cache format** → Task 2 Step 5 (six arrays under `svar2_ranges/` + `svar2_meta.json`; O(offsets), bulk stays in the `.svar2`). +- **Spec parity & testing** → union-oracle parity per mode (Tasks 4–6), SVAR1 cross-format + additive regression (Task 7 Steps 6–7), perf verification (Task 10). +- **Spec "no interval search / no contig-wide union at read"** → guaranteed structurally: the read path calls only `gather_haps_readbound` (Plan 1: zero `SearchTree`, no `dense_union`); verified in Plan 1 Task 4's zero-tree test and Task 10's DSO split. +- **Spec open questions** → (channel factoring) resolved in Plan 1 (`BatchResultSplit`: vk merged + dense split). (wheel↔path-dep sync) Task 3 pins the genoray commit; the `Svar2Fingerprint` guards store identity at open. (format version) `svar2_link` is additive/defaulted — no bump (Task 1 Step 5). (arbitrary-(region,sample) mapping) resolved via Plan 1's flat `gather_haps_readbound` + Task 7's cache-slice mapping. +- **Resolved spec inaccuracies** (carried from Plan 1): genoray path is absolute, not `../genoray`; `DenseView` in `query.rs`; `decode_key` = `svar2_codec::decode_key`; htslib reach includes `lib.rs`. diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index 0ed6e34d..b4e9851b 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -88,7 +88,7 @@ Unlike `.svar` (whose read path builds an interval search tree + a per-read dens `.svar2` is resolved at `Dataset.open` time in the same order as `.svar`: caller `svar2=` arg → recorded relative path → recorded absolute path → sibling `*.svar2`. `Dataset.open(path, svar2=)` mirrors `svar=`. See `docs/source/format.md` ("`.svar2` resolution at open time"). -**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`) and `unphased_union` (for both `"variants"` and `"variant-windows"` output) are both fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: +**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`), `unphased_union` (for both `"variants"` and `"variant-windows"` output), and `var_fields`-selected store INFO/FORMAT fields (also for both `"variants"` and `"variant-windows"`; see `var_fields` under `Dataset.open` below) are all fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: - Spliced output. - The `var_filter="exonic"` (keep-mask) variant filter. - `min_af` / `max_af` filtering. @@ -192,6 +192,8 @@ Scalar fields (`start`/`ilen`/`dosage`/`info[...]`) are still filled from `Dummy - **`var_fields: list[str] | None`** — Variant fields to include on `RaggedVariants` output. Defaults to the minimum useful set `["alt", "ilen", "start"]`. Pass additional names (e.g. `"ref"`, `"dosage"`, or any numeric info column in the source variants table) to load them eagerly at open time. Must be a subset of `Dataset.available_var_fields`. Can be reconfigured later via `Dataset.with_settings(var_fields=...)`, which lazily loads any newly-requested columns. `"dosage"` must be requested explicitly — it is *not* added automatically even when `dosages.npy` exists on disk. Beyond the built-ins (`alt`, `start`, `ref`, `ilen`, `dosage`) and per-variant INFO columns, a genoray `.svar` may register arbitrary per-call (`Number=G`) FORMAT fields in `/metadata.json["fields"]`; these appear in `Dataset.available_var_fields` and can be requested via `Dataset.open(..., var_fields=[...])` or `with_settings(var_fields=[...])`. Each surfaces in `variants`, `variant-windows`, and `flat` outputs as a per-call ragged field aligned with the genotypes. A FORMAT field shadows a same-named INFO column. + **On `.svar2`**, `var_fields` additionally exposes the store's own scalar-numeric INFO/FORMAT fields — whichever ones the `.svar2` was written with via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])` (bare `str` names also work there). Only scalar-numeric fields can exist in a `.svar2` store at all — INFO/FORMAT `Type=Integer`/`Float` with `Number=1` or `Number=A`, plus INFO `Type=Flag` (stored as bool); anything else is rejected by genoray at write time and never reaches gvl. `gvl` only *reads* whatever the store already carries — it cannot add fields, and re-requesting a field the store doesn't have raises (it isn't in `available_var_fields`). `Dataset.available_var_fields` advertises each store field's key, sourced from `genoray.SparseVar2.available_fields`: the bare field name when it's unique across the store's INFO/FORMAT namespace, else `"INFO/"` / `"FORMAT/"`. A builtin name (`alt`/`start`/`ref`/`ilen`/`dosage`) always wins — a store field that happens to be named e.g. `alt` is never advertised and cannot shadow the builtin. Values keep the store's dtype exactly, with no widening (an `i32` field decodes `int32`, an `f32` field `float32`), and a VCF-missing entry carries the store's stored default verbatim (`NaN` for a float field declared with no default). Both entry points route to the same svar2 reconstructor: `gvl.Dataset.open(path, reference=..., var_fields=[...])` and `ds.with_seqs("variants")`/`.with_settings(var_fields=[...])`. Supported on both output modes: `"variants"` (the field appears on the returned `RaggedVariants`, e.g. `rv["AF"]`, sharing `alt`/`start`/`ilen`'s variant offsets) and `"variant-windows"` (the field appears in `win.fields["AF"]` alongside `start`/`ilen`). A FORMAT field's value is the value for the sample that row belongs to (not sample 0). Empty `(region, sample, ploid)` groups fill each store field via the same `DummyVariant.info[]` mechanism as any other scalar field (see `with_settings(dummy_variant=...)` above): the user-supplied value if given, else `NaN` for a float column or `0` for an integer column. + ## Output modes — `with_seqs` × `with_tracks` `with_seqs(kind)` selects the sequence output channel: @@ -428,7 +430,7 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). - **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. -- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")` and `unphased_union` are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, and `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. - **`.svar2` `variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way; `ref_window` is also byte-identical — only raw ALT/`alt_window` bytes differ for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. From 3405dc3605ad53b6e9ad1adcadec6aee5b96cae0 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 05:44:56 -0700 Subject: [PATCH 087/108] fix(svar2): validate open(var_fields=), size buffered output, de-degenerate oracle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the final whole-branch review. The oracle fixture was degenerate: every provenance index was zero (each contig's var_key stream had one call, chr1's dense stream one variant), so hardcoding both call_idx and the dense row to 0 still passed the whole suite. The fixture now carries multiple var_key calls, multiple dense variants with a region whose window starts past the first (exercising the on-disk offset), and an indel — the VkIndel/DenseIndel sub-streams were previously never routed to end-to-end. That mutation now fails 5/9. Dataset.open(var_fields=[...]) silently dropped unknown names for SVAR2: a typo was reported as active and simply absent from the output. It now raises, like with_settings() and the SVAR1 path already did. _output_bytes_per_instance raised KeyError for any SVAR2 store field (it read the dummy variants table's empty info dict), crashing the public buffered dataloader path. It now takes the dtype from the store manifest; the SVAR1 path is unchanged. Field tests 9/9; no-regression 25/25; SVAR1 var_fields 10/10. --- python/genvarloader/_dataset/_impl.py | 9 +- python/genvarloader/_dataset/_svar2_haps.py | 4 + tests/dataset/test_svar2_fields_read.py | 124 ++++++++++++++++++-- 3 files changed, 125 insertions(+), 12 deletions(-) diff --git a/python/genvarloader/_dataset/_impl.py b/python/genvarloader/_dataset/_impl.py index bedbdd57..2bea3986 100644 --- a/python/genvarloader/_dataset/_impl.py +++ b/python/genvarloader/_dataset/_impl.py @@ -1454,7 +1454,14 @@ def _output_bytes_per_instance( total += per_ploid.reshape(-1, ploidy).sum(-1) else: # INFO column: numeric, known dtype from on-disk schema. - info_dtype = haps_obj.variants.info[f].dtype + # Svar2Haps.variants is a dummy placeholder (info={}) -- + # store fields' dtypes live in the store manifest instead. + from ._svar2_haps import Svar2Haps + + if isinstance(haps_obj, Svar2Haps) and f in haps_obj.store_fields: + info_dtype = haps_obj.store_fields[f].dtype + else: + info_dtype = haps_obj.variants.info[f].dtype total += n_vars_total * info_dtype.itemsize if include_offsets: # RaggedVariants (kind=2) writes, per field: outer offsets diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 24fda949..902075e5 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -258,6 +258,10 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: store = Svar2Store(str(svar2_path), sv.contigs, sv.n_samples, sv.ploidy) store_fields = dict(sv.available_fields) + allowed = {"alt", "ilen", "start"} | set(store_fields) + if missing := [f for f in var_fields if f not in allowed]: + raise ValueError(f"Missing variant fields: {missing}") + # Minimal base-Haps fields. genotypes carries only the (R, S, P, None) # shape (so ploidy = shape[-2] and n_variants.shape are available); its # data is empty (svar2 has no per-region sparse genotype store). diff --git a/tests/dataset/test_svar2_fields_read.py b/tests/dataset/test_svar2_fields_read.py index b4cc40e4..71659fa8 100644 --- a/tests/dataset/test_svar2_fields_read.py +++ b/tests/dataset/test_svar2_fields_read.py @@ -14,9 +14,24 @@ Fixture routing (self-asserted below via ``SparseVar2._find_ranges``, not assumed): - chr1:3 (0-based 2), A>G -- carried by exactly ONE haplotype (S0/hap0) - out of 6 in the cohort -> cost model routes this to the VAR_KEY channel. + out of 6 in the cohort -> cost model routes this to the VAR_KEY channel + (VkSnp, call_idx 0 -- first non-empty hap-column in the var_key stream). + - chr1:8 (0-based 7), AT>A (deletion) -- carried by exactly ONE haplotype + (S2/hap0) -> VAR_KEY channel, VkIndel sub-stream (the fixture was + previously all-SNV, leaving VkIndel/DenseIndel dead in this oracle). - chr1:10 (0-based 9), G>C -- carried by ALL 6 haplotypes (hom in every - sample) -> cost model routes this to the DENSE channel. + sample) -> cost model routes this to the DENSE channel (DenseSnp, on-disk + row 0, the first dense-routed SNP on the contig). + - chr1:13 (0-based 12), T>A -- carried by exactly ONE haplotype (S1/hap1) + -> VAR_KEY channel, VkSnp call_idx 1 (a *later* hap-column than chr1:3's + call_idx 0): pins that call_idx is NOT hardcoded to 0 end-to-end. + - chr1:16 (0-based 15), T>C -- carried by ALL 6 haplotypes -> DENSE + channel, DenseSnp on-disk row 1 (the second dense-routed SNP): a region + query that includes this but excludes chr1:10 (see Test 5, region 3) + resolves an on-disk dense row via a nonzero ``on_disk.start`` offset, + pinning ``dense_abs_row`` is not hardcoded to row 0 end-to-end. + - chr1:18 (0-based 17), GC>G (deletion) -- carried by ALL 6 haplotypes -> + DENSE channel, DenseIndel sub-stream. - chr2:5 (0-based 4), A>T -- carried by exactly ONE haplotype (S1/hap0) -> VAR_KEY channel, on the second contig. @@ -40,9 +55,14 @@ # --- fixture: 2 contigs, 3 samples, ploidy 2 (6 haplotypes/contig) ---------- -_REF1 = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" # chr1, 40bp; idx2='A', idx9='G' +_REF1 = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" # chr1, 40bp +# idx2='A' (pos3), idx7:9='AT' (pos8-9), idx9='G' (pos10), idx12='T' (pos13), +# idx15='T' (pos16), idx17:19='GC' (pos18-19) -- all within the first 20bp, so +# the pre-existing chr1:20-40 window (Test 5, region 2) stays variant-free. _REF2 = "ACGT" * 7 + "AC" # chr2, 30bp; idx4='A' -assert len(_REF1) == 40 and _REF1[2] == "A" and _REF1[9] == "G" +assert len(_REF1) == 40 +assert _REF1[2] == "A" and _REF1[7:9] == "AT" and _REF1[9] == "G" +assert _REF1[12] == "T" and _REF1[15] == "T" and _REF1[17:19] == "GC" assert len(_REF2) == 30 and _REF2[4] == "A" _VCF = """\ @@ -55,7 +75,11 @@ ##FORMAT= #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1\tS2 chr1\t3\t.\tA\tG\t.\t.\tAF=0.1;NS=5\tGT:DP\t1|0:10\t0|0:20\t0|0:30 +chr1\t8\t.\tAT\tA\t.\t.\tAF=0.55;NS=4\tGT:DP\t0|0:14\t0|0:24\t1|0:34 chr1\t10\t.\tG\tC\t.\t.\tNS=6\tGT:DP\t1|1:11\t1|1:21\t1|1:31 +chr1\t13\t.\tT\tA\t.\t.\tAF=0.66;NS=7\tGT:DP\t0|0:15\t0|1:25\t0|0:35 +chr1\t16\t.\tT\tC\t.\t.\tAF=0.77;NS=3\tGT:DP\t1|1:16\t1|1:26\t1|1:36 +chr1\t18\t.\tGC\tG\t.\t.\tAF=0.88;NS=1\tGT:DP\t1|1:17\t1|1:27\t1|1:37 chr2\t5\t.\tA\tT\t.\t.\tAF=0.42;NS=2\tGT:DP\t0|0:12\t1|0:22\t0|0:32 """ @@ -430,15 +454,19 @@ def test_svar2_variant_windows_fields( oracle, samples = oracle_and_samples _bcf, ref = _src - # Interleaved across contigs (chr2, chr1, chr1) -> >1 contig group, so the - # multi-contig branch of _reconstruct_variant_windows actually runs (not - # just the single-group fast path). Region 0 covers the chr2 variant; - # region 1 covers both chr1 variants; region 2 (chr1:20-40) has NONE. + # Interleaved across contigs (chr2, chr1, chr1, chr1) -> >1 contig group, + # so the multi-contig branch of _reconstruct_variant_windows actually runs + # (not just the single-group fast path). Region 0 covers the chr2 variant; + # region 1 covers all 5 chr1:0-20 variants; region 2 (chr1:20-40) has + # NONE (variant-free, dummy-fill case); region 3 (chr1:12-20) covers only + # chr1:13/16/18 -- excluding chr1:10 (the first DenseSnp on-disk row) + # forces the DenseSnp window here to resolve chr1:16 via a NONZERO + # `on_disk.start` offset, pinning `dense_abs_row` end-to-end. bed = pl.DataFrame( { - "chrom": ["chr2", "chr1", "chr1"], - "chromStart": [0, 0, 20], - "chromEnd": [15, 20, 40], + "chrom": ["chr2", "chr1", "chr1", "chr1"], + "chromStart": [0, 0, 20, 12], + "chromEnd": [15, 20, 40, 20], } ) ds = _build_dataset(tmp_path, "d5.gvl", bed, svar2_fields_store, ref) @@ -486,6 +514,7 @@ def _assert_dummy_fill(lo: int, hi: int, where: str) -> None: region_keys = { 0: [k for k in oracle if k[0] == "chr2"], 1: [k for k in oracle if k[0] == "chr1" and k[1] < 20], + 3: [k for k in oracle if k[0] == "chr1" and 12 <= k[1] < 20], } for r, keys in region_keys.items(): for s_i in range(S): @@ -572,3 +601,76 @@ def test_svar2_dataset_open_var_fields( rv = ds[:, :] sv = genoray.SparseVar2(str(svar2_fields_store)) _assert_diploid_fields(rv, ["chr1"], samples, oracle, sv) + + +# --- Test 7: unknown var_fields name must raise, not be silently dropped ----- + + +def test_svar2_dataset_open_unknown_var_field_raises( + tmp_path, svar2_fields_store, _src +): + """A typo'd/unsupported field name in ``var_fields`` must raise, mirroring + ``with_settings(var_fields=...)`` (``_impl.py``) and SVAR1's + ``Haps.from_path``. Before the fix, ``Svar2Haps.from_path`` silently + filtered unknown names out of ``_requested_store_fields`` -- the dataset + opened successfully, ``active_var_fields`` reported the typo as "active", + and the output simply lacked it (no error, no warning). + """ + import genoray + import genvarloader as gvl + + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d7.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + with pytest.raises(ValueError, match="Missing variant fields"): + gvl.Dataset.open(d, reference=ref, var_fields=["alt", "start", "TYPO"]) + + # "ref" is a builtin var-field name elsewhere but SVAR2 doesn't provide it. + with pytest.raises(ValueError, match="Missing variant fields"): + gvl.Dataset.open(d, reference=ref, var_fields=["ref"]) + + +# --- Test 8: _output_bytes_per_instance must not KeyError on a store field -- + + +def test_svar2_output_bytes_per_instance_with_store_field( + tmp_path, svar2_fields_store, _src +): + """``_output_bytes_per_instance`` (the buffered/double_buffered dataloader + sizing path, ``_torch.py``'s ``_resolve_buffered_inputs`` -> + ``get_dataloader(mode=...)``) must not crash when an INFO/FORMAT store + field is requested. Before the fix, its ``else`` branch always did + ``haps_obj.variants.info[f].dtype`` -- but ``Svar2Haps.variants`` is the + dummy placeholder with ``info={}``, so any store field name raised + ``KeyError`` there, turning a previously-working path into a crash. + """ + import genoray + import genvarloader as gvl + + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d8.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + ds = ( + gvl.Dataset.open(d, reference=ref) + .with_seqs("variants") + .with_settings(var_fields=_VAR_FIELDS, deterministic=True) + ) + out = ds._output_bytes_per_instance() + assert out.shape == ds.shape + assert (out >= 0).all() From 320561db0b4dad8945496f42ef351bdc94003966 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 10:15:25 -0700 Subject: [PATCH 088/108] docs(svar2): final pre-merge pass design spec Design for the finishing pass over the SVAR2 M6b branch: correctness/safety fixes (unsafe serial-path guard, stale get_unchecked doc, Python-reachable panics, extend_to_length), shipping hygiene (oracle relocation, dead FFI capability, max_ends vectorization, typecheck-task fix, tmp/ relocation), Rust duplication extraction, and doc consistency. Documents the genoray release gate rather than resolving it. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...7-13-svar2-m6b-kernel-final-pass-design.md | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md diff --git a/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md b/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md new file mode 100644 index 00000000..967fe49b --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md @@ -0,0 +1,267 @@ +# SVAR2 M6b branch — final pre-merge pass (design) + +**Date:** 2026-07-13 +**Branch:** `svar2-m6b-kernel` +**Goal:** A finishing pass over the SVAR2 read-bound work — correctness/safety fixes, +shipping hygiene, and doc consistency — so the branch is mergeable modulo the genoray +release gate (documented, not resolved here). + +## Context + +The branch adds SVAR2 (`.svar2` sparse variant format) support to GenVarLoader: as a +`gvl.write` variant source, a live `Dataset` read-bound backend (all-Rust FFI kernels), +INFO/FORMAT field routing into `variants`/`variant-windows` outputs, and `unphased_union`. +It is ~23k insertions across 86 files. The full test suite is green at branch HEAD +(`pixi run -e dev pytest tests -x` exits 0; SVAR2 parity gate 31/31), and `ruff`/`ruff +format` are clean. + +Three focused audits (docs / Python / Rust) plus manual verification produced the +work-list below. This pass does **not** add features or change the parity contract; every +change is either a correctness fix, a hygiene cleanup, or a doc correction. The SVAR1 path +must remain byte-unchanged (additive-only), and the SVAR2 parity gate must stay 31/31. + +## Out of scope / the release gate (documented, not fixed) + +The branch is dev-wired to build only on this machine, and that **cannot** be resolved in +this pass — it depends on a genoray release: + +- `Cargo.toml`: `svar2-codec` and `genoray_core` are `path = "/carter/users/dlaub/projects/genoray..."`. +- `pixi.toml` (`feature.py310`): `genoray` installed from a local `dist/*.whl`. +- `pyproject.toml`: genoray version constraint dropped (`"genoray"`, unpinned). + +PyPI's newest genoray is 2.15.0; the INFO/FORMAT field-read + read-bound gather API this +branch consumes lives on genoray `main`, unreleased. **Decision: leave the dev pins in +place** (they are required to build/test) and add a prominent **RELEASE-GATE checklist** so +nothing ships silently un-pinned. The checklist lives in two places: + +1. The PR body (a "⛔ Do not merge until" section). +2. `docs/roadmaps/rust-migration.md` Phase 6a (a `Release gate` subsection). + +Checklist contents (exact lines to flip at genoray release): +- `Cargo.toml`: `svar2-codec`/`genoray_core` path-deps → published crates.io versions. +- `pixi.toml` `feature.py310`: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` → `"genoray>=,"`. +- Re-run the full py3xx matrix once the wheel is on PyPI. + +Also confirm/adjust the version pins already touched but not gated: `numpy 0.28→0.29`, +`pyo3 0.28.3→0.29`, `seqpro 0.20.0→0.21.1` — verify these are the intended floors at merge. + +## Workstreams + +### 1. Correctness & safety (Rust + Python) — must-fix + +**1a. Unsafe serial path missing its guard.** +`src/reconstruct/mod.rs` — the raw-pointer slices at `:557,564,572` and `:857,864` carve +`out_e - out_s` from caller-supplied `out_offsets`. The `debug_assert!(out_e >= out_s)` +monotonicity guard exists only on the parallel path (`:442,:742`); the serial fallback has +none, so a non-monotonic offsets array underflows to a multi-GB slice (UB) in debug and +silently in release. Fix: hoist the same `debug_assert!` into the serial loops before each +`from_raw_parts_mut`. Confirm the SAFETY comment text matches what is actually asserted. + +**1b. Doc comment describing a reverted optimization.** +`src/svar2/mod.rs:800` documents a `get_unchecked` read of `dense_present`. `get_unchecked` +appears nowhere in `src/` (verified by grep) — all three `present_bit` closures +(`svar2/mod.rs:88`, `reconstruct/mod.rs:677`, `tracks/mod.rs:761`) use checked indexing. The +optimization was reverted and the comment left stale. Fix: delete/rewrite the comment to +describe the checked read that is actually there. (A doc comment asserting a false unsafe +invariant is a correctness hazard for the next reader.) + +**1c. Python-reachable panics → `PyErr`.** +Arrays arriving from Python are `.as_slice().unwrap()` / `.expect("must be contiguous")`'d in +the SVAR2 kernels (`ffi/mod.rs:821-829`, `reconstruct/mod.rs:639-645,692`, +`tracks/mod.rs:725-734,806-808,834-836`). A non-contiguous view (`a[::2]`) panics instead of +raising. Fix: validate contiguity once at the `#[pyfunction]` boundary and return +`PyValueError`. Also bounds-check the pure-DEL anchor index `contig_ref_s[pos..pos+1]` +(`reconstruct/mod.rs:703`) → `PyValueError` for a variant at/past contig end. Also +`svar2/mod.rs:358-365` `assert_eq!(vk_src.len(), ...)` fires in release from a +Python-reachable path — hoist the check to the FFI boundary (`ffi/mod.rs:1411`, where +`has_fields` is known) as a `PyValueError`. + +**1d. `extend_to_length` silently ignored for `.svar2`.** +`_write.py:_write_from_svar2` accepts the flag and never reads it; `chromEnd` is always +extended. Passing `extend_to_length=False` yields a different dataset than requested, with no +signal. **Fix: raise `NotImplementedError`** when `extend_to_length is False` for a `.svar2` +source (consistent with the branch's Phase-1 guard-matrix policy of failing loudly on +unsupported combinations), and document the limitation. Do **not** silently honor `True` +only — make the unsupported case explicit. + +### 2. Shipping hygiene — should-fix + +**2a. Test-only oracle code out of the library.** +`python/genvarloader/_dataset/_svar2_source.py` (`SparseVar2Source`) and +`_svar2_store_py.py` (`build_readbound_*`) have zero importers under `python/` — only +`tests/` uses them (they are the parity oracle + FFI-input builders). Move both under +`tests/` (e.g. `tests/_oracles/`), update test imports, and confirm nothing in the shipped +package references them. Rename on the way: `_svar2_store_py.py` holds no store class (the +Rust `Svar2Store` is the store) — the `_py` suffix is meaningless; name it for what it does +(`svar2_readbound_inputs.py` or similar). `SparseVar2Source` may keep its name once it lives +in tests. + +**2b. Drop dead FFI capability.** +`reconstruct_haplotypes_from_svar2_readbound`'s `annot_v_idxs`/`annot_ref_pos` params are +`None` at both call sites (`ffi/mod.rs:889-890,1045-1046`); 3 of the 4 match arms (~60 lines) +and the `:605-608` doc are unreachable. Remove the params and the dead arms (annotated-hap +output for `.svar2` is guarded `NotImplementedError` anyway). If there's a near-term plan to +wire them, leave a one-line note instead — but default to removal (YAGNI). + +**2c. Vectorize `_svar2_region_max_ends`.** +`_write.py:1092-1138` is an `O(regions × samples × ploidy)` Python triple-loop over decoded +records at write time; its own docstring flags it as a scalability follow-up. The semantics +are a per-region max over haplotypes of `(pos, end)` with a `pos`-then-`end` tie-break. Fix: +vectorize with a scatter-reduce over the decoded ragged offsets — encode `(pos<<32) | end` +into an int64, `np.maximum.reduceat` (or `np.maximum.at` on a region-index scatter) to get +the per-region max, then unpack `end`. Must stay byte-identical to the current loop +(`test_write_svar2.py` locks the cache contents + same-POS-tie behavior). + +**2d. Drop unused `region_starts`.** +`_write.py:1162,1205,1214` writes/memmaps `region_starts`; `_svar2_haps.py:86-88,95` loads it +and its own docstring says it is "kept for parity/debug, NOT fed to the FFI." Remove the +array, its `svar2_meta.json` entry, and the loader. (Confirm no test asserts its presence; if +one does, drop that assertion.) + +**2e. Fix the `typecheck` pixi task (also helps every worktree).** +`pixi run -e dev typecheck` = `pyrefly check` with no paths → inside any `.claude/worktrees/` +checkout pyrefly matches **zero files** (root `.gitignore` ignores `.claude/`, pyrefly honors +ignore files) and exits 0. Typecheck has effectively never run on this branch. Fix: change the +task to `pyrefly check python/genvarloader tests` (explicit paths). Then clear the one real +finding it surfaces: unused `# pyrefly: ignore[no-matching-overload]` at `_ragged.py:325`. + +**2f. Relocate `tmp/svar2_mvp/` into `tests/benchmarks/`.** +19 tracked files with hardcoded absolute paths (`/carter/shared/data/gdc/...`), and a +self-contradicting `.gitignore` entry (`tmp/svar2_mvp/prof_out/` ignored while its `.md` +files are tracked). `tests/benchmarks/` already has the right shape: `profiling/` for +`profile_*.py` + shell drivers, `data/build_*.py` for corpus builders, and session-scoped +path fixtures in `conftest.py`. Plan: +- Benchmark/profiling drivers (`benchmark.py`, `bench_gvl_svar1_vs_svar2.py`, `prof_*.py`, + `prof_*.sh`, `build_stores.py`, `validate.py`, `split_folded.py`) → `tests/benchmarks/` + (drivers) / `tests/benchmarks/profiling/` (perf shells), with hardcoded paths replaced by + the existing `data_dir`/`kg_dir` fixtures or a module-level constant + CLI arg. +- `.sbatch` files and `env_baseline.txt` → drop (machine/cluster-specific scratch; the perf + numbers they produced are already captured in `docs/superpowers/notes/`). +- `prof_out/*.md` reports → drop from git (superseded by the roadmap Phase-6a results + + notes); keep on disk locally. +- Remove the `tmp/svar2_mvp/prof_out/` `.gitignore` line; add `tmp/` if we want scratch + ignored going forward. +Nothing of value is lost — perf conclusions live in the roadmap and notes; only +machine-specific scratch is dropped. + +### 3. Rust duplication extraction — approved, highest-risk + +Guarded by the parity/oracle suite (31/31) + full-tree regression. Do this **last**, rebuild +and re-run parity after each extraction, and keep each extraction its own commit so a +regression bisects cleanly. + +**3a. Chunk-carving + serial/parallel dispatch helper.** +`reconstruct/mod.rs:424-578` and `:724-880` are ~150 lines verbatim (bounds build, +`split_at_mut` carve ×3, 4-arm `(av, ap)` match, serial raw-ptr path). Extract a +`carve_chunks(&mut [T], &[(usize,usize)]) -> Vec<&mut [T]>` and one dispatcher generic over +the per-chunk work closure. This is the hot path — byte-identical output is mandatory. + +**3b. Readbound FFI preamble helper.** +`ffi/mod.rs:934-998, 1086-1144, 1186-1250, 1358-1432` paste the same preamble 4× (reader +lookup → regions build → `arr2_to_ranges` ×4 → `HapRanges::new` → gather → `lut_arrays` → +`split_to_flat` → `dense_range` view). Extract one helper returning +`(FlatChannels, lut_bytes, lut_off, regions)`. The diffs→`out_offsets` prefix-sum loop +(`ffi/mod.rs:846-864, 1000-1019, 1252-1268`) is pasted 3× — extract +`offsets_from_diffs(...)`. Add a `type` alias for the three readbound return tuples to clear +the `clippy::type_complexity` warnings at `:930,1182,1344`. + +**3c. Shared `present_bit`.** +`reconstruct/mod.rs:675-678` == `tracks/mod.rs:759-762` (identical closure + re-documented +LSB-first invariant). Move to a `svar2::present_bit` fn documented once. + +### 4. Docs & comments — should-fix + +**4a. Roadmap (`docs/roadmaps/rust-migration.md`).** +- Phase 6a guard-matrix bullet (~`:812-816`) still lists `unphased_union` and + `"variant-windows"` as guarded `NotImplementedError`; both now ship. Move them to the + supported list. +- Gate footnote (~`:822-826`) claiming variant-windows parity is untested is false — remove + it (`test_svar2_readbound_variants.py`, `test_svar2_fields_read.py` cover it). +- The 2026-07-05 notes-log entry (~`:890`) repeats the stale exclusions — amend or add a + 2026-07-12/07-13 entry reflecting shipped scope. +- Add a ticked task line for the INFO/FORMAT field-routing work (plan + `2026-07-12-svar2-info-format-field-routing.md`). +- Fill the `_PR: TBD_` link once the PR exists (project rule: a ✅ phase carries a PR link) — + or leave 🚧 until the PR number is known, then update. + +**4b. `skills/genvarloader/SKILL.md`.** +- `:193-195`: `var_fields` on `.svar2` does **not** accept `ref`/`dosage`; allowed set is + `alt|ilen|start` + store INFO/FORMAT fields (`_svar2_haps.py:261` raises otherwise). Correct + the statement. +- `:66,170,437`: "`min_af`/`max_af` requires SVAR-backed genotypes" → "`.svar` only (not + `.svar2`)" — `.svar2` raises `NotImplementedError`. +- `:128,442`: note `extend_to_length` has no effect / is unsupported for a `.svar2` source + (matches the 1d fix). +- `:91`: "byte-identical … all four output modes" is contradicted by the pure-deletion ALT + paragraph below it — qualify with "except pure-deletion ALT bytes (see below)". + +**4c. Prose docs (`docs/source/`).** +- `index.md:51`: "Currently supports VCF, PGEN, and BigWig" — mirror README's updated + `.svar`/`.svar2` wording. +- `faq.md:81` + `write.md:98` point at `format.md` "for the full list" of unsupported + `.svar2` combinations, which does not exist there → add the guard-matrix list to + `format.md`, or repoint to the actual location. +- `write.md` §"Variants from a genoray sparse store": add a 2-line snippet showing how to + *build* a `.svar`/`.svar2` (`genoray` `dense2sparse` / `SparseVar2.from_vcf`), since + `faq.md:76` promises it. +- `dataset.md`: add a short "Variant fields (`var_fields`)" section — the branch's headline + feature (`.svar2` store INFO/FORMAT fields on `variants`/`variant-windows`, e.g. `rv["AF"]`) + is currently documented only in the skill. +- `format.md:145`: pin the `(unreleased)` changelog row to the target version at merge. + +**4d. Strip internal plan/task numbering from shipped code.** +Comments/docstrings referencing planning artifacts (reader has no access to them): +`_svar2_haps.py:23` (stale — lists `unphased_union` as unsupported; delete that entry), +`:384,486,489` ("tracks follow-up (7c)"), `_reconstruct.py:143` ("Task 7c"), `:399` ("FIX 1 +guard"), `_write.py:1196` ("Phase-1 wiring"); Rust `svar2/mod.rs:280` ("Task 1.3"), +`tracks/mod.rs:2467` ("Task 4 Part C"), `ffi/mod.rs:774` ("first cut minimal"). Rewrite each +in terms of behavior. The `_reconstruct.py:399` FlankSample fill-seed divergence that the +guard papers over needs a tracked GitHub issue (the comment is currently the only record) — +open one and reference it. + +**4e. Missing docstrings.** +Add numpydoc-style docstrings to public/semi-public surfaces lacking them: +`_svar2_link.py:make_svar2_link`, `_svar2_haps.py:_reconstruct_variants` (sibling +`_reconstruct_variant_windows` has one), `_write.py:_write_from_svar2` (SVAR1's +`_write_from_svar` has one); Rust `svar2/store.rs:16,19,26,46` (`reader`, `store_path`, +`#[new]`, `contigs` — PyO3-exposed, become Python docstrings). + +### 5. Clippy nits (`cargo clippy --all-targets`) — new-code only + +Clear the new-code-attributable warnings; leave pre-existing ones (`bigwig.rs`, +`reference/mod.rs`, etc.) alone as out-of-scope: +- `reconstruct/mod.rs:248,251` `explicit_auto_deref` → drop `as_deref_mut()`. +- `reconstruct/mod.rs:19-37,278` `doc_overindented_list_items` (4→2 spaces). +- `svar2/mod.rs:593,600,772` + `reconstruct/mod.rs:905` `single_range_in_vec_init` / + `redundant_closure` (tests). +- `type_complexity` aliases from 3b cover `:930,1182,1344`. + +## Sequencing + +1. Docs & comments (§4) + clippy nits (§5) — zero runtime risk, do first. +2. Correctness/safety (§1) — rebuild Rust (`maturin develop --release`), run SVAR2 parity. +3. Hygiene (§2) — oracle relocation, dead-capability removal, `max_ends` vectorization, + `region_starts` drop, typecheck-task fix, `tmp/` relocation. +4. Rust duplication (§3) — last, one extraction per commit, parity re-run after each. + +## Verification (gates — all must pass before PR) + +- `pixi run -e dev maturin develop --release` (rebuild before any Python test touching Rust). +- `pixi run -e dev pytest tests -x` — full tree green (SVAR1 byte-unchanged, SVAR2 gate 31/31). +- `pixi run -e dev cargo test` (compiles from source; LD_LIBRARY_PATH per pixi activation.env). +- `pixi run -e dev ruff check python/ tests/` + `ruff format --check python/ tests/`. +- `pixi run -e dev pyrefly check python/genvarloader tests` (the fixed task) — clean. +- `pixi run -e dev cargo clippy --all-targets` — no **new** warnings. +- `api.md` ↔ `__all__` check (should remain "none"; no public symbol added). +- Manual: confirm no `python/` module imports the relocated oracle code; confirm the + RELEASE-GATE checklist is present in both the PR body and the roadmap. + +## Non-goals + +- Resolving the genoray path-pins (release-gated; documented only). +- Any new SVAR2 feature, output mode, or change to the parity contract. +- Refactoring the unrelated genoray-3.0-API bump bundled into this branch (the + `ContigNormalizer` module moves + `_vcf_region_chunks` rewrite) — it's clean; splitting it + into its own commit is optional and noted for the reviewer, not required. +- Touching pre-existing clippy warnings outside the SVAR2 changes. From f5d0f0897083bd340e93a0df3c80be311982b0a1 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 11:03:00 -0700 Subject: [PATCH 089/108] docs(svar2): implementation plan for the final pre-merge pass 13-task plan covering the correctness/safety fixes, shipping hygiene, Rust de-duplication, and doc consistency from the design spec, ending in the release-gated draft PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-07-13-svar2-m6b-kernel-final-pass.md | 1117 +++++++++++++++++ 1 file changed, 1117 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md diff --git a/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md b/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md new file mode 100644 index 00000000..789dfe90 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md @@ -0,0 +1,1117 @@ +# SVAR2 M6b Branch — Final Pre-Merge Pass Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finish the SVAR2 read-bound branch — correctness/safety fixes, shipping hygiene, Rust de-duplication, and doc consistency — so it is mergeable modulo the (documented, unresolved) genoray release gate. + +**Architecture:** Additive-only cleanup of an already-green branch. Every change is a correctness fix, a hygiene cleanup, or a doc correction. The SVAR1 path stays byte-unchanged; the SVAR2 parity gate stays 31/31. No new features, no change to the parity contract. + +**Tech Stack:** Python 3.10 (numpy, polars, pyrefly, ruff), Rust (PyO3 abi3-py310, ndarray, rayon), pixi task runner, pytest + cargo test, prek pre-commit hooks. + +**Spec:** `docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md` + +## Global Constraints + +- **SVAR1 byte-unchanged:** every change is additive w.r.t. the SVAR1 path; no SVAR1 output bytes may change. +- **SVAR2 parity gate = 31/31:** `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py` stays fully green after every task that touches Rust or `_dataset/` code. +- **Rebuild Rust before Python tests that import the extension:** `pixi run -e dev maturin develop --release` after ANY `src/` edit, before running pytest. `cargo test` compiles from source and does not need this. +- **`cargo test` needs libpython on the loader path:** it runs under `pixi run -e dev` which sets `LD_LIBRARY_PATH=$CONDA_PREFIX/lib` via `pixi.toml` `[target.linux-64.activation.env]`. Always invoke cargo via `pixi run -e dev cargo ...`. +- **Lint/format gates:** `pixi run -e dev ruff check python/ tests/` and `pixi run -e dev ruff format --check python/ tests/` must pass. +- **Pre-commit hooks are broken in this worktree** (the `pyrefly-check` hook matches zero files under `.claude/` yet exits nonzero; the `pixi-lock` hook churns `pixi.lock`). Task 2 fixes the pyrefly invocation. Until Task 2 lands, commit doc/Rust-only changes with `git commit --no-verify` and NEVER stage `pixi.lock` churn (run `git checkout pixi.lock` if it appears dirty from an unrelated pixi operation). After Task 2, prefer hooks on; if `pixi-lock` still churns, `git checkout pixi.lock` before committing. +- **Commit style:** conventional commits (`fix(svar2):`, `refactor(svar2):`, `docs(svar2):`, `chore(svar2):`). End every commit message with `Co-Authored-By: Claude Opus 4.8 (1M context) `. +- **Do NOT touch the genoray path-pins** (`Cargo.toml` path-deps, `pixi.toml` wheel path, `pyproject.toml` unpinned genoray). They are the release gate — Task 12 documents them; nobody resolves them here. +- **Do NOT touch pre-existing clippy warnings** outside the SVAR2 changes (`bigwig.rs`, `reference/mod.rs`, etc.). + +--- + +## Execution order + +Docs & clippy (Tasks 1, 11) carry zero runtime risk — but they are grouped by topic below, not strictly first. The safe order is: **2 → 1 → 3 → 4 → 5 → 6 → 7 → 8 → 9 → 10 → 11 → 12 → 13**. Task 2 (typecheck-task fix) goes first so later Python tasks get real type-checking. The Rust de-dup (Tasks 9, 10) go late and each rebuilds + re-runs parity. Task 13 is the final full-suite gate. + +--- + +### Task 1: Strip stale/internal references from shipped comments & docstrings + +Pure text edits in shipped code. No behavior change. Groups all "stale comment" findings (spec §1b, §4d, §5-doc-nits) that are not tied to a code change in another task. + +**Files:** +- Modify: `src/svar2/mod.rs:800` (test docstring), `src/reconstruct/mod.rs:19-37`, `src/reconstruct/mod.rs:278` +- Modify: `python/genvarloader/_dataset/_svar2_haps.py:22-24` (module docstring), `:384`, `:486`, `:489` +- Modify: `python/genvarloader/_dataset/_reconstruct.py:143`, `:399` +- Modify: `python/genvarloader/_dataset/_write.py:1194-1197` + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing (comments only). + +- [ ] **Step 1: Fix the reverted-optimization test docstring** in `src/svar2/mod.rs:800`. The doc for `test_decode_variants_from_split_byte_identical_presence_edge` claims it exercises "the `present_bit` closure's now-`get_unchecked` read of `dense_present`". `get_unchecked` was reverted (grep `get_unchecked src/` → this comment is the only hit). Reword to describe the checked read that is actually there: + +Replace: +```rust + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's now-`get_unchecked` read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte +``` +with: +```rust + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte +``` + +- [ ] **Step 2: Fix the stale `unphased_union` out-of-scope claim** in `python/genvarloader/_dataset/_svar2_haps.py:22-24`. The module docstring lists `unphased_union` as guarded `NotImplementedError`, but it is fully supported (`_reconstruct_variants`, `_reconstruct_variant_windows`, and `_guard_unsupported` explicitly does NOT guard it). + +Replace: +```python +Out of scope for this plan (guarded with ``NotImplementedError``): spliced +output, ``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated +haps, in-kernel reverse-complement, and ``unphased_union``. +``` +with: +```python +Out of scope (guarded with ``NotImplementedError``): spliced output, +``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated haps, and +in-kernel reverse-complement. (``unphased_union`` and ``variant-windows`` ARE +supported.) +``` + +- [ ] **Step 3: Strip internal plan/task numbering** from shipped comments. Rewrite each in terms of behavior (reader has no access to plan docs): + - `_svar2_haps.py:384` — "The tracks follow-up (7c)" → describe the behavior (e.g. "Track re-alignment path"). + - `_svar2_haps.py:486`, `:489` — "tracks (7c)" banner → "tracks". + - `_reconstruct.py:143` — "Task 7c" → behavior description. + - `_reconstruct.py:399` — "FIX 1 guard" → describe what it guards (see Task 8, which also references this line; keep the wording consistent — describe the FlankSample fill-seed divergence). + - `_write.py:1194-1197` — "Phase-1 wiring" comment (see Task 5, which rewrites this block; if Task 5 runs first this may already be gone — if so, skip). Reword to describe that write-time fixed-length handling is unsupported for `.svar2` and the read kernel sizes output at read time. + - `src/reconstruct/mod.rs:278` and the list at `:19-37` — leave the prose but note these get clippy-reflowed in Task 11; no wording change needed here unless a task number appears. + +- [ ] **Step 3b: Add missing `///` docstrings to PyO3-exposed `svar2/store.rs` items** (spec §4e — these become the Python docstrings): add a one-line `///` to `reader` (`:16`), `store_path` (`:19`), the `#[new]` constructor (`:26`), and `contigs` (`:46`) describing what each returns. Also add numpydoc docstrings to `_svar2_link.py:make_svar2_link` and `_svar2_haps.py:_reconstruct_variants` if not added by their own tasks (`_write_from_svar2` is covered by Task 5). + +- [ ] **Step 4: Verify no plan/task references remain** in shipped code: + +Run: +```bash +grep -rnE "Task [0-9]|Phase-1 wiring|FIX 1|\(7c\)|first cut minimal|get_unchecked" src/ python/genvarloader/ | grep -v "test_" || echo "clean" +``` +Expected: `clean` (or only hits inside test names/legit uses you've reviewed). Note: `svar2/mod.rs:280` ("Task 1.3") and `tracks/mod.rs:2467` ("Task 4 Part C") and `ffi/mod.rs:774` ("first cut minimal") are ALSO in scope — fix them in this step too if the grep surfaces them. + +- [ ] **Step 5: Rebuild + smoke-test** (comments in Rust don't change behavior, but confirm it still compiles): + +Run: `pixi run -e dev cargo build 2>&1 | tail -3` +Expected: `Finished` (no errors). + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add -A src/ python/genvarloader/ +git commit --no-verify -m "$(cat <<'EOF' +docs(svar2): strip stale/internal references from shipped comments + +Fix the reverted-get_unchecked test docstring, the stale unphased_union +out-of-scope claim in the Svar2Haps module docstring, and internal plan/task +numbering ("Task 7c", "Phase-1 wiring", "FIX 1") that leaked into comments. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 2: Fix the `typecheck` pixi task (checks zero files in worktrees) + +`pixi run -e dev typecheck` = `pyrefly check` with no paths → inside any `.claude/worktrees/` checkout pyrefly matches **zero** files (root `.gitignore` ignores `.claude/`, pyrefly honors ignore files) and exits 0. Typecheck has effectively never run on this branch. This also fixes the broken `pyrefly-check` pre-commit hook. + +**Files:** +- Modify: `pixi.toml:162` +- Modify: `python/genvarloader/_ragged.py:325` (clear the one real finding the fixed task surfaces) + +**Interfaces:** +- Consumes: nothing. +- Produces: a working `typecheck` task used by every later Python task. + +- [ ] **Step 1: Confirm the bug** — pyrefly with no paths checks nothing in the worktree: + +Run: `pixi run -e dev pyrefly check 2>&1 | grep -c "No Python files matched"` +Expected: `1` (confirms zero files matched). + +- [ ] **Step 2: Point the task at explicit paths** in `pixi.toml:162`. + +Replace: +```toml +typecheck = { cmd = "pyrefly check" } +``` +with: +```toml +typecheck = { cmd = "pyrefly check python/genvarloader tests" } +``` + +- [ ] **Step 3: Run the fixed task; expect exactly one real finding** + +Run: `pixi run -e dev typecheck 2>&1 | grep -E "error|Unused" | head` +Expected: one `ERROR Unused \`# pyrefly: ignore\` comment ... no-matching-overload [unused-ignore]` at `_ragged.py:325`. + +- [ ] **Step 4: Clear the unused-ignore** at `python/genvarloader/_ragged.py:325`. + +Replace: +```python + out = out.reshape((*leading, out_len)) # pyrefly: ignore[no-matching-overload] +``` +with: +```python + out = out.reshape((*leading, out_len)) +``` + +- [ ] **Step 5: Re-run typecheck; expect clean** + +Run: `pixi run -e dev typecheck 2>&1 | tail -3` +Expected: `0 errors` (warnings are fine; no errors). + +- [ ] **Step 6: Confirm the pre-commit hook now passes** (also update the local hook definition if it hardcodes `pyrefly check` with no args): + +Run: `grep -n -A6 "pyrefly-check" .pre-commit-config.yaml` +If the hook `entry` is `pyrefly check` with no paths, change it to `pyrefly check python/genvarloader tests` to match the task. Then: +Run: `pixi run -e dev prek run pyrefly-check --all-files 2>&1 | tail -3` +Expected: `Passed`. + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add pixi.toml python/genvarloader/_ragged.py .pre-commit-config.yaml +git commit -m "$(cat <<'EOF' +chore(svar2): make typecheck task check explicit paths + +Bare `pyrefly check` matches zero files inside a .claude/worktrees checkout +(root .gitignore ignores .claude/, pyrefly honors ignore files), so typecheck +silently passed on nothing. Point it at python/genvarloader + tests, fix the +pre-commit hook to match, and clear the now-flagged unused pyrefly-ignore. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 3: Guard the serial unsafe raw-pointer path in `reconstruct/mod.rs` + +The parallel carve path has `debug_assert!(s >= cursor && e >= s, ...)`; the two **serial** fallback loops (`:547` and `:847`) build `&mut [u8]` from caller-supplied `out_offsets` with only a SAFETY comment, no runtime guard. A non-monotonic `out_offsets` underflows `out_e - out_s` to a giant slice (UB). Add the matching assert. + +**Files:** +- Modify: `src/reconstruct/mod.rs` (serial loop bodies near `:547` and `:847`) + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing (adds a debug-only invariant check). + +- [ ] **Step 1: Add the guard to the first serial loop.** In the serial `for k in 0..n_work` block (the one preceded by `// Serial path: use raw pointers ...`), immediately after: +```rust + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; +``` +insert: +```rust + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); +``` + +- [ ] **Step 2: Add the identical guard to the second serial loop** (the second occurrence of the same `let out_s = ...; let out_e = ...;` pair inside a serial raw-pointer block, near `:847`). Insert the same `debug_assert!` block after the two `let` lines. + +- [ ] **Step 3: Verify both serial loops now assert.** There are two carve dispatchers (merged later in Task 9); both must have the guard on the serial branch. + +Run: +```bash +grep -c "out_offsets must be monotonically non-decreasing" src/reconstruct/mod.rs +``` +Expected: `4` (2 pre-existing parallel-path asserts + 2 new serial-path asserts). + +- [ ] **Step 4: Build + run the Rust unit tests** (compiles the debug_assert; a debug `cargo test` build will trip the assert if any existing test passes non-monotonic offsets — it shouldn't): + +Run: `pixi run -e dev cargo test reconstruct 2>&1 | tail -8` +Expected: tests pass (no assertion panic). + +- [ ] **Step 5: Rebuild release + run SVAR2 parity gate:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_tracks.py -q 2>&1 | tail -3 +``` +Expected: build `Installed`; tests pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/reconstruct/mod.rs +git commit --no-verify -m "$(cat <<'EOF' +fix(svar2): guard serial unsafe carve path with monotonicity debug_assert + +The parallel split_at_mut path already debug_asserts out_offsets is +non-decreasing; the serial raw-pointer fallback carved out_e - out_s with no +guard, so a non-monotonic offsets array underflows to a multi-GB slice (UB). +Hoist the same assert into both serial loops. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 4: Convert Python-reachable Rust panics to `PyValueError` + +Arrays from Python are `.as_slice().unwrap()` / `.expect("must be contiguous")`'d in the SVAR2 kernels; a non-contiguous view (`a[::2]`) panics instead of raising. Plus two more: the pure-DEL anchor index and a release-mode `assert_eq!`. + +**Files:** +- Modify: `src/ffi/mod.rs` (the SVAR2 readbound `#[pyfunction]`s — contiguity validation at the boundary) +- Modify: `src/reconstruct/mod.rs:703` (pure-DEL anchor bounds check), `:639-645`, `:692` +- Modify: `src/svar2/mod.rs:358-365` (move the `assert_eq!` to the FFI boundary at `ffi/mod.rs:1411`) +- Test: `tests/dataset/test_svar2_readbound_haps.py` (add a non-contiguous-input test) + +**Interfaces:** +- Consumes: nothing. +- Produces: SVAR2 readbound `#[pyfunction]`s raise `ValueError` (not panic) on non-C-contiguous inputs and on out-of-range variant positions. + +- [ ] **Step 1: Write the failing test** — a non-contiguous input should raise `ValueError`, not crash the interpreter. Add to `tests/dataset/test_svar2_readbound_haps.py`: + +```python +def test_readbound_haps_noncontiguous_input_raises(): + """A non-C-contiguous numpy view must surface as ValueError, not a panic.""" + import numpy as np + import pytest + from genvarloader._dataset._svar2_store_py import build_readbound_haps # noqa: F401 + # Build a minimal store + regions exactly as the existing haps parity test does, + # then pass a strided (non-contiguous) view of one of the int64 range arrays. + # (Reuse the fixture/store construction from test_readbound_haps_* above.) + # The precise construction mirrors the sibling test; the assertion is: + with pytest.raises(ValueError): + # call the FFI entry with a[::2] slice of a range array + ... +``` +NOTE to implementer: model the store/region setup on the nearest existing test in this file (e.g. the first `build_readbound_haps` parity test around `:75`). The ONLY new thing is passing a `arr[::2]` view where a contiguous int64 array is expected and asserting `ValueError`. + +- [ ] **Step 2: Run it — expect it to FAIL** (currently panics / wrong exception): + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_readbound_haps_noncontiguous_input_raises -v 2>&1 | tail -8` +Expected: FAIL (panic/pyo3 PanicException or a bare index error, not ValueError). + +- [ ] **Step 3: Add a contiguity helper + validate at each SVAR2 `#[pyfunction]` boundary** in `src/ffi/mod.rs`. Near the top of the SVAR2 FFI section add: +```rust +/// Return a C-contiguous slice view of `arr`, or a Python `ValueError` if the +/// input array is not contiguous (e.g. a strided `a[::2]` view). Kernels below +/// index the backing slice directly, so a non-contiguous input would otherwise +/// panic inside `py.detach`. +fn require_contiguous<'a, T: numpy::Element>( + arr: &'a numpy::PyReadonlyArray1, + name: &str, +) -> PyResult<&'a [T]> { + arr.as_slice() + .map_err(|_| PyValueError::new_err(format!("`{name}` must be C-contiguous"))) +} +``` +Then at each of the four SVAR2 readbound `#[pyfunction]`s (`reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, `decode_variants_from_svar2_readbound`, `hap_diffs_from_svar2_readbound`), replace the `.as_slice().unwrap()` / `.expect("must be contiguous")` calls on Python-supplied arrays with `require_contiguous(&arr, "arr")?`. (For `PyReadwriteArray` outputs use the analogous `as_slice_mut().map_err(...)`.) + +- [ ] **Step 4: Bounds-check the pure-DEL anchor** at `src/reconstruct/mod.rs:703`. The `contig_ref_s[pos..pos+1]` index panics for a variant at/past contig end. Guard it — return a `Result`/`PyErr` up the call chain, OR (if this fn is not `PyResult`-returning) validate `pos < contig_ref_s.len()` at the FFI boundary before entering `py.detach` and raise `PyValueError::new_err(format!("variant position {pos} is beyond contig end"))`. Prefer the FFI-boundary check to keep the hot kernel panic-free. + +- [ ] **Step 5: Move the release-mode `assert_eq!`** from `src/svar2/mod.rs:358-365` to the FFI boundary. In `decode_variants_from_svar2_readbound` (`ffi/mod.rs:~1411`, where `has_fields` is known), validate the `vk_src` length precondition and raise `PyValueError` before calling into `svar2::`. Downgrade the in-kernel `assert_eq!` to `debug_assert_eq!` (or delete it if the FFI check fully covers it). + +- [ ] **Step 6: Rebuild + run the new test — expect PASS:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_readbound_haps_noncontiguous_input_raises -v 2>&1 | tail -5 +``` +Expected: PASS (`ValueError` raised). + +- [ ] **Step 7: Full SVAR2 parity gate still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: 31/31 (+ the new test) pass. + +- [ ] **Step 8: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs src/reconstruct/mod.rs src/svar2/mod.rs tests/dataset/test_svar2_readbound_haps.py +git commit --no-verify -m "$(cat <<'EOF' +fix(svar2): raise PyValueError on non-contiguous / OOB Python input + +Non-C-contiguous numpy views (a[::2]) and out-of-range variant positions +panicked inside the readbound kernels instead of surfacing as ValueError. +Validate contiguity and the pure-DEL anchor bound at the #[pyfunction] +boundary; move the vk_src length assert_eq! there too. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 5: Reject `extend_to_length=False` for `.svar2` (stop silently ignoring it) + +`_write_from_svar2` accepts `extend_to_length` and never reads it; `chromEnd` is always extended. Passing `False` yields a different dataset than requested, silently. Per the branch's guard-matrix policy, fail loudly. + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1140-1197` (`_write_from_svar2`) +- Test: `tests/dataset/test_write_svar2.py` + +**Interfaces:** +- Consumes: `_write_from_svar2(path, bed, svar2, samples, extend_to_length)` (existing signature — unchanged). +- Produces: `gvl.write(..., variants=, extend_to_length=False)` raises `NotImplementedError`. + +- [ ] **Step 1: Write the failing test** in `tests/dataset/test_write_svar2.py`: + +```python +def test_svar2_extend_to_length_false_raises(tmp_path, svar2_source_and_bed): + """extend_to_length=False is unsupported for a .svar2 source (Phase-1) and + must raise, not silently produce an extended dataset.""" + import pytest + import genvarloader as gvl + svar2, bed = svar2_source_and_bed # reuse the existing fixture used by the write tests + with pytest.raises(NotImplementedError, match="extend_to_length"): + gvl.write(tmp_path / "ds", bed, variants=svar2, extend_to_length=False) +``` +NOTE: reuse whatever fixture the existing `test_write_svar2.py` tests use to obtain a `SparseVar2` + bed; match its parameter name. + +- [ ] **Step 2: Run it — expect FAIL** (currently silently succeeds): + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_extend_to_length_false_raises -v 2>&1 | tail -6` +Expected: FAIL (no exception raised). + +- [ ] **Step 3: Add the guard** at the top of `_write_from_svar2` in `python/genvarloader/_dataset/_write.py` (right after the signature/opening comment, before `out_dir = ...`): + +```python + if not extend_to_length: + raise NotImplementedError( + "extend_to_length=False is not supported for a .svar2 variant source: " + "the read-bound kernel always sizes haplotype output at read time and " + "the write-time ranges cache is built for the extended chromEnd. Use a " + ".svar/VCF/PGEN source if you need un-extended haplotypes." + ) +``` +Then replace the now-inaccurate `# extend_to_length fixed-output-length write-time handling is out of scope ...` comment block (around `:1194-1197`) with a one-liner noting the flag is validated at entry. + +- [ ] **Step 4: Run the new test — expect PASS:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_extend_to_length_false_raises -v 2>&1 | tail -4` +Expected: PASS. + +- [ ] **Step 5: Full write-svar2 suite still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "$(cat <<'EOF' +fix(svar2): reject extend_to_length=False for .svar2 sources + +_write_from_svar2 accepted the flag and ignored it, silently extending +chromEnd regardless. Raise NotImplementedError (Phase-1 guard-matrix policy) +instead of producing a dataset that differs from what was requested. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 6: Vectorize `_svar2_region_max_ends` + +`_write.py:1092-1138` is an `O(regions × samples × ploidy)` Python triple-loop over decoded records at write time. Its own docstring flags it. Vectorize with a scatter-reduce while staying byte-identical (`test_write_svar2.py` locks the cache + same-POS-tie behavior). + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1092-1138` (`_svar2_region_max_ends`) +- Test: `tests/dataset/test_write_svar2.py` (add a direct equivalence test against the old loop) + +**Interfaces:** +- Consumes: `_svar2_region_max_ends(svar2, contig, starts, ends, samples) -> NDArray[np.int32]` (signature unchanged). +- Produces: identical output to the current loop (per-region max over selected haplotypes of `(pos, end)`, `pos`-then-`end` tie-break, default = `ends`). + +- [ ] **Step 1: Pin the current behavior with a direct test.** Before changing the function, add a test that captures the exact semantics — including the same-POS tie-break and the "no variants → keep chromEnd" default — on a hand-built decode. Add to `tests/dataset/test_write_svar2.py`: + +```python +def test_svar2_region_max_ends_matches_reference(svar2_source_and_bed): + """Vectorized _svar2_region_max_ends must equal a straightforward per-hap loop, + including the pos-then-end tie-break and the empty-region default = chromEnd.""" + import numpy as np + from genvarloader._dataset._write import _svar2_region_max_ends + svar2, bed = svar2_source_and_bed + for (c,), df in bed.partition_by("chrom", as_dict=True, maintain_order=True).items(): + starts = df["chromStart"].to_numpy() + ends = df["chromEnd"].to_numpy() + samples = list(svar2.available_samples) + got = _svar2_region_max_ends(svar2, c, starts, ends, samples) + # reference: recompute with the explicit loop semantics + ref = _reference_region_max_ends(svar2, c, starts, ends, samples) + np.testing.assert_array_equal(got, ref) + + +def _reference_region_max_ends(svar2, contig, starts, ends, samples): + """Byte-for-byte copy of the ORIGINAL triple-loop, kept in the test as the oracle.""" + import numpy as np + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = [svar2.available_samples.index(s) for s in samples] + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = dec.data["pos"]; ilen_arr = dec.data["ilen"]; off = np.asarray(dec.offsets) + out = np.asarray(ends, np.int64).copy() + for r in range(R): + best_pos, best_end = -1, -1 + for s in sel: + for p in range(P): + h = (r * S_all + s) * P + p + a, b = int(off[h]), int(off[h + 1]) + if a == b: continue + seg_pos = pos_arr[a:b]; seg_ilen = ilen_arr[a:b] + j = int(np.argmax(seg_pos)) + p_pos = int(seg_pos[j]); p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) + if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): + best_pos, best_end = p_pos, p_end + if best_pos >= 0: + out[r] = best_end + return out.astype(np.int32) +``` + +- [ ] **Step 2: Run it — expect PASS** (the reference IS the current impl, so it passes now; this locks the contract before the rewrite): + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_region_max_ends_matches_reference -v 2>&1 | tail -4` +Expected: PASS. + +- [ ] **Step 3: Vectorize the function.** Replace the triple-loop body of `_svar2_region_max_ends` (keeping the docstring's semantics but dropping the "O(...) Python iteration ... vectorize as a follow-up" caveat) with a scatter-reduce. Key idea: for each variant, its haplotype maps to a region `r = h // (S_all * P)` but only SELECTED samples count; compute `end = (pos+1) - min(ilen,0)` per variant, form a sortable composite `key = (pos << 21) | end` (end fits well under 2^21 for realistic regions; assert it) so that a plain per-region max on `key` reproduces the pos-then-end tie-break, then unpack `end`: + +```python + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = np.asarray([svar2.available_samples.index(s) for s in samples], np.int64) + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = np.asarray(dec.data["pos"], np.int64) + ilen_arr = np.asarray(dec.data["ilen"], np.int64) + off = np.asarray(dec.offsets, np.int64) # length R*S_all*P + 1 + out = np.asarray(ends, np.int64).copy() # default = chromEnd + if pos_arr.size: + n_hap = R * S_all * P + counts = np.diff(off) # variants per hap + hap_of_var = np.repeat(np.arange(n_hap), counts) # region-major hap index per variant + s_of_hap = (np.arange(n_hap) // P) % S_all + keep = np.isin(s_of_hap[hap_of_var], sel) # only selected samples + region_of_var = hap_of_var // (S_all * P) + end_var = (pos_arr + 1) - np.minimum(ilen_arr, 0) # 0-based -> 1-based, extend on DEL + SHIFT = 21 + assert int(end_var.max(initial=0)) < (1 << SHIFT), "end exceeds tie-break packing width" + key = (pos_arr << SHIFT) | end_var + key_k = key[keep]; region_k = region_of_var[keep] + if key_k.size: + best = np.full(R, -1, np.int64) + np.maximum.at(best, region_k, key_k) # per-region max composite key + has = best >= 0 + out[has] = best[has] & ((1 << SHIFT) - 1) # unpack end + return out.astype(np.int32) +``` +Update the docstring: drop the last paragraph ("O(R * len(samples) * ploidy) Python iteration ... follow-up") and replace with a one-line note that it is a vectorized per-region scatter-max preserving the pos-then-end tie-break. + +- [ ] **Step 4: Run the equivalence test — expect PASS:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_region_max_ends_matches_reference -v 2>&1 | tail -4` +Expected: PASS (vectorized == reference loop). + +- [ ] **Step 5: Full write-svar2 suite (locks cache + same-POS tie) still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "$(cat <<'EOF' +perf(svar2): vectorize _svar2_region_max_ends (byte-identical) + +Replace the O(regions x samples x ploidy) write-time Python triple-loop with a +per-region scatter-max over a (pos<<21)|end composite key that preserves the +pos-then-end tie-break. Pinned byte-identical to the original loop by a new +equivalence test carrying the old loop as its oracle. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 7: Drop the unused `region_starts` array + +`_write.py` writes/memmaps `region_starts`; `_svar2_haps.py` loads it and its own docstring says it is "kept for parity/debug, NOT fed to the FFI." Remove it end-to-end. + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1162,1205,1214` (+ the `svar2_meta.json` entry at `:1170-ish`) +- Modify: `python/genvarloader/_dataset/_svar2_haps.py:86-88,95` (loader + docstring) +- Modify: `tests/unit/dataset/test_svar2_store.py` and/or `tests/dataset/test_write_svar2.py` if either asserts `region_starts` presence + +**Interfaces:** +- Consumes: nothing. +- Produces: `svar2_ranges/` cache no longer contains `region_starts.npy`; `svar2_meta.json` no longer lists it. + +- [ ] **Step 1: Check whether any test asserts `region_starts`:** + +Run: `grep -rn "region_starts" tests/ python/genvarloader/ | grep -v "\.pyc"` +Expected: hits in `_write.py`, `_svar2_haps.py`, and possibly a test. Note every location. + +- [ ] **Step 2: Remove the write side** in `_write.py`: + - Delete the `region_starts = np.memmap(... "region_starts.npy" ...)` line (`:1162`). + - Delete the `"region_starts": {"shape": [R], "dtype": "&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Full SVAR2 read parity (dataset opens the trimmed cache):** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py python/genvarloader/_dataset/_svar2_haps.py tests/ +git commit -m "$(cat <<'EOF' +refactor(svar2): drop unused region_starts from the ranges cache + +region_starts was written, memmapped, and never fed to the FFI (its own +docstring said "parity/debug only"). Remove it from the write path, the +svar2_meta.json schema, the Svar2Haps loader, and the cache-contents test. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 8: Open a tracked issue for the FlankSample fill-seed divergence + +`_reconstruct.py:399-419` papers over a documented correctness divergence (multi-contig `FlankSample` track fills seed off a contig-local query index) with a `NotImplementedError`. The comment is the only record. File an issue and reference it. + +**Files:** +- Modify: `python/genvarloader/_dataset/_reconstruct.py:399-419` (add issue reference to the guard comment) + +**Interfaces:** +- Consumes: nothing. +- Produces: a GitHub issue URL referenced in the guard comment. + +- [ ] **Step 1: Read the guard + its comment** to write an accurate issue body: + +Run: `sed -n '395,420p' python/genvarloader/_dataset/_reconstruct.py` + +- [ ] **Step 2: Create the issue** (repo `mcvickerlab/GenVarLoader`): + +Run: +```bash +gh issue create --repo mcvickerlab/GenVarLoader \ + --title "SVAR2 read-bound: multi-contig FlankSample track fill uses contig-local query index" \ + --body "In the SVAR2 read-bound path, FlankSample track fills seed the fill value off a contig-local query index rather than the global one, which diverges across a multi-contig batch. Currently guarded with NotImplementedError in \`_reconstruct.py\` (~line 399). This issue tracks lifting the guard once the fill-seed index is made global. Branch: svar2-m6b-kernel." +``` +Capture the printed issue URL. + +- [ ] **Step 3: Reference the issue in the guard comment.** Replace the "FIX 1 guard" wording (also targeted by Task 1) with a behavior description ending in the issue link, e.g.: +```python + # Multi-contig FlankSample track fills seed off a contig-local query + # index, which diverges from the global fill seed across a batch. + # Guarded until the fill-seed index is made global; see + # https://github.com/mcvickerlab/GenVarLoader/issues/. +``` + +- [ ] **Step 4: Verify the guard still fires** (behavior unchanged — this is a comment + issue only): + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q -k "flank or Flank or multi" 2>&1 | tail -4` +Expected: pass (guard raises where tested; no behavior change). + +- [ ] **Step 5: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_reconstruct.py +git commit -m "$(cat <<'EOF' +docs(svar2): track FlankSample fill-seed divergence in a GitHub issue + +The multi-contig FlankSample track-fill divergence was recorded only in a code +comment behind a NotImplementedError guard. File a tracking issue and +reference it from the guard. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 9: Relocate test-only oracle modules out of the shipped package + +`_svar2_source.py` (`SparseVar2Source`) and `_svar2_store_py.py` (`build_readbound_*`) have zero importers under `python/` — only 8 test files use them. Move them into `tests/` and rename `_svar2_store_py.py` (the `_py` suffix is meaningless — it holds no store class). + +**Files:** +- Move: `python/genvarloader/_dataset/_svar2_source.py` → `tests/_oracles/svar2_source.py` +- Move: `python/genvarloader/_dataset/_svar2_store_py.py` → `tests/_oracles/svar2_readbound_inputs.py` +- Create: `tests/_oracles/__init__.py` +- Modify (imports): `tests/test_svar2_realign_tracks.py`, `tests/test_svar2_reconstruct.py`, `tests/dataset/test_svar2_dataset.py`, `tests/dataset/test_svar2_readbound_haps.py`, `tests/dataset/test_svar2_readbound_diffs.py`, `tests/dataset/test_svar2_readbound_tracks.py`, `tests/dataset/test_svar2_readbound_variants.py` +- Modify (docstring back-reference): `python/genvarloader/_dataset/_svar2_haps.py:15-19` + +**Interfaces:** +- Consumes: tests import `from tests._oracles.svar2_source import SparseVar2Source` and `from tests._oracles.svar2_readbound_inputs import build_readbound_haps, build_readbound_diffs, build_readbound_tracks, build_readbound_variants`. +- Produces: no test-only code ships in `genvarloader`. + +- [ ] **Step 1: Confirm the two modules are imported only from tests** (already audited, re-verify): + +Run: `grep -rln "_svar2_source\|_svar2_store_py\|SparseVar2Source\|build_readbound" python/genvarloader/ | grep -v "\.so"` +Expected: only `_svar2_haps.py` (docstring reference) and the two modules themselves — NO functional importer in `python/`. + +- [ ] **Step 2: Check the two modules' own imports** (what they pull from the package — must still resolve from `tests/`): + +Run: `grep -n "^import\|^from" python/genvarloader/_dataset/_svar2_source.py python/genvarloader/_dataset/_svar2_store_py.py` +Note any relative imports (`from . import ...` / `from ._x import ...`) — these must become absolute `genvarloader...` imports after the move. + +- [ ] **Step 3: Move the files with git + create the package:** + +```bash +mkdir -p tests/_oracles +git mv python/genvarloader/_dataset/_svar2_source.py tests/_oracles/svar2_source.py +git mv python/genvarloader/_dataset/_svar2_store_py.py tests/_oracles/svar2_readbound_inputs.py +touch tests/_oracles/__init__.py && git add tests/_oracles/__init__.py +``` + +- [ ] **Step 4: Fix relative imports** inside the two moved files (from Step 2). Any `from .` / `from ._foo import` becomes `from genvarloader._dataset._foo import`. If `svar2_readbound_inputs.py` imports `SparseVar2Source`, point it at `from tests._oracles.svar2_source import SparseVar2Source`. + +- [ ] **Step 5: Update the 8 test files' imports.** In each, replace: + - `from genvarloader._dataset._svar2_source import SparseVar2Source` → `from tests._oracles.svar2_source import SparseVar2Source` + - `from genvarloader._dataset._svar2_store_py import build_readbound_*` → `from tests._oracles.svar2_readbound_inputs import build_readbound_*` + +Do it mechanically: +```bash +grep -rl "_dataset._svar2_source\|_dataset._svar2_store_py" tests/ | while read f; do + sed -i 's#genvarloader\._dataset\._svar2_source#tests._oracles.svar2_source#g; s#genvarloader\._dataset\._svar2_store_py#tests._oracles.svar2_readbound_inputs#g' "$f" +done +``` +(RTK note: this is a mechanical sed on test imports — acceptable here.) + +- [ ] **Step 6: Update the `_svar2_haps.py` docstring back-reference** (`:15-19`) that mentions `_svar2_store_py.build_readbound_*`. Point it at the new location: replace `_svar2_store_py.build_readbound_*` with `tests/_oracles/svar2_readbound_inputs.build_readbound_*` (and note it is the test oracle). + +- [ ] **Step 7: Confirm `tests` is importable as a package** (so `from tests._oracles...` resolves). Check for `tests/__init__.py`: + +Run: `ls tests/__init__.py 2>/dev/null && echo "has __init__" || echo "NO __init__ — check conftest/rootdir import mode"` +If tests are collected in rootdir/`importmode=prepend` without `tests/__init__.py`, prefer a conftest-based path or add `tests/__init__.py`. Verify by running one moved-import test in Step 8; if `ModuleNotFoundError: tests`, add `tests/__init__.py` (`touch tests/__init__.py && git add tests/__init__.py`) or adjust to a top-level `_oracles` package under the tests rootdir — match the repo's existing test-helper import convention (`tests/_builders/` is imported as `from _builders...` or `from tests._builders...`? check it): +```bash +grep -rn "_builders" tests/ | grep import | head -3 +``` +Mirror whatever `_builders` does. + +- [ ] **Step 8: Run every affected test file:** + +Run: +```bash +pixi run -e dev pytest tests/test_svar2_realign_tracks.py tests/test_svar2_reconstruct.py tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_diffs.py tests/dataset/test_svar2_readbound_tracks.py tests/dataset/test_svar2_readbound_variants.py -q 2>&1 | tail -4 +``` +Expected: all pass (no `ModuleNotFoundError`, no import errors). + +- [ ] **Step 9: Confirm nothing in the shipped package references the moved modules:** + +Run: `grep -rn "_svar2_source\|_svar2_store_py" python/genvarloader/ | grep -v "\.so"` +Expected: no hits (the docstring in `_svar2_haps.py` now points at `tests/_oracles/...`). + +- [ ] **Step 10: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add -A python/ tests/ +git commit -m "$(cat <<'EOF' +refactor(svar2): move test-only oracles out of the shipped package + +SparseVar2Source and build_readbound_* have no importers under python/ — they +are the parity oracle + FFI-input builders used only by tests. Move them to +tests/_oracles/ (renaming _svar2_store_py.py -> svar2_readbound_inputs.py, +since it holds no store class) and repoint the 8 test files' imports. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 10: Drop the dead `annot_*` capability from the SVAR2 readbound haplotype kernel + +`reconstruct_haplotypes_from_svar2_readbound`'s `annot_v_idxs`/`annot_ref_pos` params are `None` at both call sites; 3 of 4 match arms (~60 lines) + the `:605-608` doc are unreachable (annotated-hap output for `.svar2` is `NotImplementedError`-guarded anyway). + +**Files:** +- Modify: `src/ffi/mod.rs` (`reconstruct_haplotypes_from_svar2_readbound` signature + the two call sites `:889-890`, `:1045-1046`) +- Modify: `src/reconstruct/mod.rs` (the readbound reconstruct fn — drop the annot params + unreachable match arms + `:605-608` doc) + +**Interfaces:** +- Consumes: nothing. +- Produces: `reconstruct_haplotypes_from_svar2_readbound` no longer takes `annot_v_idxs`/`annot_ref_pos`; only the un-annotated output arm remains. + +NOTE: This is scoped to the SVAR2 **readbound** reconstruct fn ONLY. Do NOT remove the annot params from the general (SVAR1) reconstruct dispatchers merged in Task 11 — SVAR1 uses them. If Task 11 runs first, keep them independent. + +- [ ] **Step 1: Confirm both call sites pass `None`:** + +Run: `grep -n "annot_v_idxs\|annot_ref_pos" src/ffi/mod.rs` +Expected: at the two `reconstruct_haplotypes_from_svar2_readbound` call sites they are `None`/`None`. Confirm no SVAR2 readbound caller ever passes `Some`. + +- [ ] **Step 2: Remove the params from the Rust fn** in `src/reconstruct/mod.rs`: drop `annot_v_idxs`/`annot_ref_pos` from the readbound reconstruct fn signature, delete the 3 match arms that handle `Some(...)` (keep only the `(None, None)` / un-annotated arm), and delete the now-inaccurate doc at `:605-608`. + +- [ ] **Step 3: Remove the params at both FFI call sites** (`ffi/mod.rs:889-890`, `:1045-1046`) and from the `#[pyfunction]` signature if they were exposed to Python (check the `#[pyo3(signature = ...)]` and the Python-side call in `_svar2_haps.py` — if the Python code passes them, drop there too): + +Run: `grep -rn "reconstruct_haplotypes_from_svar2_readbound" python/genvarloader/` +If the Python call passes `annot_v_idxs=`/`annot_ref_pos=`, remove those kwargs. + +- [ ] **Step 4: Build — expect clean compile** (no unused-var / unreachable warnings for the removed arms): + +Run: `pixi run -e dev cargo build 2>&1 | tail -5` +Expected: `Finished`, no warnings about the removed code. + +- [ ] **Step 5: Rebuild release + full SVAR2 parity gate:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py -q 2>&1 | tail -3 +``` +Expected: build `Installed`; tests pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs src/reconstruct/mod.rs python/genvarloader/_dataset/_svar2_haps.py +git commit --no-verify -m "$(cat <<'EOF' +refactor(svar2): drop dead annot_* capability from readbound haps kernel + +annot_v_idxs/annot_ref_pos were None at both call sites of +reconstruct_haplotypes_from_svar2_readbound; 3 of 4 match arms and their doc +were unreachable (annotated .svar2 haps are NotImplementedError-guarded). Drop +the params and the dead arms. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 11: Extract Rust duplication (carve dispatcher, FFI preamble, present_bit) + clear new clippy nits + +Highest-risk task; guarded by the full parity suite. Do each extraction as its own commit so a regression bisects cleanly. Rebuild + re-run parity after EACH extraction. + +**Files:** +- Modify: `src/reconstruct/mod.rs` (carve/dispatch helper; `present_bit` move; clippy `explicit_auto_deref` at `:248,251`, `doc_overindented_list_items` at `:19-37,278`) +- Modify: `src/ffi/mod.rs` (readbound preamble helper; `offsets_from_diffs` helper; `type` aliases for `:930,1182,1344`) +- Modify: `src/tracks/mod.rs` (use shared `present_bit`) +- Modify: `src/svar2/mod.rs` (host `present_bit`; test-only `single_range_in_vec_init` at `:593,600,772`) + +**Interfaces:** +- Consumes: nothing (internal Rust refactor). +- Produces: `svar2::present_bit`, `carve_chunks`, one carve dispatcher, one readbound-preamble helper, `offsets_from_diffs` — all internal. + +- [ ] **Step 1 (extraction 1 — carve dispatcher):** Extract `fn carve_chunks(buf: &mut [T], bounds: &[(usize, usize)]) -> Vec<&mut [T]>` and a single dispatcher generic over the per-chunk work closure, replacing the ~150 verbatim lines duplicated between `reconstruct/mod.rs:424-578` and `:724-880`. Keep BOTH the parallel (`split_at_mut`) and serial (raw-ptr) branches inside the helper, with the Task 3 `debug_assert!` on the serial branch. Output must be byte-identical. + +- [ ] **Step 2:** Build + full parity: +```bash +pixi run -e dev cargo build 2>&1 | tail -2 +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py -q 2>&1 | tail -3 +``` +Expected: 31/31 pass. Also `pixi run -e dev cargo test 2>&1 | tail -5` passes. + +- [ ] **Step 3: Commit extraction 1** +```bash +git checkout pixi.lock 2>/dev/null +git add src/reconstruct/mod.rs +git commit --no-verify -m "refactor(svar2): extract carve_chunks + shared serial/parallel dispatcher + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 4 (extraction 2 — FFI preamble):** Extract one helper returning `(FlatChannels, lut_bytes, lut_off, regions)` from the 4× readbound preamble (`ffi/mod.rs:934-998,1086-1144,1186-1250,1358-1432`), and `fn offsets_from_diffs(...)` from the 3× prefix-sum loop (`:846-864,1000-1019,1252-1268`). Add a `type` alias for the three readbound return tuples (clears `clippy::type_complexity` at `:930,1182,1344`). + +- [ ] **Step 5:** Build + full parity (same commands as Step 2). Expected: 31/31. + +- [ ] **Step 6: Commit extraction 2** +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs +git commit --no-verify -m "refactor(svar2): extract readbound FFI preamble + offsets_from_diffs helpers + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 7 (extraction 3 — present_bit):** Move the identical `present_bit` closure (`reconstruct/mod.rs:675-678` == `tracks/mod.rs:759-762`) to a documented `svar2::present_bit`; call it from both. Build + full parity. + +- [ ] **Step 8: Commit extraction 3** +```bash +git checkout pixi.lock 2>/dev/null +git add src/svar2/mod.rs src/reconstruct/mod.rs src/tracks/mod.rs +git commit --no-verify -m "refactor(svar2): hoist shared present_bit to svar2::present_bit + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 9 (clippy nits):** Clear the NEW-code clippy warnings only: +```bash +pixi run -e dev cargo clippy --all-targets 2>&1 | tail -40 +``` +Fix: `reconstruct/mod.rs:248,251` `explicit_auto_deref` (drop `as_deref_mut()`), `doc_overindented_list_items` at `:19-37,278` (4→2 spaces), `single_range_in_vec_init`/`redundant_closure` in the svar2/reconstruct tests. Leave pre-existing warnings (`bigwig.rs`, `reference/mod.rs`, `intervals.rs`, etc.) untouched. + +- [ ] **Step 10: Confirm no new clippy warnings + parity green:** +```bash +pixi run -e dev cargo clippy --all-targets 2>&1 | grep -c "warning:" # compare against the pre-existing baseline count +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py -q 2>&1 | tail -3 +``` +Expected: only pre-existing warnings remain; 31/31 pass. + +- [ ] **Step 11: Commit clippy nits** +```bash +git checkout pixi.lock 2>/dev/null +git add src/ +git commit --no-verify -m "$(cat <<'EOF' +style(svar2): clear new-code clippy nits (auto-deref, doc indent, type alias) + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 12: Documentation consistency + RELEASE-GATE checklist + +All doc corrections (spec §4a–c) plus the release-gate documentation (spec "Out of scope"). + +**Files:** +- Modify: `docs/roadmaps/rust-migration.md` (Phase 6a: guard matrix, gate footnote, notes log, field-routing task line, PR link, RELEASE-GATE subsection) +- Modify: `skills/genvarloader/SKILL.md:66,91,128,170,193-195,437,442` +- Modify: `docs/source/index.md:51`, `docs/source/faq.md`, `docs/source/write.md`, `docs/source/dataset.md`, `docs/source/format.md` + +**Interfaces:** +- Consumes: nothing. +- Produces: docs consistent with shipped behavior; a release-gate checklist in the roadmap. + +- [ ] **Step 1: Roadmap `docs/roadmaps/rust-migration.md` Phase 6a:** + - In the guard-matrix bullet (~`:812-816`), remove `unphased_union` and `"variant-windows"` (both ship now); move them to a supported note. + - Delete the gate footnote (~`:822-826`) claiming variant-windows parity is untested — it is covered by `test_svar2_readbound_variants.py` + `test_svar2_fields_read.py`. + - Amend the 2026-07-05 notes-log entry (~`:890`) OR add a 2026-07-13 entry reflecting shipped scope (unphased_union + variant-windows + INFO/FORMAT field routing done). + - Add a ticked task line: `- [x] var_fields → .svar2 store INFO/FORMAT field routing (plan 2026-07-12-svar2-info-format-field-routing.md).` + - Fill the `_PR: TBD (branch svar2-m6b-kernel)_` link once the PR exists (Task 13 opens it — come back and fill the number, or leave 🚧 until then). + +- [ ] **Step 2: Add the RELEASE-GATE subsection** under Phase 6a in the roadmap: +```markdown +#### ⛔ Release gate (do NOT merge until genoray is released) + +This branch is dev-wired to a local genoray checkout and cannot build off this +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API lives on genoray main (unreleased). Flip ALL of these at +genoray release, then re-run the full py3xx matrix: + +- `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → published crates.io versions. +- `pixi.toml` [feature.py310.pypi-dependencies]: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"`. +- Verify the version-floor bumps already made are intended: numpy 0.29, pyo3 0.29, seqpro 0.21.1. +``` + +- [ ] **Step 3: `skills/genvarloader/SKILL.md` corrections:** + - `:193-195`: `var_fields` on `.svar2` accepts only `alt|ilen|start` + store INFO/FORMAT fields — NOT `ref`/`dosage` (they raise). State it. + - `:66,170,437`: "`min_af`/`max_af` requires SVAR-backed genotypes" → "`.svar` only (not `.svar2`, which raises `NotImplementedError`)". + - `:128,442`: note `extend_to_length` is unsupported for a `.svar2` source (raises — matches Task 5). + - `:91`: qualify "byte-identical … all four output modes" with "except pure-deletion ALT bytes (see below)". + +- [ ] **Step 4: Prose docs `docs/source/`:** + - `index.md:51`: "Currently supports VCF, PGEN, and BigWig" → mirror README's `.svar`/`.svar2` wording. + - `format.md`: add the `.svar2` guard-matrix list (unsupported combos) that `faq.md:81` and `write.md:98` promise "for the full list"; OR repoint those two to `faq.md`. Pin the `(unreleased)` changelog row (`:145`) to the target version. + - `write.md` §"Variants from a genoray sparse store": add a 2-line build snippet (how to create `.svar`/`.svar2` via genoray), since `faq.md:76` promises it. + - `dataset.md`: add a short "Variant fields (`var_fields`)" section — `.svar2` store INFO/FORMAT fields on `variants`/`variant-windows` (`rv["AF"]`, `win.fields["AF"]`), with the "only alt/ilen/start + store fields" caveat. + +- [ ] **Step 5: Verify api.md ↔ __all__ still clean** (no public symbol added by this branch): + +Run: `pixi run -e dev python -c "import genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')"` +Expected: `MISSING: none`. + +- [ ] **Step 6: Docs build sanity** (optional but preferred): + +Run: `pixi run -e docs doc 2>&1 | tail -5` +Expected: build succeeds (or the same warnings as `main` — compare; no NEW errors). + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add docs/ skills/ +git commit --no-verify -m "$(cat <<'EOF' +docs(svar2): sync roadmap/skill/prose docs + add release-gate checklist + +Roadmap: move unphased_union/variant-windows to supported, drop the stale +gate footnote, add the field-routing task line, add a ⛔ release-gate section +for the genoray path-pins. SKILL: correct var_fields (.svar2 = alt/ilen/start ++ store fields, no ref/dosage), min_af/max_af = .svar only, extend_to_length +unsupported for .svar2. Prose: index.md source list, format.md guard matrix + +changelog pin, write.md build snippet, dataset.md var_fields section. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 13: `tmp/svar2_mvp/` relocation, final full-suite gate, push, draft PR + +Relocate the scratch tree into `tests/benchmarks/`, run the complete verification gate, and open the draft PR with the release-gate warning. + +**Files:** +- Move: benchmark/profiling drivers from `tmp/svar2_mvp/` → `tests/benchmarks/` and `tests/benchmarks/profiling/` +- Delete: `.sbatch` files, `env_baseline.txt`, `prof_out/*.md` (from git) +- Modify: `.gitignore` (drop the `tmp/svar2_mvp/prof_out/` line; add `tmp/`) + +**Interfaces:** +- Consumes: existing `tests/benchmarks/conftest.py` fixtures (`data_dir`, `kg_dir`, etc.). +- Produces: no machine-specific scratch tracked in git; useful drivers live under `tests/benchmarks/` with parameterized paths. + +- [ ] **Step 1: Inventory the tracked tmp files:** + +Run: `git ls-files tmp/svar2_mvp/; echo "--- untracked ---"; git status --short tmp/svar2_mvp/` + +- [ ] **Step 2: Move the reusable drivers** (`benchmark.py`, `bench_gvl_svar1_vs_svar2.py`, `build_stores.py`, `validate.py`, `split_folded.py`, `prof_*.py`, `perf_kernel_driver.py`) into `tests/benchmarks/`, and the perf shells (`prof_perf.sh`, `e1_profile.sh`, `prof_python.py`) into `tests/benchmarks/profiling/`: +```bash +git mv tmp/svar2_mvp/benchmark.py tests/benchmarks/bench_svar2_getitem.py +# ...repeat per file, choosing descriptive names; use `git add` for currently-untracked ones after moving +``` +For untracked files (`bench_gvl_svar1_vs_svar2.py`, `perf_kernel_driver.py`, `prof_cprofile.py`), `mv` then `git add` at the new path. + +- [ ] **Step 3: Parameterize hardcoded paths.** Replace `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` and similar absolutes with the `tests/benchmarks/conftest.py` fixtures where the file becomes a pytest module, or a module-level constant + `argparse` default for standalone scripts. Match the pattern in the existing `tests/benchmarks/profiling/profile_*.py`. + +- [ ] **Step 4: Drop machine-specific scratch from git:** +```bash +git rm tmp/svar2_mvp/e2_build.sbatch tmp/svar2_mvp/e2_subsample.sbatch tmp/svar2_mvp/genoray_debug_build.sbatch tmp/svar2_mvp/e1_bucket_dso.py tmp/svar2_mvp/env_baseline.txt +git rm -r tmp/svar2_mvp/prof_out/ +git rm tmp/svar2_mvp/prof_driver.py tmp/svar2_mvp/prof_getitem.py 2>/dev/null || true +``` +(Keep only what's genuinely reusable; the perf conclusions already live in `docs/superpowers/notes/` and the roadmap.) + +- [ ] **Step 5: Fix `.gitignore`** — remove the self-contradicting `tmp/svar2_mvp/prof_out/` line, add a blanket `tmp/`: +```bash +grep -v "tmp/svar2_mvp/prof_out/" .gitignore > .gitignore.new && mv .gitignore.new .gitignore +printf 'tmp/\n' >> .gitignore +``` +Then confirm `tmp/` is gone from tracking: `git status --short tmp/` should show nothing staged except the removals/moves. + +- [ ] **Step 6: Sanity-run one relocated benchmark** (if it became a pytest module) or import-check it: +```bash +pixi run -e dev python -c "import ast; ast.parse(open('tests/benchmarks/bench_svar2_getitem.py').read()); print('parses')" +``` +Expected: `parses` (full run needs real data; a parse + import check is enough for the gate). + +- [ ] **Step 7: FINAL FULL-SUITE GATE.** Rebuild and run everything: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev cargo test 2>&1 | tail -6 +pixi run -e dev pytest tests -q 2>&1 | tail -6 +pixi run -e dev ruff check python/ tests/ 2>&1 | tail -2 +pixi run -e dev ruff format --check python/ tests/ 2>&1 | tail -2 +pixi run -e dev typecheck 2>&1 | tail -3 +pixi run -e dev cargo clippy --all-targets 2>&1 | grep -c "warning:" +``` +Expected: cargo tests pass; full pytest tree green (SVAR1 unchanged, SVAR2 31/31, all new tests pass); ruff clean; ruff format clean; typecheck 0 errors; clippy warning count == pre-existing baseline. + +- [ ] **Step 8: Commit the relocation** +```bash +git checkout pixi.lock 2>/dev/null +git add -A +git commit -m "$(cat <<'EOF' +chore(svar2): relocate tmp/svar2_mvp scratch into tests/benchmarks + +Move the reusable SVAR2 benchmark/profiling drivers under tests/benchmarks/ +with parameterized paths (via the existing conftest fixtures), drop +machine-specific sbatch/env scratch and prof_out reports from git (perf +conclusions already live in docs/superpowers/notes/ + the roadmap), and fix +the self-contradicting .gitignore (blanket-ignore tmp/). + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +- [ ] **Step 9: Push the branch:** +```bash +git push -u origin svar2-m6b-kernel 2>&1 | tail -3 +``` + +- [ ] **Step 10: Open the draft PR with the release-gate warning:** +```bash +gh pr create --draft --repo mcvickerlab/GenVarLoader --base master --head svar2-m6b-kernel \ + --title "SVAR2 read-bound dataset support (M6b)" \ + --body "$(cat <<'EOF' +## ⛔ DO NOT MERGE until genoray is released + +This branch is dev-wired to a local genoray checkout and builds only on the dev +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API is on genoray main (unreleased). Before merge, flip ALL: + +- [ ] `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → crates.io versions +- [ ] `pixi.toml`: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="` +- [ ] `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"` +- [ ] Re-run the full py3xx matrix on the released wheel + +## Summary + +Adds SVAR2 (`.svar2` sparse variant format) as a `gvl.write` source and a live, +read-bound `Dataset` backend: all-Rust FFI kernels (haplotypes, tracks, +variants, variant-windows), INFO/FORMAT field routing into variant outputs via +`var_fields`, and `unphased_union`. See `docs/roadmaps/rust-migration.md` +Phase 6a and the specs/plans under `docs/superpowers/`. + +Final pre-merge pass (this session): serial-unsafe-path guard, Python-reachable +panics → PyValueError, `extend_to_length` guard, vectorized write-time +max_ends, test-only oracle relocation, dead FFI capability removal, Rust +de-duplication, typecheck-task fix, and doc consistency. + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" 2>&1 | tail -3 +``` + +- [ ] **Step 11: Backfill the PR link** into `docs/roadmaps/rust-migration.md` Phase 6a (`_PR: TBD_` → the PR URL from Step 10), commit `--no-verify`, and push. + +- [ ] **Step 12: Report** the PR URL and the final gate results. + +--- + +## Self-Review + +**Spec coverage:** +- §Release gate → Task 12 (roadmap section) + Task 13 (PR warning). ✅ +- §1a serial unsafe guard → Task 3. §1b get_unchecked doc → Task 1. §1c panics→PyErr → Task 4. §1d extend_to_length → Task 5. ✅ +- §2a oracle relocation → Task 9. §2b dead FFI capability → Task 10. §2c max_ends vectorize → Task 6. §2d region_starts → Task 7. §2e typecheck task → Task 2. §2f tmp/ relocation → Task 13. ✅ +- §3a carve helper, §3b FFI preamble, §3c present_bit → Task 11. ✅ +- §4a roadmap, §4b SKILL, §4c prose, §4d strip internal refs → Tasks 12 + 1. §4e missing docstrings → covered opportunistically in Tasks 1/9 (see note below). ✅ +- §5 clippy → Task 11 Step 9. ✅ +- §8 FlankSample issue → Task 8. ✅ + +**Note on §4e (missing docstrings):** the spec lists docstrings to add (`make_svar2_link`, `_reconstruct_variants`, `_write_from_svar2`, `svar2/store.rs` items). These are folded into the tasks that already touch those files: `_write_from_svar2` in Task 5, the `svar2/store.rs` PyO3 items are low-risk — **add a short Step to Task 10** or fold into Task 1's comment pass. To avoid a gap, the implementer should add the four docstrings as part of whichever task first opens each file; if none does (e.g. `store.rs`), append them to Task 1. Explicitly: add `///` to `svar2/store.rs` `reader`/`store_path`/`#[new]`/`contigs` in Task 1 Step 3. + +**Placeholder scan:** no TBD/TODO left except the deliberate `_PR: TBD` which Task 13 Step 11 fills. The Task 4 Step 1 test body has a modeled-on-sibling `...` — flagged explicitly with instructions, acceptable since the exact fixture is repo-specific and the assertion (`pytest.raises(ValueError)` on a strided view) is concrete. + +**Type consistency:** helper names consistent across tasks — `require_contiguous` (Task 4), `carve_chunks`/`offsets_from_diffs`/`svar2::present_bit` (Task 11), `build_readbound_*`/`SparseVar2Source` at new paths (Task 9). `_svar2_region_max_ends` signature unchanged (Task 6). `reconstruct_haplotypes_from_svar2_readbound` loses `annot_*` in Task 10 — no later task references those params. From f79025c0af81aa36f7b24987cb4f22a8df5e595e Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 11:14:15 -0700 Subject: [PATCH 090/108] chore(svar2): make typecheck task check explicit paths Bare `pyrefly check` matches zero files inside a .claude/worktrees checkout (root .gitignore ignores .claude/, pyrefly honors ignore files), so typecheck silently passed on nothing. Point it at python/genvarloader + tests, fix the pre-commit hook to match, and clear the now-flagged unused pyrefly-ignore. Co-Authored-By: Claude Opus 4.8 (1M context) --- .pre-commit-config.yaml | 2 +- pixi.toml | 2 +- python/genvarloader/_ragged.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dc6a879..42319117 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: # (e.g. from `pixi global install pyrefly`) would otherwise be picked # up by `git push` outside of a `pixi run` context and resolve the # wrong Python interpreter. - entry: pixi run -e dev pyrefly check + entry: pixi run -e dev pyrefly check python/genvarloader tests language: system pass_filenames: false - id: pixi-lock diff --git a/pixi.toml b/pixi.toml index 4bfc3618..d0cd35ec 100644 --- a/pixi.toml +++ b/pixi.toml @@ -159,7 +159,7 @@ test-join-audit = { cmd = "pytest tests -p tests._join_audit_plugin", depends-on "gen", "gen-1kg", ] } -typecheck = { cmd = "pyrefly check" } +typecheck = { cmd = "pyrefly check python/genvarloader tests" } bench = { cmd = "pytest tests/benchmarks --codspeed -p no:cov" } bench-local = { cmd = "pytest tests/benchmarks --benchmark-only -p no:cov" } # perf on the Python process (NOT py-spy --native, which slows deep-stack paths ~10x). diff --git a/python/genvarloader/_ragged.py b/python/genvarloader/_ragged.py index 10fcdd66..3b2613d8 100644 --- a/python/genvarloader/_ragged.py +++ b/python/genvarloader/_ragged.py @@ -322,7 +322,7 @@ def to_padded(rag: Ragged[RDTYPE], pad_value: Any) -> NDArray[RDTYPE]: leading = rag.shape[:rag_dim] if leading: - out = out.reshape((*leading, out_len)) # pyrefly: ignore[no-matching-overload] + out = out.reshape((*leading, out_len)) return out From 0bb676e3551840988df22d8dee8190b31631c499 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 11:25:00 -0700 Subject: [PATCH 091/108] docs(svar2): strip stale/internal references from shipped comments Fix the reverted-get_unchecked test docstring, the stale unphased_union out-of-scope claim in the Svar2Haps module docstring, and internal plan/task numbering ("Task 7c", "Phase-1 wiring", "FIX 1", "Task 13/14/15", etc.) that leaked into shipped comments and docstrings. Also add missing docstrings to the svar2/store.rs PyO3 items, make_svar2_link, and _reconstruct_variants. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_reconstruct.py | 10 ++++--- python/genvarloader/_dataset/_svar2_haps.py | 28 +++++++++++++++----- python/genvarloader/_dataset/_svar2_link.py | 17 ++++++++++++ python/genvarloader/_dataset/_write.py | 8 +++--- src/ffi/mod.rs | 19 ++++++------- src/lib.rs | 2 +- src/svar2/mod.rs | 4 +-- src/svar2/store.rs | 4 +++ src/tracks/mod.rs | 6 ++--- 9 files changed, 69 insertions(+), 29 deletions(-) diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index 6c8058dd..a240e5dc 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -41,7 +41,7 @@ from ._utils import _ffi_array from .._threads import should_parallelize -# Fused tracks entry (Task 14): intervals → scratch → realign, one FFI crossing. +# Fused tracks entry: intervals → scratch → realign, one FFI crossing. # Imported at module level so the spy in test_fused_tracks_parity can monkeypatch it. from ..genvarloader import ( intervals_and_realign_track_fused as intervals_and_realign_track_fused, @@ -140,7 +140,7 @@ def __call__( flat: bool = False, to_rc: "NDArray[np.bool_] | None" = None, ) -> tuple[_H, _T]: - # SVAR2 read path (Task 7c): route to the split materialize→realign + # SVAR2 read path: route to the split materialize→realign # kernel. The isinstance guard keeps the SVAR1 body below byte-unchanged. from ._svar2_haps import Svar2Haps @@ -396,8 +396,10 @@ def _call_svar2( ] strat_ids, strat_params = _lower_insertion_fills(strat_list) - # FIX 1 guard: FlankSample (the only seed-dependent fill) diverges - # from SVAR1 across MULTIPLE contigs. SVAR1 realigns the whole batch + # Multi-contig FlankSample track fills seed the fill value off a + # contig-local query index, which diverges from the global fill seed + # across a batch; guarded until the fill-seed index is made global. + # SVAR1 realigns the whole batch # in ONE fused call, so the fill hash `hash4(base_seed, query, hap, # out_idx+i)` uses the GLOBAL row `query`. `_call_svar2` calls the # readbound kernel once PER CONTIG GROUP, where `query = k/ploidy` is diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 902075e5..19037c85 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -18,9 +18,10 @@ ``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2._find_ranges`` over the full cohort). -Out of scope for this plan (guarded with ``NotImplementedError``): spliced -output, ``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated -haps, in-kernel reverse-complement, and ``unphased_union``. +Out of scope (guarded with ``NotImplementedError``): spliced output, +``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated haps, and +in-kernel reverse-complement. (``unphased_union`` and ``variant-windows`` ARE +supported.) """ from __future__ import annotations @@ -380,7 +381,7 @@ def get_haps_and_shifts( ]: """Reconstruct haplotypes + return the SVAR1-shaped 7-tuple. - The tracks follow-up (7c) reuses this for the shared shifts/diffs/ + Track re-alignment reuses this for the shared shifts/diffs/ hap_lengths; ``geno_offset_idx`` is a placeholder for svar2 (the cache is re-sliced from ``idx`` there), and ``keep``/``keep_offsets`` are None. """ @@ -483,10 +484,10 @@ def get_haps_and_shifts( geno_offset_idx = np.repeat( np.asarray(idx, np.intp)[:, None], P, axis=1 - ) # svar2 placeholder; 7c re-slices the cache from idx. + ) # svar2 placeholder; realign_track_block re-slices the cache from idx. return out, geno_offset_idx, shifts, diffs, hap_lengths, None, None - # ---- tracks (7c) ---- + # ---- tracks ---- def realign_track_block( self, @@ -637,6 +638,21 @@ def _type_field_bufs( def _reconstruct_variants( self, idx: NDArray[np.integer], regions: NDArray[np.integer] ) -> RaggedVariants: + """Decode the per-query variant records (SoA) for a query block. + + Parameters + ---------- + idx + Flat ``(region, sample)`` query indices for the block. + regions + ``(n_regions, 3)`` array of ``(contig_id, start, end)``. + + Returns + ------- + RaggedVariants + Per-query ragged variant records (position, indel length, ALT + bytes, and any requested INFO/FORMAT fields). + """ regions = np.asarray(regions, np.int32) P = int(self.genotypes.shape[-2]) b = len(idx) diff --git a/python/genvarloader/_dataset/_svar2_link.py b/python/genvarloader/_dataset/_svar2_link.py index e551a36e..653c43f5 100644 --- a/python/genvarloader/_dataset/_svar2_link.py +++ b/python/genvarloader/_dataset/_svar2_link.py @@ -107,6 +107,23 @@ def _verify_svar2_fingerprint(svar2_path: Path, link: Svar2Link | None) -> None: def make_svar2_link(gvl_path: Path, svar2_path: Path) -> Svar2Link: + """Build a :class:`Svar2Link` recording the on-disk relationship between + a gvl dataset and the ``.svar2`` store it reads from. + + Parameters + ---------- + gvl_path + Path to the gvl dataset directory. + svar2_path + Path to the ``.svar2`` store the dataset links to. + + Returns + ------- + Svar2Link + Relative/absolute paths to the store plus its fingerprint + (file count and total byte size), used to detect a moved or + modified store on open. + """ svar2_resolved = svar2_path.resolve() n_files, store_bytes = _svar2_store_fingerprint(svar2_resolved) return Svar2Link( diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 20da2be1..c0acba57 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -36,7 +36,7 @@ from tqdm.auto import tqdm from .._atomic import atomic_dir -from .._ragged import INTERVAL_DTYPE # noqa: F401 # Task 3 migration reader imports this +from .._ragged import INTERVAL_DTYPE # noqa: F401 # kept for the migration reader import from .._utils import lengths_to_offsets, normalize_contig_name from .._variants._utils import path_is_pgen, path_is_vcf from ._svar2_link import Svar2Link @@ -1192,9 +1192,9 @@ def _write_from_svar2( rc = df.height starts = df["chromStart"].to_numpy() ends = df["chromEnd"].to_numpy() - # extend_to_length fixed-output-length write-time handling is out of - # scope for this Phase-1 wiring; the read-bound kernel does its own - # output-length sizing at read time regardless of this flag. + # Write-time fixed-output-length (extend_to_length) handling is not + # supported for a `.svar2` source; the read-bound kernel sizes + # haplotype output at read time regardless of this flag. d = svar2._find_ranges(c, starts, ends, samples=samples) # _find_ranges returns row-major (R*S*P, 2) for vk ranges; reshape into (R,S,P,2). diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 7ce40226..1a64d591 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -603,7 +603,7 @@ pub fn reconstruct_haplotypes_from_sparse( }); } -/// Fused haplotypes __getitem__ kernel (Task 13). +/// Fused haplotypes __getitem__ kernel. /// /// Collapses two FFI crossings into one: /// 1. Compute per-haplotype length diffs (``get_diffs_sparse`` logic). @@ -620,7 +620,8 @@ pub fn reconstruct_haplotypes_from_sparse( /// layout as the existing ``reconstruct_haplotypes_from_sparse`` FFI entry). /// /// Annotation buffers are not supported in the fused entry (annotated path -/// remains on the unfused dispatch wrappers — see Task 13 report for rationale). +/// remains on the unfused dispatch wrappers, which carry the extra +/// `annot_*` output buffers this fused entry does not allocate). /// `parallel` enables rayon batch parallelism (caller computes `should_parallelize`). #[pyfunction] #[allow(clippy::too_many_arguments)] @@ -771,7 +772,7 @@ pub fn reconstruct_haplotypes_fused<'py>( /// - ``-1`` → ragged mode (each haplotype gets its natural length = ref_len + diff). /// - ``>= 0`` → fixed-length mode (every haplotype is padded/truncated to this length). /// -/// No annotation, no to_rc — first cut minimal, mirrors the plain fused path. +/// No annotation, no to_rc; mirrors the plain fused path. #[pyfunction] #[allow(clippy::too_many_arguments)] pub fn reconstruct_haplotypes_from_svar2<'py>( @@ -2157,7 +2158,7 @@ pub fn tracks_to_intervals<'py>( ) } -/// Fused per-track __getitem__ kernel (Task 14). +/// Fused per-track __getitem__ kernel. /// /// Collapses two FFI crossings into one per track: /// 1. ``intervals_to_tracks`` core: fills a Rust-side scratch buffer from @@ -2299,9 +2300,9 @@ pub fn intervals_and_realign_track_fused( Ok(()) } -// ── Task 3: guard test — drives rc_flat_rows_inplace on a synthetic hap buffer ─ -// ── Task 4: guard test — drives reverse_flat_rows_inplace:: (reverse only) ─ -// ── Task 6: guard test — proves per-element masking over permuted offsets ──────── +// ── guard test — drives rc_flat_rows_inplace on a synthetic hap buffer ─ +// ── guard test — drives reverse_flat_rows_inplace:: (reverse only) ─ +// ── guard test — proves per-element masking over permuted offsets ──────── #[cfg(test)] mod tests { #[test] @@ -2349,9 +2350,9 @@ mod tests { } } -// ── DEBUG exports for PRNG parity tests (Task 7) ───────────────────────────── +// ── DEBUG exports for PRNG parity tests ───────────────────────────── // These thin wrappers exist solely to make the Rust PRNG functions callable from -// Python tests. Decision (final-review, Task 15): KEEP permanently as the direct +// Python tests. Decision: KEEP permanently as the direct // PRNG parity guard. The njit-internal xorshift64/hash4 leaves have no other // Python entry point, so these are the only way to assert byte-identity of the // PRNG core from test_prng_parity.py. Do NOT remove. diff --git a/src/lib.rs b/src/lib.rs index 80338888..eed4c43e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,7 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; - // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review + // DEBUG: PRNG parity exports used by the Python parity tests. m.add_function(wrap_pyfunction!(ffi::_debug_xorshift64, m)?)?; m.add_function(wrap_pyfunction!(ffi::_debug_hash4, m)?)?; Ok(()) diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index 98239b7d..bf87a274 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -277,7 +277,7 @@ pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { pub struct FieldGather { pub views: [FieldView; 4], pub is_format: bool, - /// `dtype.width_bytes()`; consumed by the FFI caller (Task 1.3), not here. + /// `dtype.width_bytes()`; consumed by the FFI caller, not here. pub width: usize, pub cohort_n_samples: usize, } @@ -797,7 +797,7 @@ mod tests { /// h / ploidy` computed via an incrementing counter instead of a division /// (needs `ploidy > 1` so some consecutive haps share a `q`, which the /// single-hap tests above never trigger), and (2) the `present_bit` - /// closure's now-`get_unchecked` read of `dense_present`, with a mix of + /// closure's read of `dense_present`, with a mix of /// present/absent bits whose per-hap `base_bit` windows straddle a byte /// boundary (hap 1's 5-bit window covers global bits 5..10, i.e. bytes 0 /// and 1). diff --git a/src/svar2/store.rs b/src/svar2/store.rs index 6e8c35df..1e7f6549 100644 --- a/src/svar2/store.rs +++ b/src/svar2/store.rs @@ -13,9 +13,11 @@ pub struct Svar2Store { } impl Svar2Store { + /// Returns the cached `ContigReader` for `contig`, if one was opened. pub fn reader(&self, contig: &str) -> Option<&ContigReader> { self.readers.get(contig) } + /// Returns the filesystem path the store was opened from. pub fn store_path(&self) -> &str { &self.store_path } @@ -23,6 +25,7 @@ impl Svar2Store { #[pymethods] impl Svar2Store { + /// Opens one query-only `ContigReader` per contig under `store_path`. #[new] fn new( store_path: &str, @@ -43,6 +46,7 @@ impl Svar2Store { }) } + /// Returns the sorted list of contigs with an opened reader. fn contigs(&self) -> Vec { let mut v: Vec = self.readers.keys().cloned().collect(); v.sort(); diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index b7dbb5d6..68260c04 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -473,8 +473,8 @@ pub fn shift_and_realign_track_sparse( /// Shift and realign tracks for a batch of (query, hap) pairs in place (writes `out`). /// /// Mirrors numba `shift_and_realign_tracks_sparse` (lines 141-228 of `_tracks.py`) -/// statement-by-statement. Serial-only (rayon deferred to Phase 5, matching Task 5 -/// precedent for initial parity verification). +/// statement-by-statement. Serial-only (rayon deferred) for initial parity +/// verification. /// /// # Parameters /// - `out`: flat output buffer (f32), written in place @@ -2464,7 +2464,7 @@ mod tests { } // ------------------------------------------------------------------ // - // shift_and_realign_tracks_from_svar2 (Task 4 Part C) // + // shift_and_realign_tracks_from_svar2 // // ------------------------------------------------------------------ // /// SVAR2 two-source track driver: a single pure DEL, carried entirely in the From 688ddc950b1f0605ebf836b6d521f80f410f7328 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 11:35:19 -0700 Subject: [PATCH 092/108] fix(svar2): guard serial unsafe carve path with monotonicity debug_assert The parallel split_at_mut path already debug_asserts out_offsets is non-decreasing; the serial raw-pointer fallback carved out_e - out_s with no guard, so a non-monotonic offsets array underflows to a multi-GB slice (UB). Hoist the same assert into both serial loops. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/reconstruct/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index d6f71de4..fb035562 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -547,6 +547,10 @@ pub fn reconstruct_haplotypes_from_sparse( for k in 0..n_work { let out_s = out_offsets[k] as usize; let out_e = out_offsets[k + 1] as usize; + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); // SAFETY: `out_offsets` is required by the calling contract to be monotonically // non-decreasing, so consecutive (out_s, out_e) pairs are strictly non-overlapping @@ -847,6 +851,10 @@ pub fn reconstruct_haplotypes_from_svar2( for k in 0..n_work { let out_s = out_offsets[k] as usize; let out_e = out_offsets[k + 1] as usize; + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); // SAFETY: `out_offsets` is required by the calling contract to be monotonically // non-decreasing, so consecutive (out_s, out_e) pairs are strictly non-overlapping From ec99aba51c37ecf8bf9ae7d77c092aa8119a9380 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:12:45 -0700 Subject: [PATCH 093/108] fix(svar2): raise PyValueError on non-contiguous / OOB Python input Non-C-contiguous numpy views (a[::2]) of the reference/track arrays panicked inside the readbound kernels (contig_ref.as_slice().unwrap(), tracks.as_slice().unwrap()) instead of surfacing as ValueError. Validate contiguity of ref_ and tracks at the #[pyfunction] boundary, and bound the pure-DEL anchor read by requiring region ends within the contig ref length (sound: gather's half-open overlap guarantees variant pos < region end). The range/count arrays are consumed stride-safely (arr2_to_ranges / as_array) and need no check; the internal vk_src assert_eq is a gvl-guaranteed invariant (matches genoray's contract check) and is left as-is. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/ffi/mod.rs | 61 +++++++++++++++++ tests/dataset/test_svar2_readbound_haps.py | 80 ++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 1a64d591..bc739dfa 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -42,6 +42,26 @@ fn arr2_to_ranges(a: numpy::ndarray::ArrayView2) -> Vec> { .collect() } +/// Validate that a Python-supplied 1-D array is C-contiguous, or return a Python +/// `ValueError`. Some SVAR2 readbound kernels below slice these arrays and then +/// call `.as_slice()`/`.as_slice_mut()` on the result; a non-contiguous view (e.g. +/// a strided `a[::2]` slice) makes that call panic — a Rust panic surfaces to +/// Python as an uncatchable `pyo3_runtime.PanicException`, not a normal exception +/// — instead of raising. Validate at the FFI boundary so a bad input surfaces as a +/// clean `ValueError`. Only apply this to arrays actually consumed via +/// `.as_slice()` on a stride-preserving view downstream; arrays consumed via +/// `.as_array()` + direct ndarray indexing, `arr2_to_ranges`, or `.to_vec()` are +/// already stride-safe and must NOT be gated here (doing so would reject +/// currently-valid strided inputs). +fn require_contiguous_1d( + arr: &PyReadonlyArray1, + name: &str, +) -> PyResult<()> { + arr.as_slice().map(|_| ()).map_err(|_| { + pyo3::exceptions::PyValueError::new_err(format!("`{name}` must be C-contiguous")) + }) +} + /// Per-(query, hap) reference-length diffs (see `genotypes::get_diffs_sparse`). /// `geno_offsets` is the normalized (2, n) int64 starts/stops array. #[pyfunction] @@ -960,6 +980,40 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + // `ref_` is sliced (`ref_.slice(s![c_s..c_e])`) and then `.as_slice().unwrap()`'d + // inside `reconstruct::reconstruct_haplotypes_from_svar2` (src/reconstruct/mod.rs) — + // a non-contiguous `ref_` (e.g. `ref_[::2]`) panics there. `ref_offsets` is only + // ever indexed directly (`ref_offsets[c_idx]`), which is stride-safe, so it is not + // gated here. + require_contiguous_1d(&ref_, "ref_")?; + + // Guard the pure-DEL anchor-base read inside + // `reconstruct::reconstruct_haplotypes_from_svar2` + // (`&contig_ref_s[pos as usize..pos as usize + 1]`, src/reconstruct/mod.rs), which + // panics (index out of bounds) if `pos >= contig_ref_s.len()`. genoray's + // `find_ranges`/`gather_haps_readbound` gather off a half-open interval overlap + // search tree built from each query's `[q_start, q_end)`, so every variant + // gathered for query `q` is guaranteed `pos < q_end == region_bounds[q, 1]` (the + // same bound this readbound convention hands the search tree — see + // `genoray::_svar2_batch._find_ranges` / `genoray_core::query::gather::find_ranges`). + // Bounding the query window against the contig length here therefore + // transitively bounds every gathered variant's anchor read — no need to inspect + // post-gather positions, and no risk of rejecting a valid query. + { + let ref_offsets_check = ref_offsets.as_array(); + if let (Some(&c_s), Some(&c_e)) = (ref_offsets_check.get(0), ref_offsets_check.get(1)) { + let contig_ref_len = c_e - c_s; + for q in 0..n_q { + let region_end = region_bounds_a[[q, 1]] as i64; + if region_end > contig_ref_len { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "region end {region_end} exceeds contig {contig} length {contig_ref_len}" + ))); + } + } + } + } + let ref_a = ref_.as_array(); let ref_offsets_a = ref_offsets.as_array(); @@ -1211,6 +1265,13 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + // `tracks` is `.as_slice().expect("tracks must be contiguous (C-order)")`'d inside + // `tracks::shift_and_realign_tracks_from_svar2` (src/tracks/mod.rs) — a + // non-contiguous `tracks` (e.g. `tracks[::2]`) panics there. `track_offsets` and + // `params` are only ever indexed directly (`track_offsets[query]`, `params[0]`), + // which is stride-safe, so neither is gated here. + require_contiguous_1d(&tracks, "tracks")?; + let tracks_a = tracks.as_array(); let track_offsets_a = track_offsets.as_array(); let params_a = params.as_array(); diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py index cd3d0dee..0caea187 100644 --- a/tests/dataset/test_svar2_readbound_haps.py +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -131,6 +131,86 @@ def test_readbound_matches_union_oracle(svar2_store, regions): pytest.fail("data mismatch but no single hap slice differed (offset bug?)") +def test_readbound_haps_noncontiguous_ref_raises(svar2_store): + """A non-C-contiguous ``ref_`` view must surface as ``ValueError``, not a Rust + panic. + + ``build_readbound_haps`` (the Python oracle wrapper) defensively + ``np.ascontiguousarray``s ``ref_`` before handing it to the FFI, so it can't be + used to inject a strided array here -- this calls + ``reconstruct_haplotypes_from_svar2_readbound`` directly, replaying the same + ``_find_ranges`` marshalling ``build_readbound_haps`` does internally (see + ``genvarloader/_dataset/_svar2_store_py.py::build_readbound_haps``), but with a + genuinely non-contiguous ``ref_``. + """ + import genoray + + from genvarloader.genvarloader import ( + Svar2Store, + reconstruct_haplotypes_from_svar2_readbound, + ) + + contig = "chr1" + regions = [(0, 40)] + ref_bytes = _REF.encode() + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + # A strided (non-contiguous) view carrying the same bytes as `_REF`: double up + # each byte, then stride over every other one to recover the original values. + doubled = np.repeat(np.frombuffer(ref_bytes, np.uint8), 2) + ref_strided = doubled[::2] + assert ref_strided.flags["C_CONTIGUOUS"] is False + assert bytes(ref_strided) == ref_bytes + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + + d = sv._find_ranges( + contig, [s for s, _ in regions], [e for _, e in regions], samples=None + ) + region_starts_r = np.asarray(d["region_starts"], np.int64) + sample_cols = np.asarray(d["sample_cols"], np.int64) + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) + + R = len(regions) + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) + orig_samples = np.tile(sample_cols, R) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) + reg_arr = np.asarray(regions, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray(np.repeat(reg_arr, S, axis=0), np.int32) + shifts_a = np.zeros((n_q, P), dtype=np.int32) + + store = Svar2Store(str(sv.path), sv.contigs, sv.n_samples, sv.ploidy) + + with pytest.raises(ValueError): + reconstruct_haplotypes_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + ref_strided, + ref_offsets, + np.uint8(ord("N")), + np.int64(-1), + False, + ) + + def test_readbound_matches_union_oracle_with_shifts(svar2_store): """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" import genoray From ae0cd7958e2b8f4d9bfc6c61a4e578d153ad1198 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:25:35 -0700 Subject: [PATCH 094/108] fix(svar2): drop over-strict anchor region guard; assert at read site The Task-4 FFI-boundary guard rejected region_bounds[q,1] > contig_ref_len, but gvl regions legitimately extend past a contig end (jitter/max_jitter padding, right-padded with pad_char), so that guard broke valid boundary reads and SVAR1 parity on the live read path. The anchor OOB is only reachable via a corrupt store (a variant pos >= contig_len); guard that with a debug_assert at the read site instead. The ref_/tracks contiguity guards are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/ffi/mod.rs | 34 +++++++++------------------------- src/reconstruct/mod.rs | 9 +++++++++ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index bc739dfa..85abbb5a 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -987,32 +987,16 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( // gated here. require_contiguous_1d(&ref_, "ref_")?; - // Guard the pure-DEL anchor-base read inside + // NOTE: the pure-DEL anchor-base read inside // `reconstruct::reconstruct_haplotypes_from_svar2` - // (`&contig_ref_s[pos as usize..pos as usize + 1]`, src/reconstruct/mod.rs), which - // panics (index out of bounds) if `pos >= contig_ref_s.len()`. genoray's - // `find_ranges`/`gather_haps_readbound` gather off a half-open interval overlap - // search tree built from each query's `[q_start, q_end)`, so every variant - // gathered for query `q` is guaranteed `pos < q_end == region_bounds[q, 1]` (the - // same bound this readbound convention hands the search tree — see - // `genoray::_svar2_batch._find_ranges` / `genoray_core::query::gather::find_ranges`). - // Bounding the query window against the contig length here therefore - // transitively bounds every gathered variant's anchor read — no need to inspect - // post-gather positions, and no risk of rejecting a valid query. - { - let ref_offsets_check = ref_offsets.as_array(); - if let (Some(&c_s), Some(&c_e)) = (ref_offsets_check.get(0), ref_offsets_check.get(1)) { - let contig_ref_len = c_e - c_s; - for q in 0..n_q { - let region_end = region_bounds_a[[q, 1]] as i64; - if region_end > contig_ref_len { - return Err(pyo3::exceptions::PyValueError::new_err(format!( - "region end {region_end} exceeds contig {contig} length {contig_ref_len}" - ))); - } - } - } - } + // (`&contig_ref_s[pos as usize..pos as usize + 1]`, src/reconstruct/mod.rs) is + // in-bounds for all valid input: gathered variants come from within-contig records + // so `pos < contig_ref_len` always holds. It is NOT bounded here by the query + // window: gvl regions legitimately extend past the contig end (jitter / max_jitter + // padding, right-padded with `pad_char`), so `region_bounds[q, 1] > contig_ref_len` + // is a normal, valid read — rejecting it would break SVAR1 parity. The only way to + // reach the anchor OOB is a corrupt store (a variant `pos >= contig_len`); that is + // caught by a `debug_assert!` at the read site (fires in test/debug builds). let ref_a = ref_.as_array(); let ref_offsets_a = ref_offsets.as_array(); diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index fb035562..2906090a 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -704,6 +704,15 @@ pub fn reconstruct_haplotypes_from_svar2( // (off-by-one: -(|v_diff|+1) instead of -|v_diff|). SNP/INS/Lookup alleles are // already non-empty and pass through unchanged. let allele = if allele.is_empty() { + // The anchor base is `ref[pos]`. Valid gathered variants are within-contig + // records (`pos < contig_ref_s.len()`); a `pos` past the contig end can only + // come from a corrupt store. Assert it in debug/test builds rather than + // silently reading OOB (release keeps the raw slice for speed). + debug_assert!( + (pos as usize) < contig_ref_s.len(), + "pure-DEL anchor position {pos} is beyond contig ref length {} (corrupt store?)", + contig_ref_s.len() + ); std::borrow::Cow::Borrowed(&contig_ref_s[pos as usize..pos as usize + 1]) } else { allele From 0e1e5d61572e511d37a58bbfcc012466b48770de Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:29:34 -0700 Subject: [PATCH 095/108] fix(svar2): reject extend_to_length=False for .svar2 sources _write_from_svar2 accepted the flag and ignored it, silently extending chromEnd regardless. Raise NotImplementedError (Phase-1 guard-matrix policy) instead of producing a dataset that differs from what was requested. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_write.py | 13 ++++++++++--- tests/dataset/test_write_svar2.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index c0acba57..89227ca4 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1147,6 +1147,14 @@ def _write_from_svar2( # symbolic/breakend variants are rejected upstream at .svar2 conversion; the # store cannot represent them, and SparseVar2 exposes no index to re-check. + if not extend_to_length: + raise NotImplementedError( + "extend_to_length=False is not supported for a .svar2 variant source: " + "the read-bound kernel always sizes haplotype output at read time and " + "the write-time ranges cache is built for the extended chromEnd. Use a " + ".svar/VCF/PGEN source if you need un-extended haplotypes." + ) + out_dir = path / "genotypes" / "svar2_ranges" out_dir.mkdir(parents=True, exist_ok=True) @@ -1192,9 +1200,8 @@ def _write_from_svar2( rc = df.height starts = df["chromStart"].to_numpy() ends = df["chromEnd"].to_numpy() - # Write-time fixed-output-length (extend_to_length) handling is not - # supported for a `.svar2` source; the read-bound kernel sizes - # haplotype output at read time regardless of this flag. + # extend_to_length is validated at function entry (False raises); the + # read-bound kernel sizes haplotype output at read time. d = svar2._find_ranges(c, starts, ends, samples=samples) # _find_ranges returns row-major (R*S*P, 2) for vk ranges; reshape into (R,S,P,2). diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py index 8c028ca5..555ae6e5 100644 --- a/tests/dataset/test_write_svar2.py +++ b/tests/dataset/test_write_svar2.py @@ -313,3 +313,28 @@ def test_write_svar2_max_ends_same_pos_tie( f"same-POS tie: svar2 max_ends {chrom_end_2.tolist()} != " f"svar1 max_ends {chrom_end_1.tolist()}" ) + + +def test_svar2_extend_to_length_false_raises(svar2_store: Path, tmp_path: Path): + """extend_to_length=False is unsupported for a .svar2 source: it must raise + NotImplementedError, not silently produce an extended dataset.""" + from genoray import SparseVar2 + + svar2 = SparseVar2(svar2_store) + bed = pl.DataFrame( + { + "chrom": ["chr1", "chr1"], + "chromStart": [0, 5], + "chromEnd": [20, 15], + } + ) + out = tmp_path / "ds.gvl" + with pytest.raises(NotImplementedError, match="extend_to_length"): + gvl.write( + out, + bed, + variants=svar2, + samples=None, + extend_to_length=False, + overwrite=True, + ) From 3c0f17d0e72f42a0e4fec8057d3517bcf8e1e18f Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:33:05 -0700 Subject: [PATCH 096/108] perf(svar2): vectorize _svar2_region_max_ends (byte-identical) Replace the O(regions x samples x ploidy) write-time Python triple-loop with a per-region scatter-max over a (pos<<21)|end composite key that preserves the pos-then-end tie-break. A single haplotype never carries two variants at the same position, so a global per-region max over the selected samples' variants equals the original per-hap-argmax loop. Pinned byte-identical by a new equivalence test carrying the old loop as its oracle (asserted non-vacuous: the DEL at POS 11 extends overlapping regions). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_write.py | 52 ++++++++++++----------- tests/dataset/test_write_svar2.py | 58 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 89227ca4..e928996d 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1106,34 +1106,38 @@ def _svar2_region_max_ends( otherwise every extension would be off by one (masked in most regions because the un-extended ``chromEnd`` already dominates the max). - This is O(R * len(samples) * ploidy) Python iteration over decoded records -- - acceptable for correctness/tests; vectorize for large-cohort write perf as a - follow-up. + Vectorized as a per-region scatter-max over a ``(pos << 21) | end`` composite + key, which reproduces the pos-then-end tie-break exactly (a single haplotype + never carries two variants at the same position, so a global per-region max + over the selected samples' variants equals the original per-hap-argmax loop). """ R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy - sel = [svar2.available_samples.index(s) for s in samples] + sel = np.asarray([svar2.available_samples.index(s) for s in samples], np.int64) dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) - pos_arr = dec.data["pos"] - ilen_arr = dec.data["ilen"] - off = np.asarray(dec.offsets) + pos_arr = np.asarray(dec.data["pos"], np.int64) + ilen_arr = np.asarray(dec.data["ilen"], np.int64) + off = np.asarray(dec.offsets, np.int64) # length R*S_all*P + 1 out = np.asarray(ends, np.int64).copy() # default = chromEnd - for r in range(R): - best_pos, best_end = -1, -1 - for s in sel: - for p in range(P): - h = (r * S_all + s) * P + p - a, b = int(off[h]), int(off[h + 1]) - if a == b: - continue - seg_pos = pos_arr[a:b] - seg_ilen = ilen_arr[a:b] - j = int(np.argmax(seg_pos)) # highest-position variant in this hap - p_pos = int(seg_pos[j]) - p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) # +1: 0-based -> 1-based - if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): - best_pos, best_end = p_pos, p_end - if best_pos >= 0: - out[r] = best_end + if pos_arr.size: + n_hap = R * S_all * P + counts = np.diff(off) # variants per hap + hap_of_var = np.repeat(np.arange(n_hap), counts) # region-major hap per variant + s_of_hap = (np.arange(n_hap) // P) % S_all + keep = np.isin(s_of_hap[hap_of_var], sel) # only selected samples + region_of_var = hap_of_var // (S_all * P) + end_var = (pos_arr + 1) - np.minimum(ilen_arr, 0) # 0-based -> 1-based, extend on DEL + SHIFT = 21 + assert int(end_var.max(initial=0)) < (1 << SHIFT), ( + "end exceeds tie-break packing width" + ) + key = (pos_arr << SHIFT) | end_var + key_k = key[keep] + region_k = region_of_var[keep] + if key_k.size: + best = np.full(R, -1, np.int64) + np.maximum.at(best, region_k, key_k) # per-region max composite key + has = best >= 0 + out[has] = best[has] & ((1 << SHIFT) - 1) # unpack end return out.astype(np.int32) diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py index 555ae6e5..91b3ab0e 100644 --- a/tests/dataset/test_write_svar2.py +++ b/tests/dataset/test_write_svar2.py @@ -338,3 +338,61 @@ def test_svar2_extend_to_length_false_raises(svar2_store: Path, tmp_path: Path): extend_to_length=False, overwrite=True, ) + + +def _reference_region_max_ends(svar2, contig, starts, ends, samples): + """Byte-for-byte copy of the ORIGINAL _svar2_region_max_ends triple-loop, + kept here as the oracle that pins the vectorized rewrite byte-identical.""" + import numpy as np + + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = [svar2.available_samples.index(s) for s in samples] + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = dec.data["pos"] + ilen_arr = dec.data["ilen"] + off = np.asarray(dec.offsets) + out = np.asarray(ends, np.int64).copy() + for r in range(R): + best_pos, best_end = -1, -1 + for s in sel: + for p in range(P): + h = (r * S_all + s) * P + p + a, b = int(off[h]), int(off[h + 1]) + if a == b: + continue + seg_pos = pos_arr[a:b] + seg_ilen = ilen_arr[a:b] + j = int(np.argmax(seg_pos)) + p_pos = int(seg_pos[j]) + p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) + if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): + best_pos, best_end = p_pos, p_end + if best_pos >= 0: + out[r] = best_end + return out.astype(np.int32) + + +def test_svar2_region_max_ends_matches_reference(svar2_store: Path): + """Vectorized _svar2_region_max_ends must equal the original per-hap loop, + including the pos-then-end tie-break and the empty-region default = chromEnd.""" + from genoray import SparseVar2 + + from genvarloader._dataset._write import _svar2_region_max_ends + + svar2 = SparseVar2(svar2_store) + # Overlaps the DEL at 0-based POS 11 with varying windows + a no-variant + # region ([20,30]) so both the extension and keep-chromEnd branches run. + starts = np.array([0, 0, 5, 12, 20], dtype=np.int64) + ends = np.array([15, 20, 10, 13, 30], dtype=np.int64) + samples = list(svar2.available_samples) + + got = _svar2_region_max_ends(svar2, "chr1", starts, ends, samples) + ref = _reference_region_max_ends(svar2, "chr1", starts, ends, samples) + np.testing.assert_array_equal(got, ref) + + # Anti-vacuity: at least one region must be EXTENDED past its chromEnd (the + # DEL at POS 11 extends windows that overlap it), else the test only checks + # the trivial default path. + assert (got != ends.astype(np.int32)).any(), ( + f"test is vacuous: no region extended (got={got.tolist()}, ends={ends.tolist()})" + ) From c7073e71d939c98c7926644a71d885df404f2cde Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:39:52 -0700 Subject: [PATCH 097/108] fix(svar2): pack bounded tie-break in max_ends key (fix >2Mb overflow) The composite key packed the absolute end (~pos-sized) into 21 low bits, so any variant past ~2Mb into a contig would assert-fail the write (or, with -O, silently corrupt the per-region ordering). Real chromosomes are hundreds of Mb; the 40bp test fixture masked it. Pack the bounded ext = 1 - min(ilen,0) instead (end = pos + ext) and unpack end = (key>>SHIFT) + (key&mask). Add a large- position regression test (stub decode at 3-5Mb) that crashed the old packing. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_write.py | 16 +++++++---- tests/dataset/test_write_svar2.py | 38 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index e928996d..87e337f8 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1125,19 +1125,25 @@ def _svar2_region_max_ends( s_of_hap = (np.arange(n_hap) // P) % S_all keep = np.isin(s_of_hap[hap_of_var], sel) # only selected samples region_of_var = hap_of_var // (S_all * P) - end_var = (pos_arr + 1) - np.minimum(ilen_arr, 0) # 0-based -> 1-based, extend on DEL + # end = pos + ext, where ext = 1 - min(ilen, 0) (1-based bump plus the + # deletion extension: SNP/INS -> 1, DEL -> 1 + |ilen|). Pack the BOUNDED + # `ext` into the low bits, NOT the absolute `end` (which is ~pos-sized and + # would overflow past ~2 Mb into any contig), so the composite key orders + # by pos then by end; recover end = pos + ext on unpack. + ext_var = 1 - np.minimum(ilen_arr, 0) # small: 1 + deletion length SHIFT = 21 - assert int(end_var.max(initial=0)) < (1 << SHIFT), ( - "end exceeds tie-break packing width" + assert int(ext_var.max(initial=0)) < (1 << SHIFT), ( + "variant footprint exceeds tie-break packing width" ) - key = (pos_arr << SHIFT) | end_var + key = (pos_arr << SHIFT) | ext_var key_k = key[keep] region_k = region_of_var[keep] if key_k.size: best = np.full(R, -1, np.int64) np.maximum.at(best, region_k, key_k) # per-region max composite key has = best >= 0 - out[has] = best[has] & ((1 << SHIFT) - 1) # unpack end + # end = pos + ext = (key >> SHIFT) + (key & mask) + out[has] = (best[has] >> SHIFT) + (best[has] & ((1 << SHIFT) - 1)) return out.astype(np.int32) diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py index 91b3ab0e..f2fb8d5e 100644 --- a/tests/dataset/test_write_svar2.py +++ b/tests/dataset/test_write_svar2.py @@ -396,3 +396,41 @@ def test_svar2_region_max_ends_matches_reference(svar2_store: Path): assert (got != ends.astype(np.int32)).any(), ( f"test is vacuous: no region extended (got={got.tolist()}, ends={ends.tolist()})" ) + + +def test_svar2_region_max_ends_large_positions(): + """Regression: the composite key must pack a BOUNDED tie-break, not the + absolute end. A variant past ~2 Mb (real chromosomes are hundreds of Mb) + must not overflow the packing / assert-fail. Uses a stub whose decode returns + large positions so we can exercise realistic coordinates without a huge store. + """ + from types import SimpleNamespace + + import numpy as np + + from genvarloader._dataset._write import _svar2_region_max_ends + + # 2 regions x 1 sample x ploidy 1 = 2 haps, 1 variant each: + # region 0: SNP at pos 3_000_000 (ilen 0) -> end 3_000_001 + # region 1: DEL at pos 5_000_000 (ilen -2) -> end 5_000_003 + class _StubSvar2: + n_samples = 1 + ploidy = 1 + available_samples = ["S0"] + + def decode(self, contig, regions): + return SimpleNamespace( + data={ + "pos": np.array([3_000_000, 5_000_000], np.int64), + "ilen": np.array([0, -2], np.int64), + }, + offsets=np.array([0, 1, 2], np.int64), + ) + + svar2 = _StubSvar2() + starts = np.array([0, 0], np.int64) + ends = np.array([10, 10], np.int64) # small chromEnd so both variants extend + got = _svar2_region_max_ends(svar2, "chrBig", starts, ends, ["S0"]) + ref = _reference_region_max_ends(svar2, "chrBig", starts, ends, ["S0"]) + np.testing.assert_array_equal(got, ref) + np.testing.assert_array_equal(got, np.array([3_000_001, 5_000_003], np.int32)) From f70a22d77412502ad4575b4e7adffea5b634b66e Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:45:19 -0700 Subject: [PATCH 098/108] refactor(svar2): drop unused region_starts from the ranges cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit region_starts was written, memmapped into _Svar2Cache, and never fed to the FFI (its own docstring said "parity/debug only" — the read path recomputes per-query starts post-jitter from the regions array). Remove it from the write path, the svar2_meta.json schema, the Svar2Haps loader (deriving R from dense_snp_range instead), and the cache-contents test. The genoray _find_ranges "region_starts" output key and the fresh read-time array are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_svar2_haps.py | 10 ++++------ python/genvarloader/_dataset/_write.py | 5 +---- tests/dataset/test_write_svar2.py | 12 ------------ 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 19037c85..8d416586 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -84,16 +84,15 @@ class _Svar2Cache: ``vk_*_range`` are ``(R, S, P, 2)`` (per region/sample/ploid byte windows into the store's var_key tables); ``dense_*_range`` are ``(R, 2)`` (per-region, - sample-independent); ``region_starts`` is ``(R,)`` (write-time starts; kept for - parity/debug, NOT fed to the FFI — the read path uses post-jitter starts); - ``sample_cols`` is ``(S,)`` (selected slot -> original store sample index). + sample-independent); ``sample_cols`` is ``(S,)`` (selected slot -> original + store sample index). Per-query starts are recomputed post-jitter at read time, + so they are not cached here. """ vk_snp_range: NDArray[np.int64] vk_indel_range: NDArray[np.int64] dense_snp_range: NDArray[np.int64] dense_indel_range: NDArray[np.int64] - region_starts: NDArray[np.int64] sample_cols: NDArray[np.int64] @@ -229,7 +228,7 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: ranges_dir / name, dtype=np.int64, mode="r", shape=tuple(shape) ) - R = int(meta["region_starts"]["shape"][0]) + R = int(meta["dense_snp_range"]["shape"][0]) S = int(meta["vk_snp_range"]["shape"][1]) P = int(meta["ploidy"]) if P != ploidy: @@ -244,7 +243,6 @@ def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: dense_indel_range=_mm( "dense_indel_range.npy", meta["dense_indel_range"]["shape"] ), - region_starts=_mm("region_starts.npy", meta["region_starts"]["shape"]), sample_cols=np.load(ranges_dir / "sample_cols.npy"), ) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 87e337f8..2565dc00 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1177,7 +1177,6 @@ def _write_from_svar2( dense_indel = np.memmap( out_dir / "dense_indel_range.npy", np.int64, "w+", shape=(R, 2) ) - region_starts = np.memmap(out_dir / "region_starts.npy", np.int64, "w+", shape=(R,)) # sample_cols: selected slot -> original sample index (same for every contig). sample_cols = np.asarray( [svar2.available_samples.index(s) for s in samples], np.int64 @@ -1191,7 +1190,6 @@ def _write_from_svar2( "vk_indel_range": {"shape": [R, S, P, 2], "dtype": " np.ndarray: vk_indel = mm("vk_indel_range") # (R, S, P, 2) dense_snp = mm("dense_snp_range") # (R, 2) dense_indel = mm("dense_indel_range") # (R, 2) - region_starts_full = mm("region_starts") # (R,) # sample_cols is written with np.save (has a .npy header): read with np.load. sample_cols = np.load(rd / "sample_cols.npy") @@ -160,10 +152,6 @@ def mm(name: str) -> np.ndarray: df["chromEnd"].to_numpy(), samples=sorted_samples, ) - # region_starts: exact per-contig match (upcast int32 -> int64). - np.testing.assert_array_equal( - region_starts_full[lo:hi], np.asarray(d["region_starts"], np.int64) - ) # vk ranges: reshape (rc, S, P, 2) -> (rc*S*P, 2) must equal _find_ranges' # row-major (R*S*P, 2). This pins the reshape done in _write_from_svar2. np.testing.assert_array_equal( From 04931653b01b708aa1f8ee67f53df47d78365fd9 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 12:51:37 -0700 Subject: [PATCH 099/108] refactor(svar2): move test-only oracles out of the shipped package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SparseVar2Source and build_readbound_* have no importers under python/ — they are the parity oracle + FFI-input builders used only by tests. Move them to tests/_oracles/ (renaming _svar2_store_py.py -> svar2_readbound_inputs.py, since it holds no store class), convert their relative imports to absolute (genvarloader._flat, genvarloader.genvarloader, genvarloader._dataset. _rag_variants), and repoint the 7 test files' imports + the Svar2Haps docstring back-reference. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_svar2_haps.py | 2 +- tests/_oracles/__init__.py | 0 .../_oracles/svar2_readbound_inputs.py | 12 ++++++------ .../_oracles/svar2_source.py | 4 ++-- tests/dataset/test_svar2_dataset.py | 2 +- tests/dataset/test_svar2_readbound_diffs.py | 6 +++--- tests/dataset/test_svar2_readbound_haps.py | 14 +++++++------- tests/dataset/test_svar2_readbound_tracks.py | 8 ++++---- tests/dataset/test_svar2_readbound_variants.py | 4 ++-- tests/test_svar2_realign_tracks.py | 2 +- tests/test_svar2_reconstruct.py | 2 +- 11 files changed, 28 insertions(+), 28 deletions(-) create mode 100644 tests/_oracles/__init__.py rename python/genvarloader/_dataset/_svar2_store_py.py => tests/_oracles/svar2_readbound_inputs.py (97%) rename python/genvarloader/_dataset/_svar2_source.py => tests/_oracles/svar2_source.py (98%) diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 8d416586..169da6e2 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -13,7 +13,7 @@ ``decode_variants_from_svar2_readbound`` / ``hap_diffs_from_svar2_readbound``). The FFI-input shaping + output wrapping mirror -``_svar2_store_py.build_readbound_*`` exactly; the only difference is the source +``tests/_oracles/svar2_readbound_inputs.build_readbound_*`` (test oracle) exactly; the only difference is the source of the per-query ranges (this module slices the on-disk cache for the specific ``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2._find_ranges`` over the full cohort). diff --git a/tests/_oracles/__init__.py b/tests/_oracles/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/genvarloader/_dataset/_svar2_store_py.py b/tests/_oracles/svar2_readbound_inputs.py similarity index 97% rename from python/genvarloader/_dataset/_svar2_store_py.py rename to tests/_oracles/svar2_readbound_inputs.py index 71c4bf84..97cb2a55 100644 --- a/python/genvarloader/_dataset/_svar2_store_py.py +++ b/tests/_oracles/svar2_readbound_inputs.py @@ -3,7 +3,7 @@ NO interval-search-tree rebuild and NO dense-union rebuild. Byte-identical to the existing union-path oracle (``SparseVar2Source.reconstruct``, -``_svar2_source.py``), which calls ``reconstruct_haplotypes_from_svar2`` over +``svar2_source.py``), which calls ``reconstruct_haplotypes_from_svar2`` over ``SparseVar2._overlap_batch``'s eagerly-unioned dense channel. This module instead marshals ``SparseVar2._find_ranges``'s per-class-split ranges through ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` (Rust side) @@ -17,8 +17,8 @@ import numpy as np -from .._flat import _Flat -from ..genvarloader import ( +from genvarloader._flat import _Flat +from genvarloader.genvarloader import ( Svar2Store, decode_variants_from_svar2_readbound, hap_diffs_from_svar2_readbound, @@ -31,7 +31,7 @@ from numpy.typing import NDArray from seqpro.rag import Ragged - from ._rag_variants import RaggedVariants + from genvarloader._dataset._rag_variants import RaggedVariants def build_readbound_haps( @@ -243,7 +243,7 @@ def build_readbound_tracks( # `tracks`/`track_offsets` are per-REGION (R of them), but the kernel reads # `track_offsets` by `query` (= r*S+s) — expand the R track windows to R*S # by repeating each region's window S times (mirrors - # `SparseVar2Source.realign_tracks`, `_svar2_source.py:108-114`). + # `SparseVar2Source.realign_tracks`, `tests/_oracles/svar2_source.py`). t = np.asarray(tracks, np.float32) toff = np.asarray(track_offsets, np.int64) tracks_rs = ( @@ -296,7 +296,7 @@ def build_readbound_variants( variant set has no output-length dependency on the query region bounds (no overlap/clip filter; the gather already restricts to overlapping variants). """ - from ._rag_variants import RaggedVariants + from genvarloader._dataset._rag_variants import RaggedVariants reg = [(int(s), int(e)) for s, e in regions] R = len(reg) diff --git a/python/genvarloader/_dataset/_svar2_source.py b/tests/_oracles/svar2_source.py similarity index 98% rename from python/genvarloader/_dataset/_svar2_source.py rename to tests/_oracles/svar2_source.py index 3ed2fc39..d86f6d2f 100644 --- a/python/genvarloader/_dataset/_svar2_source.py +++ b/tests/_oracles/svar2_source.py @@ -20,8 +20,8 @@ import numpy as np from seqpro.rag import Ragged -from .._flat import _Flat -from ..genvarloader import ( +from genvarloader._flat import _Flat +from genvarloader.genvarloader import ( reconstruct_haplotypes_from_svar2, shift_and_realign_tracks_from_svar2, ) diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index 61e7e06e..f2627ef7 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -425,7 +425,7 @@ def test_svar2_variants_match_svar2_oracle( """ from genoray import SparseVar2 - from genvarloader._dataset._svar2_store_py import build_readbound_variants + from tests._oracles.svar2_readbound_inputs import build_readbound_variants _bcf, ref = _src _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) diff --git a/tests/dataset/test_svar2_readbound_diffs.py b/tests/dataset/test_svar2_readbound_diffs.py index d94627d1..8b097945 100644 --- a/tests/dataset/test_svar2_readbound_diffs.py +++ b/tests/dataset/test_svar2_readbound_diffs.py @@ -117,7 +117,7 @@ def _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) -> np.ndarray: Query order matches ``build_readbound_haps``/``build_readbound_diffs``: region-major, sample-minor (``q = r*S + s``), hap-minor within a query. """ - from genvarloader._dataset._svar2_store_py import build_readbound_haps + from tests._oracles.svar2_readbound_inputs import build_readbound_haps S, P = sv.n_samples, sv.ploidy R = len(regions) @@ -154,7 +154,7 @@ def _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) -> np.ndarray: def test_readbound_diffs_matches_implied_haps(svar2_store, regions): import genoray - from genvarloader._dataset._svar2_store_py import build_readbound_diffs + from tests._oracles.svar2_readbound_inputs import build_readbound_diffs contig = "chr1" ref_bytes = _REF.encode() @@ -178,7 +178,7 @@ def test_readbound_diffs_dense_snp_matches_implied_haps(svar2_store_dense_snp): reconstruct kernel implies.""" import genoray - from genvarloader._dataset._svar2_store_py import build_readbound_diffs + from tests._oracles.svar2_readbound_inputs import build_readbound_diffs contig = "chr1" ref_bytes = _REF.encode() diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py index 0caea187..11f53e61 100644 --- a/tests/dataset/test_svar2_readbound_haps.py +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -72,8 +72,8 @@ def svar2_store(tmp_path_factory) -> Path: def test_readbound_matches_union_oracle(svar2_store, regions): import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source - from genvarloader._dataset._svar2_store_py import build_readbound_haps + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps contig = "chr1" ref_bytes = _REF.encode() @@ -140,7 +140,7 @@ def test_readbound_haps_noncontiguous_ref_raises(svar2_store): used to inject a strided array here -- this calls ``reconstruct_haplotypes_from_svar2_readbound`` directly, replaying the same ``_find_ranges`` marshalling ``build_readbound_haps`` does internally (see - ``genvarloader/_dataset/_svar2_store_py.py::build_readbound_haps``), but with a + ``tests/_oracles/svar2_readbound_inputs.py::build_readbound_haps``), but with a genuinely non-contiguous ``ref_``. """ import genoray @@ -215,8 +215,8 @@ def test_readbound_matches_union_oracle_with_shifts(svar2_store): """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source - from genvarloader._dataset._svar2_store_py import build_readbound_haps + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps contig = "chr1" regions = [(0, 40), (5, 20)] @@ -317,8 +317,8 @@ def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): """ import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source - from genvarloader._dataset._svar2_store_py import build_readbound_haps + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps contig = "chr1" ref_bytes = _REF.encode() diff --git a/tests/dataset/test_svar2_readbound_tracks.py b/tests/dataset/test_svar2_readbound_tracks.py index e48f7b14..a8b4ddf1 100644 --- a/tests/dataset/test_svar2_readbound_tracks.py +++ b/tests/dataset/test_svar2_readbound_tracks.py @@ -90,8 +90,8 @@ def _synthetic_track_inputs(regions, seed=0): def test_readbound_tracks_match_union_oracle(svar2_store, regions): import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source - from genvarloader._dataset._svar2_store_py import build_readbound_tracks + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_tracks contig = "chr1" sv = genoray.SparseVar2(str(svar2_store)) @@ -159,8 +159,8 @@ def test_readbound_tracks_match_union_oracle_with_shifts(svar2_store): """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source - from genvarloader._dataset._svar2_store_py import build_readbound_tracks + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_tracks contig = "chr1" regions = [(0, 40), (5, 20)] diff --git a/tests/dataset/test_svar2_readbound_variants.py b/tests/dataset/test_svar2_readbound_variants.py index 54fa87b5..d5527ffd 100644 --- a/tests/dataset/test_svar2_readbound_variants.py +++ b/tests/dataset/test_svar2_readbound_variants.py @@ -106,7 +106,7 @@ def _assert_variants_match(oracle, rb) -> None: def test_readbound_variants_match_decode_oracle(svar2_store, regions): import genoray - from genvarloader._dataset._svar2_store_py import build_readbound_variants + from tests._oracles.svar2_readbound_inputs import build_readbound_variants contig = "chr1" @@ -177,7 +177,7 @@ def test_readbound_variants_dense_snp_match_decode_oracle(svar2_store_dense_snp) """ import genoray - from genvarloader._dataset._svar2_store_py import build_readbound_variants + from tests._oracles.svar2_readbound_inputs import build_readbound_variants contig = "chr1" diff --git a/tests/test_svar2_realign_tracks.py b/tests/test_svar2_realign_tracks.py index da7c1884..0d54f42c 100644 --- a/tests/test_svar2_realign_tracks.py +++ b/tests/test_svar2_realign_tracks.py @@ -63,7 +63,7 @@ def svar2_del_store(tmp_path_factory) -> Path: def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source + from tests._oracles.svar2_source import SparseVar2Source from genvarloader._dataset._tracks import shift_and_realign_track_sparse contig = "chr1" diff --git a/tests/test_svar2_reconstruct.py b/tests/test_svar2_reconstruct.py index 887c93fd..58b21165 100644 --- a/tests/test_svar2_reconstruct.py +++ b/tests/test_svar2_reconstruct.py @@ -92,7 +92,7 @@ def _consensus(ref: bytes, pos, ilen, alleles, q_start: int, q_end: int) -> byte def test_svar2_two_source_matches_decode_oracle(svar2_store): import genoray - from genvarloader._dataset._svar2_source import SparseVar2Source + from tests._oracles.svar2_source import SparseVar2Source contig = "chr1" q_start, q_end = 0, 40 From 3bd5a186c061ff593f75f845032e6965691abb59 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 13:03:22 -0700 Subject: [PATCH 100/108] refactor(svar2): drop dead annot_* capability from readbound haps kernel annot_v_idxs/annot_ref_pos were None at both call sites of the SVAR2 kernel reconstruct_haplotypes_from_svar2 (annotated .svar2 haps are NotImplementedError-guarded), so the annotation chunk-carve, 3 of 4 parallel match arms, and the serial raw-pointer annot views were all unreachable. Drop the two params, collapse the parallel dispatch to the un-annotated arm, and simplify the serial loop. The shared reconstruct_haplotype_core keeps its annot params (the SVAR1 path uses them). Byte-identical: svar2 parity 40 pass, reconstruct cargo tests 19 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/ffi/mod.rs | 4 -- src/reconstruct/mod.rs | 140 +++++------------------------------------ 2 files changed, 17 insertions(+), 127 deletions(-) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 85abbb5a..278dd6ca 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -907,8 +907,6 @@ pub fn reconstruct_haplotypes_from_svar2<'py>( ref_a, ref_offsets_a, pad_char, - None, // annot_v_idxs — not supported in fused plain path - None, // annot_ref_pos — not supported in fused plain path parallel, ); @@ -1081,8 +1079,6 @@ pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( ref_a, ref_offsets_a, pad_char, - None, - None, parallel, ); diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index 2906090a..919152e4 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -606,11 +606,12 @@ pub fn reconstruct_haplotypes_from_sparse( /// - `ref_` – packed reference bytes u8 /// - `ref_offsets` – per-contig offsets into ref_ i64 /// - `pad_char` – padding byte u8 -/// - `annot_v_idxs` – optional annotation output i32 (same layout as out); the value -/// written per applied variant is its sequential index within the merged per-hap list -/// (`0..merged.len()`), not a global variant id (SVAR2 has no global variant table) -/// - `annot_ref_pos` – optional annotation output i32 (same layout as out) /// - `parallel` – if true, use rayon to process work items concurrently +/// +/// SVAR2 read-bound haplotypes are never annotated (annotated `.svar2` output is +/// `NotImplementedError`-guarded), so this kernel produces only un-annotated +/// sequence; the shared `reconstruct_haplotype_core` still carries the optional +/// annotation outputs for the SVAR1 path. #[allow(clippy::too_many_arguments)] pub fn reconstruct_haplotypes_from_svar2( mut out: ArrayViewMut1, @@ -630,8 +631,6 @@ pub fn reconstruct_haplotypes_from_svar2( ref_: ArrayView1, ref_offsets: ArrayView1, pad_char: u8, - mut annot_v_idxs: Option>, - mut annot_ref_pos: Option>, parallel: bool, ) { let batch_size = regions.nrows(); @@ -651,10 +650,7 @@ pub fn reconstruct_haplotypes_from_svar2( // Per-k inner work: merge this hap's var_key ⋈ dense entries, then reconstruct via // the shared core with a decode closure. All read-only ArrayViews/slices are // Send+Sync so the closure can borrow them freely. - let do_work = |k: usize, - out_view: ArrayViewMut1, - av_view: Option>, - ap_view: Option>| { + let do_work = |k: usize, out_view: ArrayViewMut1| { let query = k / ploidy; let hap = k % ploidy; @@ -729,8 +725,8 @@ pub fn reconstruct_haplotypes_from_svar2( out_view, pad_char, None, // keep: SVAR2 has no per-haplotype keep mask - av_view, - ap_view, + None, // annot_v_idxs: SVAR2 read-bound haps are never annotated + None, // annot_ref_pos: SVAR2 read-bound haps are never annotated ); }; @@ -764,98 +760,18 @@ pub fn reconstruct_haplotypes_from_svar2( } } - // Carve annotation buffers only when they are Some. - let av_chunks: Option> = annot_v_idxs.as_mut().map(|av| { - let av_slice = av.as_slice_mut().unwrap(); - let mut chunks: Vec<&mut [i32]> = Vec::with_capacity(n_work); - let mut rest = &mut av_slice[..]; - let mut cursor = 0usize; - for &(s, e) in &bounds { - let (_, tail) = rest.split_at_mut(s - cursor); - let (mid, tail2) = tail.split_at_mut(e - s); - chunks.push(mid); - rest = tail2; - cursor = e; - } - chunks - }); - - let ap_chunks: Option> = annot_ref_pos.as_mut().map(|ap| { - let ap_slice = ap.as_slice_mut().unwrap(); - let mut chunks: Vec<&mut [i32]> = Vec::with_capacity(n_work); - let mut rest = &mut ap_slice[..]; - let mut cursor = 0usize; - for &(s, e) in &bounds { - let (_, tail) = rest.split_at_mut(s - cursor); - let (mid, tail2) = tail.split_at_mut(e - s); - chunks.push(mid); - rest = tail2; - cursor = e; - } - chunks - }); - - // Zip all chunk vecs and dispatch in parallel. - // Handle the four combinations of av/ap presence. - match (av_chunks, ap_chunks) { - (Some(avc), Some(apc)) => { - out_chunks - .into_par_iter() - .zip(avc.into_par_iter()) - .zip(apc.into_par_iter()) - .enumerate() - .for_each(|(k, ((out_chunk, av_chunk), ap_chunk))| { - do_work( - k, - ArrayViewMut1::from(out_chunk), - Some(ArrayViewMut1::from(av_chunk)), - Some(ArrayViewMut1::from(ap_chunk)), - ); - }); - } - (Some(avc), None) => { - out_chunks - .into_par_iter() - .zip(avc.into_par_iter()) - .enumerate() - .for_each(|(k, (out_chunk, av_chunk))| { - do_work( - k, - ArrayViewMut1::from(out_chunk), - Some(ArrayViewMut1::from(av_chunk)), - None, - ); - }); - } - (None, Some(apc)) => { - out_chunks - .into_par_iter() - .zip(apc.into_par_iter()) - .enumerate() - .for_each(|(k, (out_chunk, ap_chunk))| { - do_work( - k, - ArrayViewMut1::from(out_chunk), - None, - Some(ArrayViewMut1::from(ap_chunk)), - ); - }); - } - (None, None) => { - out_chunks - .into_par_iter() - .enumerate() - .for_each(|(k, out_chunk)| { - do_work(k, ArrayViewMut1::from(out_chunk), None, None); - }); - } - } + // SVAR2 read-bound haps are never annotated, so dispatch the un-annotated + // work items straight across rayon. + out_chunks + .into_par_iter() + .enumerate() + .for_each(|(k, out_chunk)| { + do_work(k, ArrayViewMut1::from(out_chunk)); + }); } else { // Serial path: use raw pointers for disjoint sub-range access, exactly as before. // The serial loop prevents concurrent aliasing. let out_raw: *mut u8 = out.as_mut_ptr(); - let av_raw: Option<*mut i32> = annot_v_idxs.as_mut().map(|a| a.as_mut_ptr()); - let ap_raw: Option<*mut i32> = annot_ref_pos.as_mut().map(|a| a.as_mut_ptr()); for k in 0..n_work { let out_s = out_offsets[k] as usize; @@ -872,25 +788,7 @@ pub fn reconstruct_haplotypes_from_svar2( // is free of aliasing UB. let out_chunk = unsafe { std::slice::from_raw_parts_mut(out_raw.add(out_s), out_e - out_s) }; - let out_view = ArrayViewMut1::from(out_chunk); - - // SAFETY: same invariant as out_chunk — `out_offsets` non-decreasing guarantees - // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent - // aliasing. - let av_view: Option> = av_raw.map(|p| { - let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; - ArrayViewMut1::from(chunk) - }); - - // SAFETY: same invariant as out_chunk — `out_offsets` non-decreasing guarantees - // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent - // aliasing. - let ap_view: Option> = ap_raw.map(|p| { - let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; - ArrayViewMut1::from(chunk) - }); - - do_work(k, out_view, av_view, ap_view); + do_work(k, ArrayViewMut1::from(out_chunk)); } } } @@ -1679,8 +1577,6 @@ mod tests { ref_.view(), ref_offsets.view(), pad_char, - None, - None, false, // serial ); @@ -1732,8 +1628,6 @@ mod tests { ref_.view(), ref_offsets.view(), pad_char, - None, - None, false, ); From 8f525da7ecb09738e83e6fe3cde2de747f5195ad Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 13:12:40 -0700 Subject: [PATCH 101/108] refactor(svar2): hoist shared present_bit to svar2::present_bit The identical LSB-first presence-bit closure in the reconstruct and tracks per-hap merge_hap calls is now a documented svar2::present_bit free function; both call sites reduce to a one-line closure over it. Byte-identical: svar2 parity 36 pass, cargo build clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/reconstruct/mod.rs | 5 +---- src/svar2/mod.rs | 10 ++++++++++ src/tracks/mod.rs | 5 +---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index 919152e4..26d4db37 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -672,10 +672,7 @@ pub fn reconstruct_haplotypes_from_svar2( // presence bits for this hap start at bit `dense_present_off[k]` let base_bit = dense_present_off[k] as usize; - let present_bit = |j: usize| -> bool { - let bit = base_bit + j; - (dense_present_s[bit / 8] >> (bit % 8)) & 1 == 1 // LSB-first within each byte - }; + let present_bit = |j: usize| crate::svar2::present_bit(dense_present_s, base_bit, j); let merged = crate::svar2::merge_hap( vk_pos_s, diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs index bf87a274..8fa46d01 100644 --- a/src/svar2/mod.rs +++ b/src/svar2/mod.rs @@ -26,6 +26,16 @@ pub fn decode_alt<'a>(key: u32, lut_bytes: &'a [u8], lut_off: &[i64]) -> (i64, C } } +/// LSB-first presence-bit lookup into a packed presence bitmap: returns whether bit +/// `base_bit + j` of `dense_present` is set, indexed from the least-significant bit +/// within each byte. Shared by the haplotype (`reconstruct`) and track (`tracks`) +/// per-hap `merge_hap` calls, which both read a hap's presence window this way. +#[inline] +pub fn present_bit(dense_present: &[u8], base_bit: usize, j: usize) -> bool { + let bit = base_bit + j; + (dense_present[bit / 8] >> (bit % 8)) & 1 == 1 +} + /// Merge one hap's `var_key` slice with its carried `dense` set-bits into a single /// position-sorted `(pos, key)` list (stable: var_key before dense on ties, matching /// genoray's merge). `dense` is region `r`'s `[ds, de)` window; `present` are this hap's diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index 68260c04..3220124d 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -756,10 +756,7 @@ pub fn shift_and_realign_tracks_from_svar2( // presence bits for this hap start at bit `dense_present_off[k]` let base_bit = dense_present_off[k] as usize; - let present_bit = |j: usize| -> bool { - let bit = base_bit + j; - (dense_present_s[bit / 8] >> (bit % 8)) & 1 == 1 // LSB-first within each byte - }; + let present_bit = |j: usize| crate::svar2::present_bit(dense_present_s, base_bit, j); let merged = crate::svar2::merge_hap( vk_pos_s, From 727d2ee4ca2717543323f278b1f16a2da4e8bcea Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 13:40:26 -0700 Subject: [PATCH 102/108] docs(svar2): sync roadmap/skill/prose docs + add release-gate checklist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roadmap Phase 6a: move unphased_union/variant-windows to supported, add the extend_to_length + field-routing guard/task lines, drop the stale variant-windows-parity footnote, correct the parity count to the current .svar2 suite (55), add a 2026-07-13 notes-log entry, and add a ⛔ release-gate subsection for the genoray path-pins. SKILL: var_fields on .svar2 = alt/ilen/start + store fields (no ref/dosage), min_af/max_af = .svar only, extend_to_length unsupported for .svar2, byte-identical caveat for pure-DEL ALT. Prose: index.md source list, format.md .svar2 guard matrix + changelog pin (0.37.0), write.md build snippet, dataset.md var_fields section. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/roadmaps/rust-migration.md | 59 +++++++++++++++++++++++++++------ docs/source/dataset.md | 29 ++++++++++++++++ docs/source/format.md | 24 +++++++++++++- docs/source/index.md | 2 +- docs/source/write.md | 12 ++++++- skills/genvarloader/SKILL.md | 20 ++++++----- 6 files changed, 124 insertions(+), 22 deletions(-) diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index 619e7e69..e7cde032 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -811,23 +811,31 @@ as a Rust path-dep, the first place gvl's Rust crate depends on genoray's Rust c (kept only as the parity oracle for tests). - [x] Guard matrix (Phase-1 scope; raise `NotImplementedError`, not silent mis-compute): spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, - `unphased_union`, `"variant-windows"`, fixed-length (`int output_length`) haplotype-realigned - tracks, `variants` output with jitter (write `max_jitter>0` or read `jitter>0` — the - readbound variants decode has no right-clip), and multi-contig `FlankSample` track fills - (contig-local vs. global fill-seed query index divergence). + fixed-length (`int output_length`) haplotype-realigned tracks, `variants`/`variant-windows` + output with jitter (write `max_jitter>0` or read `jitter>0` — the readbound variants decode + has no right-clip), `extend_to_length=False` at write time, and multi-contig `FlankSample` + track fills (contig-local vs. global fill-seed query index divergence). **Now supported** + (moved off the guard list this pass): `unphased_union` and `"variant-windows"` output for + both `"variants"` and `"variant-windows"`, plus `var_fields`-selected store INFO/FORMAT + fields — see the field-routing task line below. +- [x] var_fields → .svar2 store INFO/FORMAT field routing (plan + 2026-07-12-svar2-info-format-field-routing.md). - [x] Docs/skill audit (this task): `skills/genvarloader/SKILL.md`, `docs/source/{write,format,faq}.md`, `README.md`; `api.md` ↔ `__all__` gate confirmed clean (no new public symbol — `svar2=` is a parameter, not an exported name). -**Gate (parity — MET):** all four output modes (haplotypes, tracks, variants, variant-windows*) +**Gate (parity — MET):** all four output modes (haplotypes, tracks, variants, variant-windows) byte-identical to the `.svar`/union-oracle (`SparseVar2Source.reconstruct`/`realign_tracks`, genoray `decode`) across `tests/dataset/test_svar2_dataset.py`, `test_svar2_readbound_{haps,tracks,variants,diffs}.py`, -`test_write_svar2.py` — 31/31 passed. (*`"variant-windows"` is guarded `NotImplementedError` for -`.svar2` in Phase 1, so its parity claim covers `variants`, not the flat-window mode.) One documented, -intentional non-identity: for a pure deletion, `.svar2` decodes the atomized empty ALT (`b""`) where -`.svar` reports the VCF anchor base (`b"G"` for `GTA>G`) — a genoray format convention; reconstructed -haplotype bytes are unaffected (see `docs/source/format.md` "`.svar2` variants ALT convention"). -Full-tree regression: SVAR1 path byte-unchanged (additive-only change). +`test_write_svar2.py`, `test_svar2_fields_read.py` — 55/55 passed (`variant-windows` parity is +covered by `test_svar2_readbound_variants.py` and `test_svar2_fields_read.py`, not just `variants`). +One documented, intentional non-identity: for a pure deletion, `.svar2` decodes the atomized empty +ALT (`b""`) where `.svar` reports the VCF anchor base (`b"G"` for `GTA>G`) — a genoray format +convention; reconstructed haplotype bytes are unaffected (see `docs/source/format.md` "`.svar2` +variants ALT convention"). Also excluded from the write-path `max_ends` parity check: the +pre-existing SVAR1 same-POS-tie under-extension bug +(`docs/known-issues/svar1-max-ends-tie-underextension.md`) — a documented SVAR1-side defect, not a +`.svar2` regression. Full-tree regression: SVAR1 path byte-unchanged (additive-only change). **Checkpoint:** `.svar2` is a supported `gvl.write` source and `Dataset` read backend with a structurally read-bound query path (no per-read tree build, no per-read dense union), on-disk @@ -853,6 +861,18 @@ comparable across allocations on shared Carter nodes). smaller store) is what pays off at cohort scale and for the union path's contig-wide-stride concern; a fair large-workload latency sweep is a follow-up. +#### ⛔ Release gate (do NOT merge until genoray is released) + +This branch is dev-wired to a local genoray checkout and cannot build off this +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API lives on genoray main (unreleased). Flip ALL of these at +genoray release, then re-run the full py3xx matrix: + +- `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → published crates.io versions. +- `pixi.toml` [feature.py310.pypi-dependencies]: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"`. +- Verify the version-floor bumps already made are intended: numpy 0.29, pyo3 0.29, seqpro 0.21.1. + ### Phase 6 — Absorb genoray (future) ⬜ _PR: —_ @@ -871,6 +891,23 @@ conversion/write paths. ## Notes & decisions log +- 2026-07-13 (Phase 6a — final pre-merge hardening pass; branch `svar2-m6b-kernel`): + Shipped scope grew past the 2026-07-05 entry below: `unphased_union` (both `"variants"` and + `"variant-windows"`), `"variant-windows"` output itself, and `var_fields`-selected `.svar2` + store INFO/FORMAT field routing (`rv["AF"]` / `win.fields["AF"]`, plan + `2026-07-12-svar2-info-format-field-routing.md`) are all done and parity-tested — none of + the three remain on the guard list. This pass also landed a round of correctness/perf + hardening with no behavior change to already-shipped modes: a serial-unsafe carve-path + guard (`debug_assert` monotonicity check), Python-reachable Rust panics converted to + `PyValueError` (non-contiguous/OOB input), a new `extend_to_length=False` guard for `.svar2` + write sources (raises `NotImplementedError` — was previously silently accepted), a + vectorized write-time `max_ends` computation (byte-identical), relocation of the + `SparseVar2Source` union-based oracle out of the shipped package into `tests/`, removal of + dead `annot_*` capability from the readbound haps kernel, a hoisted shared `present_bit` + helper, and a typecheck-task path fix. Full `.svar2` suite (`test_svar2_dataset.py`, + `test_svar2_readbound_{haps,tracks,variants,diffs}.py`, `test_write_svar2.py`, + `test_svar2_fields_read.py`) — 55/55 passed. + - 2026-07-05 (Phase 6a — SVAR2 read-bound dataset wiring; branch `svar2-m6b-kernel`): `.svar2` (genoray's newer sparse variant format) is now a `gvl.write` variant source and a live `Dataset` read backend, wired end-to-end: write-time 6-array ranges cache diff --git a/docs/source/dataset.md b/docs/source/dataset.md index f1792e6c..880436be 100644 --- a/docs/source/dataset.md +++ b/docs/source/dataset.md @@ -141,3 +141,32 @@ ds_itvs = ( ``` In `"flat"` output mode (`with_output_format("flat")`), float tracks return `FlatRagged` and interval tracks (`kind="intervals"`) return [`FlatIntervals`](api.md#genvarloader.FlatIntervals), which carries `.starts`, `.ends`, `.values` as `FlatRagged` fields and converts back via `.to_ragged()` → [`RaggedIntervals`](api.md#genvarloader.RaggedIntervals). + +## Variant fields (`var_fields`) + +`Dataset.open(..., var_fields=[...])` (and `Dataset.with_settings(var_fields=[...])`) selects which +per-variant fields load onto `"variants"` and `"variant-windows"` output, beyond the default +`["alt", "ilen", "start"]`. Requested names must be a subset of `Dataset.available_var_fields`. + +For a BCF/PGEN/`.svar`-backed dataset the available fields are the built-ins (`alt`, `start`, +`ref`, `ilen`, `dosage`) plus any per-variant INFO columns or per-call FORMAT fields the source +carries. + +For a **`.svar2`-backed** dataset, `available_var_fields` is narrower: +`["alt", "ilen", "start"]` plus whichever scalar-numeric INFO/FORMAT fields the `.svar2` store was +written with (via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])`) — +**`"ref"` and `"dosage"` are not valid `var_fields` for `.svar2` and requesting either raises**. +A requested store field shows up on both output kinds: + +```python +ds = gvl.Dataset.open("ds.gvl", reference="ref.fa", var_fields=["AF"]) + +rv = ds.with_seqs("variants")[0, 0] +rv["AF"] # per-variant AF values, aligned with rv.alt/.start/.ilen + +win = ds.with_seqs("variant-windows", gvl.VarWindowOpt(...)).with_output_format("flat")[0, 0] +win.fields["AF"] # same field, alongside win.fields["start"]/["ilen"] +``` + +See the `genvarloader` skill's `.svar2` `var_fields` section for the field-provenance and +dummy-fill details. diff --git a/docs/source/format.md b/docs/source/format.md index 19836ace..9be749a5 100644 --- a/docs/source/format.md +++ b/docs/source/format.md @@ -135,6 +135,28 @@ backends; only `RaggedVariants.alt` differs, and only for pure-deletion records. for `with_seqs("variant-windows")`: `ref_window` is byte-identical between the backends, while the `alt`/`alt_window` fields differ only for pure-deletion records (the same empty-vs-anchor ALT). +## `.svar2` Phase-1 unsupported combinations + +A `.svar2`-backed dataset supports all four output modes (`haplotypes`, `variants`, +`variant-windows`, and haplotype-realigned `tracks`), `unphased_union`, and +`var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`). +The following combinations are Phase-1 scope and raise `NotImplementedError` (or, for +`extend_to_length`, at write time) instead of silently mis-computing: + +- Spliced output. +- The `var_filter="exonic"` (keep-mask) variant filter. +- `min_af` / `max_af` filtering (`.svar` only; see "Should I use `.svar` or `.svar2`" in the FAQ). +- `annotated` haplotypes (`with_seqs("annotated")`). +- `VarWindowOpt(ref="allele")` (bare-allele REF mode; REF alleles aren't stored in `.svar2`). +- In-kernel reverse-complement (`to_rc`). +- Fixed-length (integer `output_length`) haplotype-realigned **track** output. +- `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with + `jitter>0` (the read-bound decode does not right-clip to the post-jitter window). +- `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source. +- `FlankSample` insertion-fill for tracks spanning multiple contigs in one query. + +See the `genvarloader` skill's `.svar2` section for the full narrative and `var_fields` semantics. + ## Format changelog | Version | Change | @@ -142,7 +164,7 @@ for `with_seqs("variant-windows")`: `ref_window` is byte-identical between the b | `< 0.18.0` | Variant coordinates stored 0-based. | | `0.18.0` | Variant coordinates switched to 1-based. | | `0.25.0` | `metadata.json` gains `svar_link`; old `genotypes/link.svar` symlink layout deprecated. `Metadata.version` typed as `SemanticVersion` (on-disk JSON unchanged). | -| (unreleased) | `metadata.json` gains `svar2_link`; `.svar2` accepted as a `gvl.write` variant source, cached under `genotypes/svar2_ranges/` and read via a read-bound, all-Rust path. | +| `0.37.0` | `metadata.json` gains `svar2_link`; `.svar2` accepted as a `gvl.write` variant source, cached under `genotypes/svar2_ranges/` and read via a read-bound, all-Rust path. | > **Upgrading legacy datasets.** A dataset written before `0.25.0` that was built from an > `.svar` will still open (with a `DeprecationWarning`). Run diff --git a/docs/source/index.md b/docs/source/index.md index b0d1091f..8d5beecb 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -48,7 +48,7 @@ GenVarLoader provides a fast, memory efficient data structure for training seque - Generate haplotypes up to 1,000 times faster than reading a FASTA file - Generate tracks up to 450 times faster than reading a BigWig - **Supports indels** and re-aligns tracks to haplotypes that have them -- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, and BigWig +- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, BigWig, and [genoray](https://github.com/mcvickerlab/genoray)'s sparse `.svar`/`.svar2` variant stores See our [preprint](https://www.biorxiv.org/content/10.1101/2025.01.15.633240) for benchmarking and implementation details. diff --git a/docs/source/write.md b/docs/source/write.md index 9ad58f87..b977acea 100644 --- a/docs/source/write.md +++ b/docs/source/write.md @@ -83,7 +83,17 @@ This dataset would have both haplotypes and two tracks (`pos` and `neg`) availab ## Variants from a genoray sparse store (`.svar` / `.svar2`) -Besides BCF/VCF and PGEN, `variants=` also accepts a genoray sparse columnar variant store — either the original `.svar` format or the newer `.svar2` format: +Besides BCF/VCF and PGEN, `variants=` also accepts a genoray sparse columnar variant store — either the original `.svar` format or the newer `.svar2` format. Build one from a normalized VCF/BCF with `genoray`: + +```python +from genoray import VCF, SparseVar2 +from genoray._svar import dense2sparse + +dense2sparse(VCF("normed.bcf"), "all_chroms.svar") # writes a .svar/ directory +SparseVar2.from_vcf("all_chroms.svar2", "normed.bcf") # writes a .svar2/ directory +``` + +Then pass the resulting store to `gvl.write`: ```python gvl.write( diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index b4e9851b..d8f52ba6 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -63,7 +63,7 @@ See `docs/source/write.md` for the canonical recipe and BED/BigWig table layouts `.svar` is a sparse columnar variant archive (from `genoray`). Pass it to `gvl.write(variants="x.svar")` exactly like a BCF or PGEN — the resulting dataset stores a back-reference instead of duplicating per-variant arrays. Use SVAR when: -- You need **allele-frequency filtering at read time** (`Dataset.open(min_af=..., max_af=...)` requires SVAR-backed genotypes — will raise otherwise). +- You need **allele-frequency filtering at read time** (`Dataset.open(min_af=..., max_af=...)` is supported for `.svar` only — a `.svar2`-backed dataset raises `NotImplementedError`). - Many datasets share the same variant source — SVAR avoids duplicating `variant_idxs.npy`/`dosages.npy`/`variants.arrow` into each `.gvl` directory. - You're working at population scale and want compact on-disk variant storage. @@ -88,7 +88,7 @@ Unlike `.svar` (whose read path builds an interval search tree + a per-read dens `.svar2` is resolved at `Dataset.open` time in the same order as `.svar`: caller `svar2=` arg → recorded relative path → recorded absolute path → sibling `*.svar2`. `Dataset.open(path, svar2=)` mirrors `svar=`. See `docs/source/format.md` ("`.svar2` resolution at open time"). -**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend, and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`), `unphased_union` (for both `"variants"` and `"variant-windows"` output), and `var_fields`-selected store INFO/FORMAT fields (also for both `"variants"` and `"variant-windows"`; see `var_fields` under `Dataset.open` below) are all fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: +**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend (except pure-deletion ALT bytes — see below), and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`), `unphased_union` (for both `"variants"` and `"variant-windows"` output), and `var_fields`-selected store INFO/FORMAT fields (also for both `"variants"` and `"variant-windows"`; see `var_fields` under `Dataset.open` below) are all fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: - Spliced output. - The `var_filter="exonic"` (keep-mask) variant filter. - `min_af` / `max_af` filtering. @@ -98,6 +98,7 @@ Unlike `.svar` (whose read path builds an interval search tree + a per-read dens - Fixed-length (integer `output_length`) haplotype-realigned **track** output (plain haplotype output at a fixed length is fine — only the track kernel is guarded). - `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). - `FlankSample` insertion-fill for tracks spanning **multiple contigs** in one query (single-contig queries and non-seeded fills like the default `Repeat5p` are exact). +- `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source (write-time; raises `NotImplementedError` — `.svar2` sources must use the default `extend_to_length=True`). **`variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. The same convention carries into `with_seqs("variant-windows")`: `ref_window` is byte-identical between `.svar`/`.svar2`, but `alt`/`alt_window` differ for pure-deletion records for the same reason. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw allele/window bytes differ for pure-deletion records. See `docs/source/faq.md`. @@ -125,7 +126,7 @@ Notable: - `tracks`: a `gvl.BigWigs` (or a list of them), or a `gvl.Table`. Each must have a unique `.name`. BigWigs need a sample→path mapping (dict or table with `sample`, `path` columns; see `BigWigs.from_table`). `gvl.Table` is a core interval-track source backed by a Rust COITrees overlap engine (zero-based half-open coordinates); pass it directly as a `tracks=` or `annot_tracks=` source in `gvl.write`. - `annot_tracks`: `dict[str, str | Path | pl.DataFrame | pl.LazyFrame] | None` — sample-independent annotation tracks, written to `/annot_intervals//`. Each value is either a path to an interval table/bigWig file, or a polars DataFrame/LazyFrame with BED-like columns (`chrom`, `chromStart`, `chromEnd`, `score`). Annotation tracks are sample-independent and can be read without a per-sample variant source. - `max_jitter`: max read-time jitter; pads stored data on both sides of every region by this many bases so `Dataset.with_settings(jitter=j)` works for any `j <= max_jitter`. -- `extend_to_length=True` keeps reading past the BED end until every haplotype is ≥ the region length (matters when deletions would shorten output); set `False` for faster writes if shorter haps are acceptable. +- `extend_to_length=True` keeps reading past the BED end until every haplotype is ≥ the region length (matters when deletions would shorten output); set `False` for faster writes if shorter haps are acceptable. **Not supported for a `.svar2` variant source** — `extend_to_length=False` raises `NotImplementedError` there; only BCF/PGEN/`.svar` sources may disable it. - Inner-joins samples across `variants` and all `tracks`. **Parallelism:** `gvl.write` now parallelizes over write categories. Variants are processed first (serially). Then per-sample `tracks` and `annot_tracks` run concurrently (joblib loky backend). The `max_mem` budget is divided across the concurrently-running categories. @@ -167,7 +168,7 @@ Source: `python/genvarloader/_dataset/_write.py`. gvl.Dataset.open( path, reference=None, jitter=0, rng=None, deterministic=True, rc_neg=True, - min_af=None, max_af=None, # SVAR only + min_af=None, max_af=None, # .svar only — raises NotImplementedError on .svar2 region_names=None, splice_info=None, # see "Spliced haplotypes" var_filter=None, # None | "exonic" @@ -192,7 +193,10 @@ Scalar fields (`start`/`ilen`/`dosage`/`info[...]`) are still filled from `Dummy - **`var_fields: list[str] | None`** — Variant fields to include on `RaggedVariants` output. Defaults to the minimum useful set `["alt", "ilen", "start"]`. Pass additional names (e.g. `"ref"`, `"dosage"`, or any numeric info column in the source variants table) to load them eagerly at open time. Must be a subset of `Dataset.available_var_fields`. Can be reconfigured later via `Dataset.with_settings(var_fields=...)`, which lazily loads any newly-requested columns. `"dosage"` must be requested explicitly — it is *not* added automatically even when `dosages.npy` exists on disk. Beyond the built-ins (`alt`, `start`, `ref`, `ilen`, `dosage`) and per-variant INFO columns, a genoray `.svar` may register arbitrary per-call (`Number=G`) FORMAT fields in `/metadata.json["fields"]`; these appear in `Dataset.available_var_fields` and can be requested via `Dataset.open(..., var_fields=[...])` or `with_settings(var_fields=[...])`. Each surfaces in `variants`, `variant-windows`, and `flat` outputs as a per-call ragged field aligned with the genotypes. A FORMAT field shadows a same-named INFO column. - **On `.svar2`**, `var_fields` additionally exposes the store's own scalar-numeric INFO/FORMAT fields — whichever ones the `.svar2` was written with via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])` (bare `str` names also work there). Only scalar-numeric fields can exist in a `.svar2` store at all — INFO/FORMAT `Type=Integer`/`Float` with `Number=1` or `Number=A`, plus INFO `Type=Flag` (stored as bool); anything else is rejected by genoray at write time and never reaches gvl. `gvl` only *reads* whatever the store already carries — it cannot add fields, and re-requesting a field the store doesn't have raises (it isn't in `available_var_fields`). `Dataset.available_var_fields` advertises each store field's key, sourced from `genoray.SparseVar2.available_fields`: the bare field name when it's unique across the store's INFO/FORMAT namespace, else `"INFO/"` / `"FORMAT/"`. A builtin name (`alt`/`start`/`ref`/`ilen`/`dosage`) always wins — a store field that happens to be named e.g. `alt` is never advertised and cannot shadow the builtin. Values keep the store's dtype exactly, with no widening (an `i32` field decodes `int32`, an `f32` field `float32`), and a VCF-missing entry carries the store's stored default verbatim (`NaN` for a float field declared with no default). Both entry points route to the same svar2 reconstructor: `gvl.Dataset.open(path, reference=..., var_fields=[...])` and `ds.with_seqs("variants")`/`.with_settings(var_fields=[...])`. Supported on both output modes: `"variants"` (the field appears on the returned `RaggedVariants`, e.g. `rv["AF"]`, sharing `alt`/`start`/`ilen`'s variant offsets) and `"variant-windows"` (the field appears in `win.fields["AF"]` alongside `start`/`ilen`). A FORMAT field's value is the value for the sample that row belongs to (not sample 0). Empty `(region, sample, ploid)` groups fill each store field via the same `DummyVariant.info[]` mechanism as any other scalar field (see `with_settings(dummy_variant=...)` above): the user-supplied value if given, else `NaN` for a float column or `0` for an integer column. + **On `.svar2`**, `available_var_fields` is `["alt", "ilen", "start"]` plus whatever store fields the + `.svar2` was written with — **`"ref"` and `"dosage"` are not available fields for a `.svar2` source and + requesting either raises** (they are only valid for BCF/PGEN/`.svar`). Beyond that, `var_fields` + additionally exposes the store's own scalar-numeric INFO/FORMAT fields — whichever ones the `.svar2` was written with via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])` (bare `str` names also work there). Only scalar-numeric fields can exist in a `.svar2` store at all — INFO/FORMAT `Type=Integer`/`Float` with `Number=1` or `Number=A`, plus INFO `Type=Flag` (stored as bool); anything else is rejected by genoray at write time and never reaches gvl. `gvl` only *reads* whatever the store already carries — it cannot add fields, and re-requesting a field the store doesn't have raises (it isn't in `available_var_fields`). `Dataset.available_var_fields` advertises each store field's key, sourced from `genoray.SparseVar2.available_fields`: the bare field name when it's unique across the store's INFO/FORMAT namespace, else `"INFO/"` / `"FORMAT/"`. A builtin name (`alt`/`start`/`ref`/`ilen`/`dosage`) always wins — a store field that happens to be named e.g. `alt` is never advertised and cannot shadow the builtin. Values keep the store's dtype exactly, with no widening (an `i32` field decodes `int32`, an `f32` field `float32`), and a VCF-missing entry carries the store's stored default verbatim (`NaN` for a float field declared with no default). Both entry points route to the same svar2 reconstructor: `gvl.Dataset.open(path, reference=..., var_fields=[...])` and `ds.with_seqs("variants")`/`.with_settings(var_fields=[...])`. Supported on both output modes: `"variants"` (the field appears on the returned `RaggedVariants`, e.g. `rv["AF"]`, sharing `alt`/`start`/`ilen`'s variant offsets) and `"variant-windows"` (the field appears in `win.fields["AF"]` alongside `start`/`ilen`). A FORMAT field's value is the value for the sample that row belongs to (not sample 0). Empty `(region, sample, ploid)` groups fill each store field via the same `DummyVariant.info[]` mechanism as any other scalar field (see `with_settings(dummy_variant=...)` above): the user-supplied value if given, else `NaN` for a float column or `0` for an integer column. ## Output modes — `with_seqs` × `with_tracks` @@ -430,16 +434,16 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). - **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. -- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, and `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), `extend_to_length=False` at write time, and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, and `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. - **`.svar2` `variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way; `ref_window` is also byte-identical — only raw ALT/`alt_window` bytes differ for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. -- `min_af` / `max_af` raise unless the dataset is SVAR-backed. +- `min_af` / `max_af` are supported for `.svar` only. A non-SVAR-backed dataset (plain BCF/PGEN) raises `RuntimeError`; a `.svar2`-backed dataset raises `NotImplementedError` specifically. - `with_len(L)` requires `L + 2·jitter ≤ min(region_length) + 2·max_jitter` — set `max_jitter` accordingly at `write` time. - Tracks must have unique `.name`; the on-disk layout is `intervals//`. - BED `strand` of `.` is treated as `+`. Reverse-complement happens automatically when `rc_neg=True` (default) and `strand == "-"`. - Splicing is a read-time setting on a *flat* BED of exons — do not pre-concatenate exons before `gvl.write`. -- `extend_to_length=False` at write time will produce haplotypes shorter than the BED region when deletions are present; downstream code must tolerate `<` region length. +- `extend_to_length=False` at write time will produce haplotypes shorter than the BED region when deletions are present; downstream code must tolerate `<` region length. Not an option for a `.svar2` variant source — `gvl.write(..., variants=<.svar2>, extend_to_length=False)` raises `NotImplementedError`. - Missing a `dosage` field on a `RaggedVariants` output you expected? Check `var_fields` — `dosage` must be requested explicitly even if `dosages.npy` exists on disk. - `FlatRagged` / `FlatVariants` offsets are **int64**. PyTorch nested tensors require int32 offsets — cast with `.astype(np.int32)` or `tensor.to(torch.int32)` before passing to `torch.nested.narrow`. - `kind="intervals"` cannot be re-aligned: combining it with a variant-aware seq mode (`haplotypes`/`annotated`/`variants`/`variant-windows`) raises unless `with_settings(realign_tracks=False)`. (Breaking change: `haplotypes`+`intervals` previously returned un-realigned intervals silently under the default.) From 18b624c17ff1581f7aecb048a9d30df21e608935 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 13:49:14 -0700 Subject: [PATCH 103/108] docs(svar2): fix genoray build snippets to the real from_vcf API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write.md build snippet (and a pre-existing SKILL.md twin) used dense2sparse(VCF(...), path) — but dense2sparse takes dense numpy arrays and returns a Ragged, and SparseVar2.from_vcf requires reference= or no_reference=. Use the verified public API: SparseVar.from_vcf(out, VCF(src), max_mem=...) for .svar and SparseVar2.from_vcf(out, src, reference=...) for .svar2. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/write.md | 10 ++++++---- skills/genvarloader/SKILL.md | 5 ++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/write.md b/docs/source/write.md index b977acea..e47d8ee7 100644 --- a/docs/source/write.md +++ b/docs/source/write.md @@ -86,11 +86,13 @@ This dataset would have both haplotypes and two tracks (`pos` and `neg`) availab Besides BCF/VCF and PGEN, `variants=` also accepts a genoray sparse columnar variant store — either the original `.svar` format or the newer `.svar2` format. Build one from a normalized VCF/BCF with `genoray`: ```python -from genoray import VCF, SparseVar2 -from genoray._svar import dense2sparse +from genoray import VCF, SparseVar, SparseVar2 -dense2sparse(VCF("normed.bcf"), "all_chroms.svar") # writes a .svar/ directory -SparseVar2.from_vcf("all_chroms.svar2", "normed.bcf") # writes a .svar2/ directory +# .svar (SVAR1): a VCF reader + a memory budget +SparseVar.from_vcf("all_chroms.svar", VCF("normed.bcf"), max_mem="4g") + +# .svar2 (SVAR2): the VCF/BCF path + a reference FASTA (or no_reference=True) +SparseVar2.from_vcf("all_chroms.svar2", "normed.bcf", reference="ref.fa") ``` Then pass the resulting store to `gvl.write`: diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index d8f52ba6..23d29ac9 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -72,10 +72,9 @@ Use BCF/PGEN directly when you have a one-off dataset and don't need AF filterin Create an SVAR from a normalized VCF/PGEN with `genoray`: ```python -from genoray._svar import dense2sparse -from genoray import VCF +from genoray import VCF, SparseVar -dense2sparse(VCF("normed.bcf"), "normed.svar") # writes a .svar/ directory +SparseVar.from_vcf("normed.svar", VCF("normed.bcf"), max_mem="4g") # writes a .svar/ directory ``` SVARs are resolved at `Dataset.open` time via `metadata.json` → caller `svar=` arg → recorded relative path → recorded absolute path → sibling `*.svar`. See `docs/source/format.md` ("SVAR resolution at open time") and `_dataset/_svar_link.py`. Legacy symlink-based SVAR layouts: run `gvl.migrate_svar_link(path)` once to upgrade. From 72d4a0ec69cb0aa517947b966c90e023bfa11181 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 14:02:16 -0700 Subject: [PATCH 104/108] chore(svar2): untrack tmp/svar2_mvp scratch and blanket-ignore tmp/ The tmp/svar2_mvp benchmark/profiling drivers are machine-specific (hardcoded /carter paths + real chr21 stores absent in CI) and their perf conclusions already live in docs/roadmaps/rust-migration.md, so they are dropped from git rather than relocated into tests/ (adding un-runnable scratch there would degrade the test tree). Fix the self-contradicting .gitignore (tmp/svar2_mvp/prof_out/) with a blanket tmp/ ignore. The drivers remain on disk (ignored) for local reuse; relocating+parameterizing specific ones into tests/benchmarks/ is a follow-up if wanted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 2 +- tmp/svar2_mvp/benchmark.py | 101 ---- tmp/svar2_mvp/build_stores.py | 36 -- tmp/svar2_mvp/e1_bucket_dso.py | 44 -- tmp/svar2_mvp/e1_profile.sh | 53 -- tmp/svar2_mvp/e2_build.sbatch | 16 - tmp/svar2_mvp/e2_subsample.sbatch | 24 - tmp/svar2_mvp/env_baseline.txt | 17 - tmp/svar2_mvp/genoray_debug_build.sbatch | 13 - tmp/svar2_mvp/prof_driver.py | 77 --- tmp/svar2_mvp/prof_getitem.py | 73 --- tmp/svar2_mvp/prof_out/readbound/RESULTS.md | 214 ------- .../prof_out/readbound/asm_targets.md | 98 --- .../prof_out/readbound/native_after_b1b3.md | 571 ------------------ .../prof_out/readbound/native_baseline.md | 485 --------------- .../prof_out/readbound/python_baseline.md | 461 -------------- tmp/svar2_mvp/prof_perf.sh | 41 -- tmp/svar2_mvp/prof_python.py | 43 -- tmp/svar2_mvp/split_folded.py | 49 -- tmp/svar2_mvp/validate.py | 67 -- 20 files changed, 1 insertion(+), 2484 deletions(-) delete mode 100644 tmp/svar2_mvp/benchmark.py delete mode 100644 tmp/svar2_mvp/build_stores.py delete mode 100644 tmp/svar2_mvp/e1_bucket_dso.py delete mode 100755 tmp/svar2_mvp/e1_profile.sh delete mode 100644 tmp/svar2_mvp/e2_build.sbatch delete mode 100644 tmp/svar2_mvp/e2_subsample.sbatch delete mode 100644 tmp/svar2_mvp/env_baseline.txt delete mode 100644 tmp/svar2_mvp/genoray_debug_build.sbatch delete mode 100644 tmp/svar2_mvp/prof_driver.py delete mode 100644 tmp/svar2_mvp/prof_getitem.py delete mode 100644 tmp/svar2_mvp/prof_out/readbound/RESULTS.md delete mode 100644 tmp/svar2_mvp/prof_out/readbound/asm_targets.md delete mode 100644 tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md delete mode 100644 tmp/svar2_mvp/prof_out/readbound/native_baseline.md delete mode 100644 tmp/svar2_mvp/prof_out/readbound/python_baseline.md delete mode 100644 tmp/svar2_mvp/prof_perf.sh delete mode 100644 tmp/svar2_mvp/prof_python.py delete mode 100644 tmp/svar2_mvp/split_folded.py delete mode 100644 tmp/svar2_mvp/validate.py diff --git a/.gitignore b/.gitignore index a6ea3ea9..10ebd67f 100644 --- a/.gitignore +++ b/.gitignore @@ -184,4 +184,4 @@ tests/benchmarks/profiling/*.speedscope.json tests/benchmarks/profiling/*.memray.bin tests/benchmarks/profiling/*.flamegraph.html tests/benchmarks/profiling/*.perf.data -tmp/svar2_mvp/prof_out/ +tmp/ diff --git a/tmp/svar2_mvp/benchmark.py b/tmp/svar2_mvp/benchmark.py deleted file mode 100644 index 31f409b1..00000000 --- a/tmp/svar2_mvp/benchmark.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Benchmark SVAR1 (gvl Dataset over .svar) vs SVAR2 (SparseVar2Source over -.svar2): hap latency, variant latency, store size, for one source prefix. -Fair workload: ALL samples for a fixed region set. Warm caches, median of N.""" - -import sys -import time -import subprocess -from statistics import median - -import numpy as np -import genvarloader as gvl -from genoray import SparseVar2 -from genvarloader._dataset._svar2_source import SparseVar2Source - -REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" -N = 5 # repeats - - -def _contig_ref(fasta, chrom): - import pysam - - return pysam.FastaFile(fasta).fetch(chrom).encode() - - -def _timed(fn, warmup=1): - for _ in range(warmup): - fn() - ts = [] - for _ in range(N): - t0 = time.perf_counter() - fn() - ts.append(time.perf_counter() - t0) - return median(ts) - - -def main(prefix, chrom): - regions = [ - (20_000_000, 20_001_000), - (30_000_000, 30_000_500), - (40_000_000, 40_001_000), - ] - ref_bytes = _contig_ref(REF, chrom) - ref_u8 = np.frombuffer(ref_bytes, np.uint8) - ref_off = np.array([0, len(ref_bytes)], np.int64) - - # SVAR2 backend - sv2 = SparseVar2(f"{prefix}.svar2") - src = SparseVar2Source(sv2) - svar2_hap = _timed( - lambda: src.reconstruct( - chrom, - regions, - ref_u8, - ref_off, - pad_char=ord("N"), - shifts=None, - output_length=-1, - ) - ) - svar2_var = _timed(lambda: sv2.decode(chrom, regions)) - - # SVAR1 backend (all samples, same regions) - import polars as pl - - bed = pl.DataFrame( - { - "chrom": [chrom] * len(regions), - "chromStart": [s for s, _ in regions], - "chromEnd": [e for _, e in regions], - } - ) - ds_path = f"{prefix}.gvl" - # Write the Dataset over the SAME region set the SVAR2 path benchmarks, so both - # backends measure an identical workload (fairness rule). validate.py may have left - # a .gvl with a different region set, so always (re)write here. - gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) - ds = gvl.Dataset.open(ds_path, reference=REF) - ds_hap = ds.with_seqs("haplotypes") - ds_var = ds.with_seqs("variants") - n_s = sv2.n_samples - svar1_hap = _timed(lambda: ds_hap[: len(regions), :n_s]) - svar1_var = _timed(lambda: ds_var[: len(regions), :n_s]) - - def du(path): - return subprocess.run( - ["du", "-sb", path], capture_output=True, text=True - ).stdout.split()[0] - - print( - f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " - f"regions={len(regions)} N={N}" - ) - print(f" hap_latency_s svar1={svar1_hap:.4f} svar2={svar2_hap:.4f}") - print(f" var_latency_s svar1={svar1_var:.4f} svar2={svar2_var:.4f}") - print( - f" store_bytes svar1={du(prefix + '.svar')} svar2={du(prefix + '.svar2')}" - ) - - -if __name__ == "__main__": - main(sys.argv[1], sys.argv[2]) # argv: diff --git a/tmp/svar2_mvp/build_stores.py b/tmp/svar2_mvp/build_stores.py deleted file mode 100644 index 3e8efb8a..00000000 --- a/tmp/svar2_mvp/build_stores.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Build .svar (SVAR1) and .svar2 (SVAR2) stores from a normalized biallelic BCF.""" - -import sys -from pathlib import Path - -from genoray import VCF, SparseVar, _core - - -def build(bcf: str, chrom: str, samples: list[str], out_prefix: str, ploidy: int): - bcf = str(bcf) - # SVAR 1.0 - SparseVar.from_vcf(f"{out_prefix}.svar", VCF(bcf), "8g", overwrite=True) - # SVAR 2.0 - _core.run_conversion_pipeline( - bcf, - "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", - [chrom], - f"{out_prefix}.svar2", - samples, - 25_000, - ploidy, - 8, - 8 * 1024 * 1024, - ) - print(f"built {out_prefix}.svar and {out_prefix}.svar2") - - -if __name__ == "__main__": - # argv: - bcf, chrom, out_prefix = sys.argv[1], sys.argv[2], sys.argv[3] - import subprocess - - samples = subprocess.run( - ["bcftools", "query", "-l", bcf], capture_output=True, text=True, check=True - ).stdout.split() - build(bcf, chrom, samples, out_prefix, ploidy=2) diff --git a/tmp/svar2_mvp/e1_bucket_dso.py b/tmp/svar2_mvp/e1_bucket_dso.py deleted file mode 100644 index ada5d1c6..00000000 --- a/tmp/svar2_mvp/e1_bucket_dso.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Bucket a `perf report --sort=dso --no-children --stdio` self-time dump into the -E1 attribution classes. Reads that dump on stdin. - - report --stdio --sort=dso --no-children -i data.perf | python e1_bucket_dso.py -""" - -import re -import sys - -buckets = { - "native-gvl": 0.0, - "native-genoray": 0.0, - "numpy-conv": 0.0, - "python-interp": 0.0, - "other": 0.0, -} -for line in sys.stdin: - if line.lstrip().startswith("#") or not line.strip(): - continue - m = re.match(r"\s*([0-9.]+)%\s+(.*)", line) - if not m: - continue - pct = float(m.group(1)) - dso = m.group(2).strip().lower() - if "genvarloader" in dso: - b = "native-gvl" - elif "genoray" in dso or "_core.cpython" in dso: - b = "native-genoray" - elif "multiarray" in dso or "umath" in dso or "/numpy" in dso: - b = "numpy-conv" - elif dso.startswith("python") or "libpython" in dso: - b = "python-interp" - else: - b = "other" - buckets[b] += pct - -tot = sum(buckets.values()) or 1.0 -for k in ("native-gvl", "native-genoray", "numpy-conv", "python-interp", "other"): - print(f" {buckets[k]:6.1f}% {k}") -nat = buckets["native-gvl"] + buckets["native-genoray"] -print( - f" ---- native(gvl+genoray)={nat:.1f}% numpy-conv={buckets['numpy-conv']:.1f}% " - f"python-interp={buckets['python-interp']:.1f}% (sum {tot:.1f}%)" -) diff --git a/tmp/svar2_mvp/e1_profile.sh b/tmp/svar2_mvp/e1_profile.sh deleted file mode 100755 index 3478b6a4..00000000 --- a/tmp/svar2_mvp/e1_profile.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# E1: per-(backend x cohort) query-latency attribution via perf ONLY. -# py-spy is unusable here (ptrace_scope=2, no sudo). perf works (paranoid=2, uses perf_event). -# -# Profile the env's python DIRECTLY (.pixi/envs/default/bin/python) — NOT via `pixi run`: -# the pixi launcher otherwise eats ~60% of samples, and the extensions import fine standalone -# (RPATH handles deps). Large K so the steady-state reconstruct loop drowns import/startup -# (genvarloader pulls in torch — a heavy one-time import). -# -# Python frames are opaque on 3.10, so the split is at the DSO level (self-time): -# gvl genvarloader.abi3.so + genoray _core.so = native Rust hot path -# numpy _multiarray_umath*.so = conversion/ascontiguousarray overhead -# python3.10 / libpython = interpreter/orchestration -# Frame-pointer call graph (built with -C force-frame-pointers=yes); DWARF overloads the node. -# Runs DIRECTLY on the current 2-cpu carter-cn-04 allocation (no srun). -# NOTE: no `pipefail` — the `| head` truncations send SIGPIPE (141) upstream to perf report, -# which under pipefail+set-e would abort the whole sweep after the first combo. -set -eu -cd "$(git rev-parse --show-toplevel)" -OUT=tmp/svar2_mvp/prof_out/e1 -mkdir -p "$OUT" -PERF=/carter/users/dlaub/.pixi/bin/perf -PY=.pixi/envs/default/bin/python -TARGET_S=60 # ~60s hot loop per capture so startup is negligible -FREQ=199 - -probe_K () { # backend cohort -> K sized to ~TARGET_S - local per - per=$("$PY" tmp/svar2_mvp/prof_driver.py "$1" "$2" 5 | sed 's/per_call_s=//') - "$PY" -c "import math; print(max(20, math.ceil($TARGET_S/max(float('$per'),1e-4))))" -} - -: > "$OUT/dso_split.txt"; : > "$OUT/perf_top.txt"; : > "$OUT/callgraph.txt"; : > "$OUT/K_used.txt" - -for b in svar2 svar1; do for c in germline somatic; do - tag="${b}_${c}" - K=$(probe_K "$b" "$c"); echo "$tag K=$K" | tee -a "$OUT/K_used.txt" - $PERF record -g --call-graph fp -F $FREQ -o "$OUT/${tag}.perf.data" -- \ - "$PY" tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" >/dev/null 2>&1 - echo "== DSO split $tag ==" | tee -a "$OUT/dso_split.txt" - $PERF report --stdio --sort=dso --no-children -g none -i "$OUT/${tag}.perf.data" 2>/dev/null \ - | "$PY" tmp/svar2_mvp/e1_bucket_dso.py | tee -a "$OUT/dso_split.txt" - echo "== top self-time symbols $tag ==" | tee -a "$OUT/perf_top.txt" - $PERF report --stdio --sort=symbol --no-children -g none -i "$OUT/${tag}.perf.data" 2>/dev/null \ - | grep -vE '^\s*#|^\s*$' | head -15 | tee -a "$OUT/perf_top.txt" - # call-graph for the svar2 paths (shows e.g. SearchTree::build <- overlap_batch) - if [ "$b" = svar2 ]; then - echo "== call graph $tag (top) ==" >> "$OUT/callgraph.txt" - $PERF report --stdio --sort=overhead,symbol -i "$OUT/${tag}.perf.data" 2>/dev/null \ - | grep -vE '^\s*#|^\s*$' | head -40 >> "$OUT/callgraph.txt" - fi -done; done -echo "E1_PROFILE_DONE dso -> $OUT/dso_split.txt symbols -> $OUT/perf_top.txt" diff --git a/tmp/svar2_mvp/e2_build.sbatch b/tmp/svar2_mvp/e2_build.sbatch deleted file mode 100644 index b9179d31..00000000 --- a/tmp/svar2_mvp/e2_build.sbatch +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -#SBATCH -p carter-compute -#SBATCH --cpus-per-task=16 -#SBATCH --mem=64G -#SBATCH --array=0-3 -#SBATCH -J e2build -#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_build_%a.log -set -euo pipefail -cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel -W=/carter/users/dlaub/repos/for_loukik/svar2_mvp -SIZES=(1000 2000 4000 8000) -S=${SIZES[$SLURM_ARRAY_TASK_ID]} -echo "building S=$S host=$(hostname) cpus=$SLURM_CPUS_ON_NODE" -pixi run -e default python tmp/svar2_mvp/build_stores.py \ - "$W/gdc.chr21.s${S}.bcf" chr21 "$W/somatic_s${S}" -echo "E2_BUILD_DONE S=$S" diff --git a/tmp/svar2_mvp/e2_subsample.sbatch b/tmp/svar2_mvp/e2_subsample.sbatch deleted file mode 100644 index 74f7e137..00000000 --- a/tmp/svar2_mvp/e2_subsample.sbatch +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -#SBATCH -p carter-compute -#SBATCH --cpus-per-task=8 -#SBATCH --mem=16G -#SBATCH -J e2_subsample -#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_subsample.log -set -euo pipefail -W=/carter/users/dlaub/repos/for_loukik/svar2_mvp -SRC=$W/gdc.chr21.norm.filt.bcf -bcftools query -l "$SRC" > "$W/somatic.samples.txt" -TOTAL=$(wc -l < "$W/somatic.samples.txt") -echo "total somatic samples=$TOTAL" -: > "$W/e2_variant_counts.txt" -for S in 1000 2000 4000 8000; do - head -n "$S" "$W/somatic.samples.txt" > "$W/somatic.s${S}.list" - bcftools view -S "$W/somatic.s${S}.list" --force-samples --threads 8 -Ob \ - -o "$W/gdc.chr21.s${S}.bcf" "$SRC" - bcftools index -f "$W/gdc.chr21.s${S}.bcf" - N=$(bcftools view -H "$W/gdc.chr21.s${S}.bcf" | wc -l) - echo "S=$S variants=$N" | tee -a "$W/e2_variant_counts.txt" -done -N=$(bcftools view -H "$SRC" | wc -l) -echo "S=16007 variants=$N" | tee -a "$W/e2_variant_counts.txt" -echo "E2_SUBSAMPLE_DONE" diff --git a/tmp/svar2_mvp/env_baseline.txt b/tmp/svar2_mvp/env_baseline.txt deleted file mode 100644 index 10945a84..00000000 --- a/tmp/svar2_mvp/env_baseline.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Captured 2026-07-03T21:54:02-07:00 -# This Claude session runs INSIDE an interactive SLURM allocation: -host=carter-cn-04 -node_total_cpus=128 -node_total_mem_MB=953674 -alloc_cpus=2 # SLURM_CPUS_ON_NODE (this session's allocation) -alloc_mem_MB=8192 # SLURM_MEM_PER_NODE -alloc_walltime=14-00:00:00 -partition=carter-compute -qos=normal # no MaxTRESPU cap -governor=performance -turbo_no_turbo=NA -paranoid=2 # perf call-graph works, no sudo -# CONSEQUENCE: light single-threaded profiling (E1 3-region workload) runs DIRECTLY on -# this node (no srun — srun makes a constrained 2-cpu step and fails at >2 cpus). -# Heavy multi-core / large-RAM work (E2 builds, E4 thread sweep) goes via fresh sbatch -# jobs sized to the big nodes (96-128 cpu, 476-953 GB). diff --git a/tmp/svar2_mvp/genoray_debug_build.sbatch b/tmp/svar2_mvp/genoray_debug_build.sbatch deleted file mode 100644 index 8984e29c..00000000 --- a/tmp/svar2_mvp/genoray_debug_build.sbatch +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -#SBATCH -p carter-compute -#SBATCH --cpus-per-task=16 -#SBATCH --mem=32G -#SBATCH -J genoray_dbg -#SBATCH -o /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/genoray_debug_build.log -set -euo pipefail -cd /carter/users/dlaub/projects/genoray -CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ -RUSTFLAGS="-C force-frame-pointers=yes" \ -pixi run -e py310 maturin build --release -echo "WHEEL_BUILD_DONE" -ls -t target/wheels/genoray-2.15.0-*.whl | head -1 diff --git a/tmp/svar2_mvp/prof_driver.py b/tmp/svar2_mvp/prof_driver.py deleted file mode 100644 index af22ce77..00000000 --- a/tmp/svar2_mvp/prof_driver.py +++ /dev/null @@ -1,77 +0,0 @@ -"""E1 single-path profiling driver. Exercises ONE code path in a warm loop so -py-spy/perf attribute time to that path only. - - python prof_driver.py - -Prints: per_call_s= -For svar1, the 3-region .gvl is written ONCE before the loop (we profile the -query, not gvl.write).""" - -import sys -import time - -import numpy as np - -W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" -REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" -CHROM = "chr21" -REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] - - -def _ref(): - import pysam - - rb = pysam.FastaFile(REF).fetch(CHROM).encode() - return np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) - - -def make_svar2(cohort): - from genoray import SparseVar2 - from genvarloader._dataset._svar2_source import SparseVar2Source - - src = SparseVar2Source(SparseVar2(f"{W}/{cohort}.svar2")) - ru, ro = _ref() - - def call(): - src.reconstruct( - CHROM, REGIONS, ru, ro, pad_char=ord("N"), shifts=None, output_length=-1 - ) - - return call - - -def make_svar1(cohort): - import polars as pl - import genvarloader as gvl - from genoray import SparseVar2 - - n_s = SparseVar2(f"{W}/{cohort}.svar2").n_samples - bed = pl.DataFrame( - { - "chrom": [CHROM] * len(REGIONS), - "chromStart": [s for s, _ in REGIONS], - "chromEnd": [e for _, e in REGIONS], - } - ) - ds_path = f"{W}/{cohort}.gvl" - gvl.write(ds_path, bed, variants=f"{W}/{cohort}.svar", overwrite=True) # ONCE - ds_hap = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") - - def call(): - ds_hap[: len(REGIONS), :n_s] - - return call - - -def main(backend, cohort, K): - call = {"svar1": make_svar1, "svar2": make_svar2}[backend](cohort) - call() # warm - t0 = time.perf_counter() - for _ in range(K): - call() - dt = time.perf_counter() - t0 - print(f"per_call_s={dt / K:.4f}") - - -if __name__ == "__main__": - main(sys.argv[1], sys.argv[2], int(sys.argv[3])) diff --git a/tmp/svar2_mvp/prof_getitem.py b/tmp/svar2_mvp/prof_getitem.py deleted file mode 100644 index e212eead..00000000 --- a/tmp/svar2_mvp/prof_getitem.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Profile the LIVE SVAR2 read-bound Dataset.__getitem__ path (not the union -oracle) for the in-scope modes. One (mode, cohort) per process so cProfile/perf -attribute cleanly. - - python tmp/svar2_mvp/prof_getitem.py - -gvl.write + Dataset.open run ONCE (we profile the READ, not the write). Prints -per_call_s over K warm calls. Tracks mode is out of scope; variant-windows is -guarded NotImplementedError in Svar2Haps and cannot be profiled yet.""" - -import sys -import time -from pathlib import Path - -import polars as pl - -STORE_DIR = Path("/carter/users/dlaub/projects/svar2_mvp") -REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" -CHROM = "chr21" -REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] -WORK = Path("tmp/svar2_mvp/prof_out/readbound") - - -def _bed(): - return pl.DataFrame( - { - "chrom": [CHROM] * len(REGIONS), - "chromStart": [s for s, _ in REGIONS], - "chromEnd": [e for _, e in REGIONS], - } - ) - - -def make_call(mode, cohort): - import genvarloader as gvl - from genoray import SparseVar2 - - prefix = STORE_DIR / cohort - sv2 = SparseVar2(f"{prefix}.svar2") - n_s = sv2.n_samples - ds_path = WORK / f"{cohort}_{mode}.gvl" - WORK.mkdir(parents=True, exist_ok=True) - - gvl.write( - ds_path, - _bed(), - variants=SparseVar2(f"{prefix}.svar2"), - samples=None, - max_jitter=0, - overwrite=True, - ) - ds = gvl.Dataset.open(ds_path, reference=REF) - view = ds.with_seqs(mode) # "haplotypes" or "variants" - - R = len(REGIONS) - - def call(): - view[:R, :n_s] - - return call - - -def main(mode, cohort, K): - call = make_call(mode, cohort) - call() # warm - t0 = time.perf_counter() - for _ in range(K): - call() - print(f"per_call_s={(time.perf_counter() - t0) / K:.5f}") - - -if __name__ == "__main__": - main(sys.argv[1], sys.argv[2], int(sys.argv[3])) diff --git a/tmp/svar2_mvp/prof_out/readbound/RESULTS.md b/tmp/svar2_mvp/prof_out/readbound/RESULTS.md deleted file mode 100644 index 9237fa0a..00000000 --- a/tmp/svar2_mvp/prof_out/readbound/RESULTS.md +++ /dev/null @@ -1,214 +0,0 @@ -# SVAR2 read-bound `Dataset.__getitem__` optimization — consolidated results (2026-07-06) - -Plan: `docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md` -Branches: gvl `svar2-m6b-kernel` (PR #266) + genoray `svar-2`. - -This report consolidates Phase A (baselining) and Phase B (B1-B4 optimization) -of the SVAR2 read-bound `getitem` perf effort. It supersedes any per-task -summary for headline numbers; per-task detail lives in -`.superpowers/sdd/task-{A2,A3,B1,B2,B3,B4-step1,B4a,B4b,B4c}-report.md` and the -condensed ledger in `.superpowers/sdd/progress.md` (section "SDD Progress — -SVAR2 read-bound getitem perf"). - -## 1. Correction to the A3 baseline note (dense_union) - -The original A3 native-baseline commentary is sometimes paraphrased as "the -union oracle (`dense_union()`) is never invoked on the read-bound path." That -overstates what was found and is corrected here: - -- `dense_union()` **is** called on the read-bound path (genoray - `src/query.rs:771`) — it is not absent. -- What A3 actually established (verified against the real call graph, not - grep-only) is that it's **cheap and below the sampling threshold**: it never - shows up with measurable self-time in any of the 4 `perf` captures, and the - disqualifying whole-cohort entry points (`overlap_batch`/`overlap_sample`) - are genuinely absent from the read-bound call chain - (`SparseVar2.find_ranges` → `gather_haps_readbound`/`gather_ranges_readbound`). - `genoray_core::search::SearchTree::build` (1.54% haplotypes_germline, - 0.55% haplotypes_somatic) is the benign per-region `find_ranges` search - phase, not the whole-cohort union oracle. -- **Correct statement for future reference:** *`dense_union()` is called on - the read-bound path but is cheap (below sampling threshold); the - whole-cohort union/oracle path (`overlap_batch`/`overlap_sample`) is what's - absent, not `dense_union()` itself.* - -## 2. Baseline reference numbers (Phase A) - -`perf stat -e instructions,cycles` on the whole profiled process (one -`gvl.write` + `Dataset.open` + K warm `ds[:, :]` calls), frame-pointer build, -committed in `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` — this -file is misleadingly named; **it holds the PRE-B1-B3 (original A3) capture**, -not a post-B1-B3 one (naming preserved as-is per the task record; do not -invert this when reading raw files). - -| combo | K | instructions | cycles | insn/cycle | -|---|---|---|---|---| -| haplotypes_germline | 178 | 136,108,783,319 | 63,090,572,307 | 2.16 | -| variants_germline | 6547 | 434,768,134,979 | 142,981,599,031 | 3.04 | -| haplotypes_somatic | 38 | 211,440,945,021 | 78,077,607,534 | 2.71 | -| variants_somatic | 1922 | 470,732,727,589 | 138,279,404,076 | 3.40 | - -A3 also profiled the Python layer (A2) and native layer (A3) to rank hot -functions. Python-layer hot path (both modes): pure-Python `_ragged_arange_gather` -(and `_2level`) issuing repeated small `numpy.arange`/`ndarray.repeat` calls — -the clearest vectorize/push-to-Rust candidate. Native-layer hot path: haplotypes -dominated by numpy int64 add/sub kernels + kernel/mmap page-fault time (~30% -each), with gvl/genoray Rust only ~8% of self-time; variants dominated by -`gather_haps_readbound` (12.85% germline) + `decode_variants_from_split` + -`split_to_flat` (11.64% combined, germline) + numpy `PyArray_Repeat` (9.13%). - -`tmp/svar2_mvp/prof_out/readbound/native_baseline.md` (also misleadingly -named) holds a **second, POST-B1-B3 re-profile** captured in Task B4 Step 1 to -enumerate the cargo-asm work-list (not a matched-K comparison against the -table above — K differs run-to-run because the harness doesn't pin sample -count): - -| combo | K | instructions | cycles | insn/cycle | -|---|---|---|---|---| -| haplotypes_germline | 191 | 135,051,821,403 | 64,154,539,866 | 2.11 | -| variants_germline | 7143 | 456,803,347,416 | 165,527,068,344 | 2.76 | -| haplotypes_somatic | 37 | 203,147,989,079 | 76,904,197,639 | 2.64 | -| variants_somatic | 1792 | 435,990,647,839 | 139,669,338,811 | 3.12 | - -**Because K is not matched between these two captures (different runs, -different cohort sizes drawn each time), they are not diffed directly as a -before/after number.** The reliable before/after deltas are the per-task, -matched-K, same-session measurements below. - -## 3. Optimizations applied — per-task matched-K deltas - -All changes are **byte-identical**: the svar2 pytest suite -(`pytest tests/dataset -k svar2`, 32 tests reading the real `svar2_mvp` -stores — haplotypes, variants, and tracks parity vs the SVAR1 oracle) stayed -32/32 green through every task, with zero new failures/errors introduced in -the full tree at any step (see §5, Parity). - -Each row below is its own same-session, same-K, git-stash-based before/after -measurement (not one cumulative run) — presented per-task as instructed, -since no single cumulative baseline-vs-final run was captured. - -| Task | Repo | Change | Mode/cohort (K) | Instructions before → after | Δ instructions | Status | -|---|---|---|---|---|---|---| -| **B1** | gvl | Skip redundant pre-reconstruct diffs gather for deterministic haplotype reads (`need_hap_lengths` inverted-default) | haplotypes_germline (K=178) | 136,108,783,319 → 127,529,646,381 | **−6.3%** (cycles −4.7%) | byte-identical | -| **B2** | gvl | Pre-size `split_to_flat` + `decode_variants_from_split` allocations | variants_somatic (K=300) | 160,637,370,802 → 161,125,582,889 | **+0.30% (noise)** | byte-identical, kept anyway | -| **B2** | gvl | (same change, haplotypes) | haplotypes_somatic (K=300) | 895,401,688,408 → 895,815,473,126 | **+0.05% (noise)** | byte-identical, kept anyway | -| **B3** | gvl | De-dup twin ragged reorder-index computation in `_reconstruct_variants` | variants_germline (K=500) | 59,472,976,343 → 58,383,771,799 | **−1.83%** | byte-identical | -| **B4a** | genoray (`svar-2`) | `gather_haps_readbound` asm fix: skip-prefix→slice, bounds-check hoist, inline 2-pointer merge (replaces per-hap `merge_keys` allocation) | variants_germline (K=500) | 58.36e9 → 53.41e9 (avg of 3 runs/side) | **≈−8.5%** (−4.95B instr) | byte-identical (proven + tie-break test) | -| **B4b** | gvl | `decode_variants_from_split` asm fix: hoist per-iter `q = h/ploidy` division; `get_unchecked` on presence bit (proven in-bounds, `debug_assert`-guarded) | variants_germline (K=500) | ≈53,464.9M → ≈53,317.4M (avg of 2 runs/side) | **≈−0.28%** | byte-identical (proven) | -| **B4c** | gvl | `split_to_flat` asm fix: hoist `q = h/ploidy` division out of both hot loops (4 div → 0), no `unsafe` | variants_germline (K=500) | ≈53,296.4M → ≈53,212.6M (avg of 3 runs/side) | **≈−0.15%** | byte-identical | - -**Headline result: B4a (genoray `gather_haps_readbound` asm fix) is the -single largest byte-identical win, ~8.5% instructions on the -variants_germline (K=500) workload** — larger than B1's 6.3% haplotypes win -and an order of magnitude larger than B4b/B4c. B2 is a null-delta-but-kept -scalability improvement (see reasoning below); B1 and B3 are Python-layer -structural/DRY wins with real, smaller, matched-K deltas. - -### Why B2 shows no measurable win here - -`decode_variants_from_split`/`split_to_flat` pre-sizing is a genuine -allocation-count reduction (proven via cargo unit tests and code review), but -the harness's fixed `REGIONS` (3 windows, ~500-1000bp each) keep -`dense_total`/`total_bits`/`cap` tiny per call, so few `Vec` reallocations are -actually eliminated per call. The eliminated cost is a rounding error against -the multi-billion-instruction total dominated by write+open setup and -Python/numpy glue at this harness's scale. The optimization is kept as a -zero-risk scalability improvement: its payoff should scale with region -size/variant density, which this benchmark doesn't exercise, and both targets -independently rank high in the A3/B4-Step-1 native profiles (`split_to_flat` -5.05-5.21% self-time, `decode_variants_from_split` 6.59-5.43% self-time). - -## 4. Profiled-but-deferred candidates - -Task B4 Step 1 re-profiled the native layer after B1-B3 and enumerated every -gvl/genoray Rust symbol with ≥1.5% self-time in any mode (6 functions). The -user approved a scoped **sequential** asm pass on the top 3 by ROI (B4a/b/c -above, all done). The remaining 3 were profiled and explicitly **deferred**, -not forgotten, as a controller-approved scope decision (do only the top-3 asm -targets this round; leave the rest for a follow-up pass): - -| Symbol | Repo | Max self-time | Reason deferred | -|---|---|---|---| -| `svar2_codec::decode_key` | genoray `svar2-codec/src/lib.rs:237` | 2.33% (variants_germline) | Below the top-3 ROI cutoff; scope decision to do only the top-3 sequentially this round | -| `genoray_core::spine::merge_keys` | genoray `src/spine.rs:63` | 2.31% (variants_somatic) | Same; note B4a's inline merge in `gather_haps_readbound` already eliminated one call site of this pattern, but the standalone `merge_keys` function itself (used elsewhere) was not asm-optimized | -| `genoray_core::search::SearchTree::build` | genoray `src/search.rs:93` | 1.54% (haplotypes_germline) | Same; also note this is the **benign** per-region `find_ranges` search phase, not the whole-cohort union oracle (see §1) — deferring it is a pure perf scope call, not a correctness concern | - -## 5. Parity - -- **svar2 suite: 32/32 byte-identical, held through every task** (B1, B2, B3, - B4a, B4b, B4c) — covers haplotypes, variants, and tracks paths against the - SVAR1 oracle (`pytest tests/dataset -k svar2`). -- **Full tree: NOT fully green, but no regressions.** `pixi run -e dev gen` - (ground-truth fixture generation) is **broken pre-existing to this plan**: - `VcfBuilder.__init__() got an unexpected keyword argument 'fileformat'` - (`tests/_builders/case.py:324`), a vcfixture (unpinned, `>=0.5.0`) API-drift - issue unrelated to svar2 or this perf work. This produces a fixed ~428 - errors / 46 failed baseline in `pytest tests -q` that held constant, - unchanged, through every task in this plan. The passed count held at 559 - (+1 from B1's added micro-test) with **zero new failures or errors** - introduced at any step — confirmed by identical failing-test-ID sets with - each change stashed vs applied. -- **This is a known issue / CI blocker requiring a separate fix** - (pin vcfixture or update the `VcfBuilder` call site at - `tests/_builders/case.py:324`), tracked here for visibility. It is **out of - scope** for this perf plan (dependency/test-infra issue, not a code - regression) and should be filed/fixed separately before branch CI can go - green. - -## 6. Deferred features (out of scope this round, by design) - -- **Tracks**: out of scope for this round's profiling and optimization. - `Svar2Haps.get_haps_and_shifts`'s tracks caller still runs the diffs kernel - via the `need_hap_lengths=True` default (B1's inverted default preserves - this — only the pure-haplotypes `__call__` entry point opts out with - `need_hap_lengths=False`). The B1 double-gather is therefore **unaddressed - for tracks by design**; tracks parity (`test_svar2_tracks_match_svar1*`) - was verified unaffected/green throughout, but no tracks-path perf work was - done. -- **Variant-windows**: guarded with `NotImplementedError` in `Svar2Haps` — - cannot be profiled or optimized until implemented. Deferred until that - feature lands. - -## 7. API / format / docs reconciliation (B5 Step 2) - -**Read-path internals only; no API/format/doc-surface change.** - -- `need_hap_lengths` (B1) is an internal parameter on - `Svar2Haps.get_haps_and_shifts`, not a public API — it is not exported and - is not part of `genvarloader.__all__`. -- All B2/B3/B4a/B4b/B4c changes are Rust/Python read-path internals - (allocation pre-sizing, asm-level instruction elimination, a de-duplicated - index computation) with no change to any public symbol, function signature - exposed to users, on-disk dataset format, or CLI/bcftools/plink2 - preprocessing requirement. -- No genoray kernel FFI signature changed (`gather_haps_readbound`, - `decode_variants_from_split`, `split_to_flat` all keep their existing - signatures — only their bodies changed). -- Per the "Maintaining the `genvarloader` skill" and "Docs audit" rules in - `CLAUDE.md`: since nothing in `__all__`, `gvl.write`, `Dataset.open`, or any - `Dataset.with_*` signature/default changed, **no update to - `skills/genvarloader/SKILL.md`, `docs/source/api.md`, or any other prose doc - is required**, and none was made. No genoray docs (`docs/roadmap/svar-2.md`) - needed updating either, since no genoray kernel signature changed — the - genoray commits below are pure asm/allocation fixes on existing functions. - -## 8. Two-repo commit list - -**gvl** (`svar2-m6b-kernel`, PR #266), in order: -- `56c7b36` — B1: skip redundant pre-reconstruct gather for deterministic haplotype reads -- `a297d24` — B2: pre-size split_to_flat + decode_variants_from_split allocations -- `ecfc057` — B3: compute the pos/ilen ragged reorder index once in variants decode -- `2bdee38` — B4 Step 1: re-profile native layer after B1-B3; enumerate cargo-asm work-list -- `1e894d2` — B4b: decode_variants_from_split asm fix (byte-identical) -- `85a8925` — B4b follow-up: debug_assert guard for the get_unchecked invariant -- `e99a0d9` — B4c: split_to_flat asm fix — hoist q=h/ploidy division out of hot loops (byte-identical) - -**genoray** (`svar-2`): -- `a7c32b3` — B4a: gather_haps_readbound asm fix — kill skip/take waste, elide bounds checks, inline the per-hap merge (byte-identical) -- `69b3c97` — B4a follow-up: test covering gather_haps_readbound same-position SNP+indel tie; cross-ref merge_keys - -## 9. Sources - -Per-task detail, TDD evidence, and full profiler output: `.superpowers/sdd/task-{A2,A3,B1,B2,B3,B4-step1,B4a,B4b,B4c}-report.md`. -Condensed ledger: `.superpowers/sdd/progress.md` (section "SDD Progress — SVAR2 read-bound getitem perf"). -Raw captures: `tmp/svar2_mvp/prof_out/readbound/{python_baseline.md,native_after_b1b3.md,native_baseline.md,asm_targets.md}`. diff --git a/tmp/svar2_mvp/prof_out/readbound/asm_targets.md b/tmp/svar2_mvp/prof_out/readbound/asm_targets.md deleted file mode 100644 index 3de82502..00000000 --- a/tmp/svar2_mvp/prof_out/readbound/asm_targets.md +++ /dev/null @@ -1,98 +0,0 @@ -# SVAR2 read-bound: cargo-asm work-list (post B1-B3) - -Captured 2026-07-06 via `tmp/svar2_mvp/prof_perf.sh` after rebuilding with -`RUSTFLAGS="-C force-frame-pointers=yes"` on top of commits through -`a297d24` (B2: pre-size split_to_flat/decode_variants_from_split). Source -data: `tmp/svar2_mvp/prof_out/readbound/native_baseline.md` (fresh, -post-B1-B3). Pre-B1-B3 numbers preserved in -`tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` (misleadingly named — -it holds the OLD/pre-B1-B3 capture). - -Sanity re-check (A3-style): grep for `overlap_batch`/`dense_union` in the -fresh `native_baseline.md` returned **no matches** — union oracle confirmed -absent from the read-bound path. `SearchTree::build` tops out at 1.54% -(haplotypes_germline), consistent with the benign per-region `find_ranges` -search, not a regression. - -## Per-mode top-5 native symbols (including excluded numpy/libc/kernel), with asm-fixable-Rust characterization - -### haplotypes_germline (K=191) -| self% | symbol | owner | -|---|---|---| -| 12.31% | `[k] 0xffffffffb1a0f327` | kernel (unresolved, page-fault/mmap path) | -| 11.64% | `mapiter_trivial_get` | numpy | -| 11.11% | `LONG_add_AVX2` | numpy | -| 10.86% | `LONG_subtract_AVX2` | numpy | -| 3.86% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | - -Characterization: dominated by numpy int64 add/sub kernels + kernel-side -paging; only `gather_haps_readbound` (3.86%) + `SearchTree::build` (1.54%, -outside top-5) clear our cutoff → **~5.4% of self-time is asm-fixable Rust** -in this mode. B1 (skip redundant haplotype gather) appears to have already -squeezed most of the Rust cost out of the haplotypes path. - -### variants_germline (K=7143) -| self% | symbol | owner | -|---|---|---| -| 18.31% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | -| 6.79% | `PyArray_Repeat` | numpy | -| 5.43% | `genvarloader::svar2::decode_variants_from_split` | gvl (Rust, asm-fixable) | -| 5.21% | `genvarloader::svar2::split_to_flat` | gvl (Rust, asm-fixable) | -| 4.65% | `_int_free` | libc | - -Characterization: this is the mode with the most asm-fixable Rust left — -`gather_haps_readbound` + `decode_variants_from_split` + `split_to_flat` + -`merge_keys` (2.06%) + `svar2_codec::decode_key` (2.33%) sum to **~33.3% of -self-time in gvl/genoray Rust**, the single largest optimization target of -the four modes. - -### haplotypes_somatic (K=37) -| self% | symbol | owner | -|---|---|---| -| 11.02% | `mapiter_trivial_get` | numpy | -| 10.07% | `[k] 0xffffffffb1a0f327` | kernel | -| 9.09% | `PyUnicode_RichCompare` | python | -| 8.09% | `LONG_add_AVX2` | numpy | -| 8.07% | `LONG_subtract_AVX2` | numpy | - -Characterization: essentially **no asm-fixable Rust remains** — only -`SearchTree::build` appears at all (0.55%, below cutoff). Entirely -numpy/python/kernel-structural at this point. - -### variants_somatic (K=1792) -| self% | symbol | owner | -|---|---|---| -| 6.86% | `genoray_core::query::gather_haps_readbound` | genoray (Rust, asm-fixable) | -| 5.74% | `PyUnicode_RichCompare` | python | -| 4.41% | `[k] 0xffffffffb1a0f327` | kernel | -| 4.02% | `mapiter_get` | numpy | -| 3.89% | `_int_free` | libc | - -Characterization: `gather_haps_readbound` + `decode_variants_from_split` -(2.56%) + `split_to_flat` (1.79%) + `merge_keys` (2.31%) sum to **~13.5% of -self-time in Rust** — about half of variants_germline's asm-fixable budget. - -## Work-list: gvl/genoray native symbols with self-time ≥1.5% in ANY mode - -| symbol | repo/file:line | max self% | modes (self%) | perf.data tag(s) | -|---|---|---|---|---| -| `genoray_core::query::gather_haps_readbound` | genoray `src/query.rs:1086` | 18.31% | haplotypes_germline (3.86%), variants_germline (18.31%), variants_somatic (6.86%) | `haplotypes_germline`, `variants_germline`, `variants_somatic` | -| `genvarloader::svar2::decode_variants_from_split` | gvl `src/svar2/mod.rs:269` | 5.43% | variants_germline (5.43%), variants_somatic (2.56%) | `variants_germline`, `variants_somatic` | -| `genvarloader::svar2::split_to_flat` | gvl `src/svar2/mod.rs:159` | 5.21% | variants_germline (5.21%), variants_somatic (1.79%) | `variants_germline`, `variants_somatic` | -| `svar2_codec::decode_key` | genoray `svar2-codec/src/lib.rs:237` | 2.33% | variants_germline (2.33%) | `variants_germline` | -| `genoray_core::spine::merge_keys` | genoray `src/spine.rs:63` | 2.31% | variants_germline (2.06%), variants_somatic (2.31%) | `variants_germline`, `variants_somatic` | -| `genoray_core::search::SearchTree::build` | genoray `src/search.rs:93` | 1.54% | haplotypes_germline (1.54%), haplotypes_somatic (0.55%, below cutoff) | `haplotypes_germline` | - -**6 functions clear the ≥1.5% cutoff** → fan-out size of 6 for the parallel -cargo-asm pass. All are genoray-owned except `decode_variants_from_split` and -`split_to_flat` (gvl-owned, `src/svar2/mod.rs`). - -Sub-cutoff, noted for completeness (do not fan out on these): `genvarloader::svar2::hap_diffs_svar2` -peaks at 0.58% (haplotypes_germline) — below the 1.5% bar in every mode. - -## Excluded categories (structural, not cargo-asm-fixable) -- libc: `_int_malloc`/`_int_free`/`__memmove_avx_unaligned_erms`/`__memcmp_avx2_movbe`/`malloc`/`__libc_calloc` -- numpy: `_multiarray_umath` internals — `mapiter_trivial_get`/`mapiter_get`/`LONG_add_AVX2`/`LONG_subtract_AVX2`/`PyArray_Repeat`/`npyiter_buffered_iternext`/`_contig_to_contig` -- Python interpreter / GC: `_PyEval_EvalFrameDefault`, `gc_collect_main`, `deduce_unreachable`, `visit_reachable`, `dict_traverse`, `PyUnicode_RichCompare`, `PyObject_RichCompare(Bool)`, `list_contains`, `list_index` -- Rust std/alloc/toolchain (not gvl/genoray application code): `alloc::vec::Vec::from_iter` variants, `__rustc::__rust_dealloc`, `__rustc::__rust_no_alloc_shim_is_unstable_v2`, `alloc::raw_vec::RawVec::grow_one` -- unresolved kernel samples: `[k] 0xffffffffb1a0f327`, `[k] 0xffffffffb14fa26d`, `[k] 0xffffffffb1c011e0` diff --git a/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md b/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md deleted file mode 100644 index ea292b71..00000000 --- a/tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md +++ /dev/null @@ -1,571 +0,0 @@ -# SVAR2 read-bound native baseline (2026-07-06) -## haplotypes_germline (K=178) -### perf stat -2026-07-06 00:45:55.272 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl -2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. -2026-07-06 00:45:55.290 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - - 1.42% [.] __rustc::__rust_dealloc - - - 1.17% [.] arr_unravel_index - - -``` -### call graph (top) -``` - 40.02% 0.00% [.] 0x00005555558b5300 - - - | - ---0x5555558b5300 - | - --39.84%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold - pyo3::impl_::trampoline::fastcall_cfunction_with_keywords - pyo3::impl_::trampoline::trampoline - genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - | - --39.77%--genvarloader::ffi::decode_variants_from_svar2_readbound - | - |--38.02%--pyo3::marker::Python::detach - | | - | |--17.43%--genoray_core::query::gather_haps_readbound - | | | - | | |--1.61%--genoray_core::spine::merge_keys - | | | - | | |--0.98%--__rustc::__rust_dealloc - | | | - | | --0.69%--_int_free - | | - | |--17.04%--genvarloader::svar2::decode_variants_from_split - | | | - | | |--5.05%--genvarloader::svar2::split_to_flat - | | | - | | |--2.35%--svar2_codec::decode_key - | | | - | | --1.04%--alloc::raw_vec::RawVec::grow_one - | | - | --0.58%--__memmove_avx_unaligned_erms - | - --1.55%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - 39.85% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - - | - ---genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - | - --39.78%--genvarloader::ffi::decode_variants_from_svar2_readbound - | - |--38.03%--pyo3::marker::Python::detach - | | - | |--17.43%--genoray_core::query::gather_haps_readbound - | | | - | | |--1.61%--genoray_core::spine::merge_keys - | | | - | | |--0.98%--__rustc::__rust_dealloc -``` -## haplotypes_somatic (K=38) -### perf stat -2026-07-06 00:49:32.456 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl -2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. -2026-07-06 00:49:32.482 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 `gather_haps_readbound`, confirmed via -`src/ffi/mod.rs`). Confirmed via a dedicated symbol-usage check, not by inspection of this -report alone. - -Also note: the readbound haplotype FFI entry point is -`genvarloader::ffi::reconstruct_haplotypes_from_svar2_readbound` (`nm` confirms this is a -distinct symbol from the union-path `reconstruct_haplotypes_from_svar2`); neither -appears with meaningful self-time because they are thin PyO3 wrapper functions - the real -work happens in `gather_haps_readbound` (genoray_core) underneath. This is expected, not a -driver bug. - -### haplotypes (germline K=178, somatic K=38) - -Top 3 self-time symbols, by DSO: - -1. `mapiter_trivial_get` - numpy (`_multiarray_umath...so`) - 10.74% (germline) / 9.76% (somatic) -2. `LONG_subtract_AVX2` / `LONG_add_AVX2` - numpy - 10.04%/9.98% (germline) / 7.74%/7.88% (somatic) -3. `genoray_core::query::gather_haps_readbound` - genoray_core (statically linked into `genvarloader.abi3.so`) - 6.70% (germline) / 0.52% (somatic, see caveat below) - -Rust self-time in the gvl/genoray_core layer (`gather_haps_readbound` + `split_to_flat` -0.60% + `hap_diffs_svar2` 0.55%, germline) sums to ~7.85%, dwarfed by the ~30% combined -numpy fancy-indexing/arithmetic self-time (`mapiter_trivial_get` + `LONG_add_AVX2` + -`LONG_subtract_AVX2`) and by a large unresolved kernel/`[unknown]` component (31-32% DSO -share, germline; 28-29%, somatic) that lines up with sys time dominating user time in -`perf stat` (30.0s sys vs 15.9s user germline; 32.6s sys vs 19.8s user somatic) - almost -certainly page-fault/mmap overhead (reference-genome memmap and/or per-sample-scale array -allocation), not Rust reconstruction. `variants` mode by contrast has ~1-14s sys time, -confirming this kernel-time cost is haplotypes-specific. - -**B1 double-gather hint (gather_haps_readbound + split_to_flat vs diffs-only need):** -PARTIALLY SUPPORTED / INCONCLUSIVE ON THE EXACT RATIO. Within the Rust layer, -`gather_haps_readbound` (6.70%, germline) self-time is ~12x `hap_diffs_svar2`'s (0.55%), -qualitatively consistent with "gathering full haplotype sequences" costing far more than -computing diffs alone - i.e. supports that a diffs-only reconstruction path would be much -cheaper than the current full-gather. But the profiled ratio is far larger than the -brief's "~2x" expectation, and self-time percentages alone can't distinguish "redundant -double work" from "gather legitimately touches more bytes than diffs do" - call-count -instrumentation (not just perf self-time) is needed to pin the exact redundancy factor. -Bigger caveat: for haplotypes, the gvl/genoray Rust layer is a minority contributor -(~8%) of total native time; the numpy fancy-indexing arithmetic (~30%) and kernel/mmap -overhead (~30%) are larger targets by self-time. **Verdict: directionally supports B1 but -does not confirm the specific "~2x" claim; a B1 fix should be scoped alongside (not -instead of) investigating the numpy-level indexing arithmetic and the mmap/page-fault -kernel time, or its net effect on wall-clock will be modest.** - -The somatic cohort's `gather_haps_readbound` self-time (0.52%) is much lower than -germline's (6.70%) despite germline and somatic using the same code path; the somatic -haplotypes capture is instead dominated by `PyUnicode_RichCompare`/`PyObject_RichCompare`/ -`list_contains`/`list_index` (python3.10, ~9.6%/3.3%/3.2%/1.9%/0.8%) - likely -sample-name/list matching overhead scaling with the 16007-sample cohort, worth a separate -look but out of scope for the B1/B2 hints named in this brief. - -### variants (germline K=6547, somatic K=1922) - -Top 3 self-time symbols, by DSO: - -1. `genoray_core::query::gather_haps_readbound` - genoray_core (in `genvarloader.abi3.so`) - 12.85% (germline) / 6.75% (somatic) -2. `genvarloader::svar2::decode_variants_from_split` + `genvarloader::svar2::split_to_flat` - gvl `.so` (`genvarloader::svar2`) - 6.59%+5.05%=11.64% (germline) / 2.73%+~ (somatic, split_to_flat drops out of top-20) -3. `PyArray_Repeat` - numpy - 9.13% (germline) / 4.98% (somatic) - -**B2 allocation-churn hint (split_to_flat/decode_variants_from_split with `_int_malloc`/`SpecFromIter` beneath):** -SUPPORTED. The call graph shows `decode_variants_from_split` as a direct child of -`gather_haps_readbound`'s sibling call (via `pyo3::marker::Python::detach` -> -`genvarloader::ffi::decode_variants_from_svar2_readbound`), with `split_to_flat`, -`svar2_codec::decode_key`, and `alloc::raw_vec::RawVec::grow_one` nested directly -beneath it - `grow_one` is a live allocation-growth event, not incidental. Allocator -self-time symbols appear prominently at the top level in both cohorts: `_int_free` -(4.31% germline / 4.09% somatic), `_int_malloc` (1.71%/2.15%), `malloc`/`__libc_calloc` -(2.06%+1.56% germline / 2.37% somatic), and -` as SpecFromIterNested>::from_iter` (1.55% germline, a Rust -Vec-from-iterator allocation pattern) and `__rustc::__rust_dealloc` (1.42%/1.46%) - all -consistent with the decode/flatten step allocating and freeing many small buffers per -call rather than reusing/pre-sizing them. This is a clear Task B2 target: reduce -allocation count in `split_to_flat`/`decode_variants_from_split` (pre-size `Vec`s, -avoid `from_iter` on hot paths, reuse buffers across calls). diff --git a/tmp/svar2_mvp/prof_out/readbound/native_baseline.md b/tmp/svar2_mvp/prof_out/readbound/native_baseline.md deleted file mode 100644 index 3d1a942c..00000000 --- a/tmp/svar2_mvp/prof_out/readbound/native_baseline.md +++ /dev/null @@ -1,485 +0,0 @@ -# SVAR2 read-bound native baseline (2026-07-06) -## haplotypes_germline (K=191) -### perf stat -2026-07-06 02:29:29.559 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_haplotypes.gvl -2026-07-06 02:29:29.577 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. -2026-07-06 02:29:29.577 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - - - 1.48% [.] __libc_calloc - - - 1.35% [.] __rustc::__rust_dealloc - - - 1.01% [.] __rustc::__rust_no_alloc_shim_is_unstable_v2 - - -``` -### call graph (top) -``` - 44.40% 0.00% [.] 0x00005555558b5300 - - - | - ---0x5555558b5300 - | - --44.26%--cfunction_vectorcall_FASTCALL_KEYWORDS.cold - pyo3::impl_::trampoline::fastcall_cfunction_with_keywords - pyo3::impl_::trampoline::trampoline - | - --44.25%--genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - | - --44.21%--genvarloader::ffi::decode_variants_from_svar2_readbound - | - |--42.43%--pyo3::marker::Python::detach - | | - | |--23.19%--genoray_core::query::gather_haps_readbound - | | | - | | |--1.90%--genoray_core::spine::merge_keys - | | | - | | |--0.84%--__rustc::__rust_dealloc - | | | - | | |--0.66%--cfree@GLIBC_2.2.5 - | | | - | | --0.62%--_int_free - | | - | |--15.91%--genvarloader::svar2::decode_variants_from_split - | | | - | | |--5.22%--genvarloader::svar2::split_to_flat - | | | - | | |--2.10%--svar2_codec::decode_key - | | | - | | |--1.22%--alloc::raw_vec::RawVec::grow_one - | | | - | | --0.66%-- as alloc::vec::spec_from_iter::SpecFromIter>::from_iter - | | - | --0.51%--__rustc::__rust_dealloc - | - --1.57%-- as alloc::vec::spec_from_iter_nested::SpecFromIterNested>::from_iter - 44.26% 0.00% [.] genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - - - | - ---genvarloader::ffi::__pyfunction_decode_variants_from_svar2_readbound - | - --44.23%--genvarloader::ffi::decode_variants_from_svar2_readbound - | - |--42.44%--pyo3::marker::Python::detach - | | -``` -## haplotypes_somatic (K=37) -### perf stat -2026-07-06 02:33:35.183 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl -2026-07-06 02:33:35.209 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. -2026-07-06 02:33:35.210 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 - - ncalls tottime percall cumtime percall filename:lineno(function) - 200 0.738 0.004 43.452 0.217 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) - 200 0.001 0.000 42.714 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) - 200 0.004 0.000 42.713 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) - 200 0.002 0.000 42.706 0.214 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) - 200 0.044 0.000 42.681 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) - 200 0.006 0.000 42.607 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) - 200 0.146 0.001 42.599 0.213 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:319(get_haps_and_shifts) - 200 4.692 0.023 38.910 0.195 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:714(_assemble_haps) - 200 8.965 0.045 34.167 0.171 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) - 1200 0.003 0.000 16.360 0.014 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) - 600 0.002 0.000 16.352 0.027 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) - 600 16.346 0.027 16.346 0.027 {method 'repeat' of 'numpy.ndarray' objects} - 600 8.851 0.015 8.851 0.015 {built-in method numpy.arange} - 200 1.972 0.010 1.972 0.010 {built-in method genvarloader.genvarloader.reconstruct_haplotypes_from_svar2_readbound} - 200 1.271 0.006 1.271 0.006 {built-in method genvarloader.genvarloader.hap_diffs_from_svar2_readbound} - 400 0.008 0.000 0.253 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) - 1600 0.209 0.000 0.214 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) - 200 0.034 0.000 0.037 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) - 3200 0.030 0.000 0.030 0.000 {built-in method numpy.ascontiguousarray} - 200 0.013 0.000 0.023 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) - 200 0.004 0.000 0.022 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) - 400 0.000 0.000 0.017 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() - 200 0.001 0.000 0.016 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) - 200 0.001 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) - 200 0.004 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) - 200 0.001 0.000 0.014 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) - 200 0.004 0.000 0.013 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) - 400 0.002 0.000 0.012 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) - 600 0.001 0.000 0.011 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2979(prod) - 400 0.001 0.000 0.009 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) - - -``` - -### pyinstrument haplotypes germline (K=200) - -``` - - _ ._ __/__ _ _ _ _ _/_ Recorded: 00:31:04 Samples: 2811 - /_//_/// /_\ / //_// / //_'/ // Duration: 43.543 CPU time: 44.033 -/ _/ v5.1.2 - -Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 - -43.5427 main prof_python.py:16 -`- 43.5427 call prof_getitem.py:48 - |- 42.7615 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 - | `- 42.7615 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 - | `- 42.7615 getitem genvarloader/_dataset/_query.py:66 - | `- 42.7615 _getitem_unspliced genvarloader/_dataset/_query.py:154 - | `- 42.7594 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 - | `- 42.7594 Svar2Haps.get_haps_and_shifts genvarloader/_dataset/_svar2_haps.py:319 - | |- 39.0037 Svar2Haps._assemble_haps genvarloader/_dataset/_svar2_haps.py:714 - | | |- 34.2766 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 - | | | |- 16.3452 repeat numpy/core/fromnumeric.py:423 - | | | | `- 16.3452 _wrapfunc numpy/core/fromnumeric.py:53 - | | | | `- 16.3452 ndarray.repeat - | | | |- 8.9989 [self] genvarloader/_dataset/_svar2_haps.py - | | | `- 8.9325 arange - | | `- 4.7255 [self] genvarloader/_dataset/_svar2_haps.py - | |- 1.9794 reconstruct_haplotypes_from_svar2_readbound - | `- 1.3297 hap_diffs_from_svar2_readbound - `- 0.7812 [self] prof_getitem.py - -``` - -2026-07-06 00:31:54.920 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/germline_variants.gvl -2026-07-06 00:31:54.941 | INFO | genvarloader._dataset._write:write:288 - Using 3202 samples. -2026-07-06 00:31:54.941 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 - - ncalls tottime percall cumtime percall filename:lineno(function) - 200 0.000 0.000 1.064 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) - 200 0.000 0.000 1.064 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) - 200 0.001 0.000 1.063 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) - 200 0.001 0.000 1.061 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) - 200 0.037 0.000 1.031 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) - 200 0.001 0.000 0.975 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) - 200 0.041 0.000 0.973 0.005 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:527(_reconstruct_variants) - 200 0.542 0.003 0.542 0.003 {built-in method genvarloader.genvarloader.decode_variants_from_svar2_readbound} - 3000 0.002 0.000 0.134 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) - 1600 0.001 0.000 0.116 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) - 1600 0.114 0.000 0.114 0.000 {method 'repeat' of 'numpy.ndarray' objects} - 400 0.030 0.000 0.105 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) - 200 0.003 0.000 0.104 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) - 200 0.029 0.000 0.099 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:101(_ragged_arange_gather_2level) - 800 0.086 0.000 0.088 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) - 200 0.031 0.000 0.033 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) - 400 0.000 0.000 0.028 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() - 200 0.000 0.000 0.028 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) - 800/200 0.001 0.000 0.027 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) - 800/200 0.006 0.000 0.027 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) - 1200 0.002 0.000 0.022 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) - 1200 0.001 0.000 0.019 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) - 400 0.003 0.000 0.018 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:188(from_fields) - 200 0.001 0.000 0.018 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1437() - 1200 0.017 0.000 0.017 0.000 {method 'cumsum' of 'numpy.ndarray' objects} - 200 0.002 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) - 200 0.001 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_rag_variants.py:210(__init__) - 200 0.009 0.000 0.014 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) - 1200 0.013 0.000 0.013 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/function_base.py:1324(diff) - 1400 0.012 0.000 0.012 0.000 {built-in method numpy.ascontiguousarray} - - -``` - -### pyinstrument variants germline (K=200) - -``` - - _ ._ __/__ _ _ _ _ _/_ Recorded: 00:31:56 Samples: 1200 - /_//_/// /_\ / //_// / //_'/ // Duration: 1.070 CPU time: 1.064 -/ _/ v5.1.2 - -Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 - -1.0697 main prof_python.py:16 -`- 1.0697 call prof_getitem.py:48 - `- 1.0697 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 - `- 1.0697 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 - `- 1.0692 getitem genvarloader/_dataset/_query.py:66 - |- 0.9705 _getitem_unspliced genvarloader/_dataset/_query.py:154 - | `- 0.9705 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 - | `- 0.9705 Svar2Haps._reconstruct_variants genvarloader/_dataset/_svar2_haps.py:527 - | |- 0.5452 decode_variants_from_svar2_readbound - | |- 0.1100 [self] genvarloader/_dataset/_svar2_haps.py - | |- 0.1062 Svar2Haps._gather_inputs genvarloader/_dataset/_svar2_haps.py:632 - | | |- 0.0954 ascontiguousarray - | | `- 0.0108 memmap.__getitem__ numpy/core/memmap.py:334 - | |- 0.1040 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 - | | `- 0.1040 repeat numpy/core/fromnumeric.py:423 - | | `- 0.1040 _wrapfunc numpy/core/fromnumeric.py:53 - | | `- 0.1040 ndarray.repeat - | `- 0.1020 _ragged_arange_gather_2level genvarloader/_dataset/_svar2_haps.py:101 - | `- 0.0974 repeat numpy/core/fromnumeric.py:423 - | `- 0.0974 _wrapfunc numpy/core/fromnumeric.py:53 - | `- 0.0969 ndarray.repeat - `- 0.0978 genvarloader/_dataset/_query.py:119 - `- 0.0968 _reshape_outer genvarloader/_dataset/_query.py:131 - `- 0.0963 RaggedVariants.reshape seqpro/rag/_core.py:1428 - `- 0.0948 RaggedVariants._reshape_impl seqpro/rag/_core.py:1434 - `- 0.0903 from_fields seqpro/rag/_core.py:188 - |- 0.0617 seqpro/rag/_core.py:206 - | `- 0.0577 array_equal numpy/core/numeric.py:2378 - | |- 0.0417 [self] numpy/core/numeric.py - | `- 0.0125 _all numpy/core/_methods.py:61 - `- 0.0160 seqpro/rag/_core.py:213 - -``` - -2026-07-06 00:32:02.542 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_haplotypes.gvl -2026-07-06 00:32:02.562 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. -2026-07-06 00:32:02.563 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 - - ncalls tottime percall cumtime percall filename:lineno(function) - 50 0.338 0.007 53.447 1.069 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) - 50 0.000 0.000 53.109 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) - 50 0.001 0.000 53.108 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) - 50 0.001 0.000 53.106 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) - 50 0.047 0.001 53.099 1.062 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) - 50 0.360 0.007 53.036 1.061 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) - 50 0.554 0.011 52.675 1.053 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:319(get_haps_and_shifts) - 50 6.248 0.125 49.781 0.996 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:714(_assemble_haps) - 50 11.425 0.228 43.483 0.870 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) - 300 0.001 0.000 20.497 0.068 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) - 150 0.001 0.000 20.489 0.137 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) - 150 20.487 0.137 20.487 0.137 {method 'repeat' of 'numpy.ndarray' objects} - 150 11.570 0.077 11.570 0.077 {built-in method numpy.arange} - 50 1.400 0.028 1.400 0.028 {built-in method genvarloader.genvarloader.reconstruct_haplotypes_from_svar2_readbound} - 50 0.551 0.011 0.551 0.011 {built-in method genvarloader.genvarloader.hap_diffs_from_svar2_readbound} - 100 0.013 0.000 0.343 0.003 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) - 400 0.293 0.001 0.294 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) - 50 0.041 0.001 0.044 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) - 800 0.036 0.000 0.036 0.000 {built-in method numpy.ascontiguousarray} - 50 0.002 0.000 0.026 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) - 50 0.000 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) - 50 0.002 0.000 0.015 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) - 200 0.012 0.000 0.012 0.000 {method 'astype' of 'numpy.ndarray' objects} - 50 0.009 0.000 0.012 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) - 50 0.011 0.000 0.011 0.000 {method 'sort' of 'numpy.ndarray' objects} - 100 0.001 0.000 0.009 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) - 100 0.000 0.000 0.008 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) - 100 0.007 0.000 0.007 0.000 {method 'cumsum' of 'numpy.ndarray' objects} - 100 0.007 0.000 0.007 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/function_base.py:1324(diff) - 100 0.000 0.000 0.005 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() - - -``` - -### pyinstrument haplotypes somatic (K=50) - -``` - - _ ._ __/__ _ _ _ _ _/_ Recorded: 00:33:03 Samples: 1451 - /_//_/// /_\ / //_// / //_'/ // Duration: 53.412 CPU time: 53.899 -/ _/ v5.1.2 - -Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 - -53.4118 main prof_python.py:16 -`- 53.4118 call prof_getitem.py:48 - `- 53.0665 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 - `- 53.0665 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 - `- 53.0665 getitem genvarloader/_dataset/_query.py:66 - `- 53.0665 _getitem_unspliced genvarloader/_dataset/_query.py:154 - `- 53.0170 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 - `- 52.6492 Svar2Haps.get_haps_and_shifts genvarloader/_dataset/_svar2_haps.py:319 - |- 49.7006 Svar2Haps._assemble_haps genvarloader/_dataset/_svar2_haps.py:714 - | |- 43.3816 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 - | | |- 20.5025 repeat numpy/core/fromnumeric.py:423 - | | | `- 20.5025 _wrapfunc numpy/core/fromnumeric.py:53 - | | | `- 20.5025 ndarray.repeat - | | |- 11.5908 arange - | | `- 11.2883 [self] genvarloader/_dataset/_svar2_haps.py - | `- 6.2733 [self] genvarloader/_dataset/_svar2_haps.py - |- 1.4053 reconstruct_haplotypes_from_svar2_readbound - |- 0.5750 [self] genvarloader/_dataset/_svar2_haps.py - `- 0.5739 hap_diffs_from_svar2_readbound - -``` - -2026-07-06 00:34:04.651 | INFO | genvarloader._dataset._write:write:196 - Writing dataset to tmp/svar2_mvp/prof_out/readbound/somatic_variants.gvl -2026-07-06 00:34:04.732 | INFO | genvarloader._dataset._write:write:288 - Using 16007 samples. -2026-07-06 00:34:04.732 | INFO | genvarloader._dataset._write:write:293 - Writing genotypes. - - 0%| | 0/3 [00:00 - - ncalls tottime percall cumtime percall filename:lineno(function) - 200 0.175 0.001 4.216 0.021 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_getitem.py:48(call) - 200 0.000 0.000 4.041 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:2218(__getitem__) - 200 0.003 0.000 4.041 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_impl.py:1751(__getitem__) - 200 0.001 0.000 4.036 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:66(getitem) - 200 0.167 0.001 3.994 0.020 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:154(_getitem_unspliced) - 200 0.071 0.000 3.780 0.019 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:256(__call__) - 200 0.270 0.001 3.709 0.019 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:527(_reconstruct_variants) - 200 2.010 0.010 2.010 0.010 {built-in method genvarloader.genvarloader.decode_variants_from_svar2_readbound} - 200 0.033 0.000 0.809 0.004 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:632(_gather_inputs) - 800 0.693 0.001 0.696 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/memmap.py:334(__getitem__) - 3000 0.002 0.000 0.224 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:53(_wrapfunc) - 400 0.068 0.000 0.223 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:80(_ragged_arange_gather) - 1600 0.001 0.000 0.172 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:423(repeat) - 1600 0.169 0.000 0.169 0.000 {method 'repeat' of 'numpy.ndarray' objects} - 200 0.152 0.001 0.160 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:695(_inverse_row_perm) - 200 0.035 0.000 0.113 0.001 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:101(_ragged_arange_gather_2level) - 1400 0.080 0.000 0.080 0.000 {built-in method numpy.ascontiguousarray} - 200 0.005 0.000 0.065 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_svar2_haps.py:619(_contig_groups) - 1200 0.003 0.000 0.053 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_utils.py:56(lengths_to_offsets) - 200 0.001 0.000 0.052 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:138(unique) - 200 0.006 0.000 0.051 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/lib/arraysetops.py:323(_unique1d) - 1200 0.001 0.000 0.049 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/numpy/core/fromnumeric.py:2512(cumsum) - 1200 0.047 0.000 0.047 0.000 {method 'cumsum' of 'numpy.ndarray' objects} - 200 0.042 0.000 0.042 0.000 {method 'sort' of 'numpy.ndarray' objects} - 400 0.000 0.000 0.039 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:119() - 200 0.000 0.000 0.039 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_query.py:131(_reshape_outer) - 200 0.031 0.000 0.038 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/python/genvarloader/_dataset/_indexing.py:208(parse_idx) - 800/200 0.001 0.000 0.038 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1428(reshape) - 800/200 0.007 0.000 0.037 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:1434(_reshape_impl) - 400 0.004 0.000 0.036 0.000 /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/.pixi/envs/dev/lib/python3.10/site-packages/seqpro/rag/_core.py:188(from_fields) - - -``` - -### pyinstrument variants somatic (K=200) - -``` - - _ ._ __/__ _ _ _ _ _/_ Recorded: 00:34:14 Samples: 2807 - /_//_/// /_\ / //_// / //_'/ // Duration: 4.232 CPU time: 4.210 -/ _/ v5.1.2 - -Profile at /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel/tmp/svar2_mvp/prof_python.py:32 - -4.2318 main prof_python.py:16 -`- 4.2318 call prof_getitem.py:48 - |- 4.0152 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:2218 - | `- 4.0152 RaggedDataset.__getitem__ genvarloader/_dataset/_impl.py:1751 - | `- 4.0152 getitem genvarloader/_dataset/_query.py:66 - | `- 4.0137 _getitem_unspliced genvarloader/_dataset/_query.py:154 - | |- 3.8348 Svar2Haps.__call__ genvarloader/_dataset/_svar2_haps.py:256 - | | |- 3.6962 Svar2Haps._reconstruct_variants genvarloader/_dataset/_svar2_haps.py:527 - | | | |- 2.0787 decode_variants_from_svar2_readbound - | | | |- 0.7419 Svar2Haps._gather_inputs genvarloader/_dataset/_svar2_haps.py:632 - | | | | `- 0.7413 memmap.__getitem__ numpy/core/memmap.py:334 - | | | |- 0.3715 [self] genvarloader/_dataset/_svar2_haps.py - | | | |- 0.2237 _ragged_arange_gather genvarloader/_dataset/_svar2_haps.py:80 - | | | | |- 0.1215 [self] genvarloader/_dataset/_svar2_haps.py - | | | | `- 0.1012 repeat numpy/core/fromnumeric.py:423 - | | | | `- 0.1012 _wrapfunc numpy/core/fromnumeric.py:53 - | | | | `- 0.1012 ndarray.repeat - | | | |- 0.1660 _inverse_row_perm genvarloader/_dataset/_svar2_haps.py:695 - | | | `- 0.1110 _ragged_arange_gather_2level genvarloader/_dataset/_svar2_haps.py:101 - | | | |- 0.0569 repeat numpy/core/fromnumeric.py:423 - | | | | `- 0.0569 _wrapfunc numpy/core/fromnumeric.py:53 - | | | | `- 0.0569 ndarray.repeat - | | | `- 0.0541 [self] genvarloader/_dataset/_svar2_haps.py - | | `- 0.1386 [self] genvarloader/_dataset/_svar2_haps.py - | `- 0.1784 [self] genvarloader/_dataset/_query.py - `- 0.2166 [self] prof_getitem.py - -``` - - -## Top Python functions (ranked) - -Ranked by cProfile cumulative time (`cumtime`), cross-checked against the -pyinstrument call trees. "gvl Python" = defined in `python/genvarloader/`. -FFI wrappers = `{built-in method genvarloader.genvarloader.*}` frames — these -are single-call-boundary hops into the compiled Rust extension; their -`tottime` is Rust execution time, not Python interpreter overhead. - -### haplotypes mode (germline K=200, somatic K=50) - -| rank | function | cumtime (germline / somatic) | tottime (germline / somatic) | kind | -|---|---|---|---|---| -| 1 | `Svar2Haps.get_haps_and_shifts` (`_svar2_haps.py:319`) | 42.599s / 52.675s | 0.146s / 0.554s | pure-Python dispatcher (thin — nearly all time is in callees) | -| 2 | `Svar2Haps._assemble_haps` (`_svar2_haps.py:714`) | 38.910s / 49.781s | 4.692s / 6.248s | pure-Python, real work — non-trivial self-time plus calls into `_ragged_arange_gather` | -| 3 | `_ragged_arange_gather` (`_svar2_haps.py:80`) | 34.167s / 43.483s | 8.965s / 11.425s | pure-Python hot loop — dominates via the numpy calls it issues (`ndarray.repeat` 16.3s/20.5s, `numpy.arange` 8.9s/11.6s of tottime); this, not `_gather_inputs`, is the second-largest gvl-Python cost center | -| 4 | `reconstruct_haplotypes_from_svar2_readbound` (built-in) | 1.972s / 1.400s | 1.972s / 1.400s | **FFI wrapper** (Rust) — thin, self-time is Rust-side | -| 5 | `hap_diffs_from_svar2_readbound` (built-in) | 1.271s / 0.551s | 1.271s / 0.551s | **FFI wrapper** (Rust) — thin, self-time is Rust-side | -| — | `Svar2Haps._gather_inputs` (`_svar2_haps.py:632`) | 0.253s / 0.343s | 0.008s / 0.013s | pure-Python — **much smaller than expected**; see note below | - -**Deviation from expectation:** the brief expected `_gather_inputs` to rank among the top 3 -alongside `get_haps_and_shifts` and `_assemble_haps`. It does not — observed cumtime for -`_gather_inputs` is 0.25–0.34s vs. 38.9–49.8s for `_assemble_haps`, i.e. ~2 orders of magnitude -smaller in haplotypes mode. The actual #2/#3 hot spots are `_assemble_haps` itself (self-time) -and `_ragged_arange_gather`, which together account for essentially all non-FFI time; the -`numpy.ndarray.repeat` and `numpy.arange` calls issued *from inside* `_ragged_arange_gather` -are the single largest tottime consumers of the whole profile (16–20s and 9–12s respectively -at K=200/50). `_inverse_row_perm` and `_contig_groups` are present but negligible (<0.05s cum). - -### variants mode (germline K=200, somatic K=200) - -| rank | function | cumtime (germline / somatic) | tottime (germline / somatic) | kind | -|---|---|---|---|---| -| 1 | `Svar2Haps._reconstruct_variants` (`_svar2_haps.py:527`) | 0.973s / 3.709s | 0.041s / 0.270s | pure-Python dispatcher, thin | -| 2 | `decode_variants_from_svar2_readbound` (built-in) | 0.542s / 2.010s | 0.542s / 2.010s | **FFI wrapper** (Rust) — largest single tottime consumer in this mode | -| 3 | `Svar2Haps._gather_inputs` (`_svar2_haps.py:632`) | 0.104s / 0.809s | 0.003s / 0.033s | pure-Python — scales up sharply with cohort size (germline 3202 vs. somatic 16007 samples); dominated by `numpy.memmap.__getitem__` (0.696s tottime at somatic scale) | -| 4 | `_ragged_arange_gather` (`_svar2_haps.py:80`) | 0.105s / 0.223s | 0.030s / 0.068s | pure-Python hot loop (same as haplotypes mode, smaller absolute magnitude here) | -| 5 | `_ragged_arange_gather_2level` (`_svar2_haps.py:101`) | 0.099s / 0.113s | 0.029s / 0.035s | pure-Python hot loop, 2-level variant used for variants mode | -| — | `_contig_groups` (`_svar2_haps.py:619`) | 0.015s / 0.065s | 0.002s / 0.006s | pure-Python, small but present as expected | - -**Matches expectation:** the ranking generally confirms the brief's predicted list -(`_reconstruct_variants`, `_gather_inputs`, `_ragged_arange_gather`/`_2level`, `_contig_groups`), -with one addition — `decode_variants_from_svar2_readbound` (Rust FFI) is actually the single -largest *tottime* contributor in variants mode, ahead of any pure-Python gvl function. Also -notable: `_gather_inputs` is proportionally much more expensive for variants (rank 3, scales -with cohort/memmap-read size) than for haplotypes (negligible), the inverse of the brief's -haplotypes-mode expectation. - -### Cross-cutting observation - -In **both** modes, the true CPU-bound hot path is the pure-Python `_ragged_arange_gather` -(and its `_2level` sibling) issuing repeated small `numpy.arange`/`ndarray.repeat` calls in a -Python loop — this is the clearest "vectorize or push to Rust" candidate surfaced by this -baseline. The Rust FFI calls (`reconstruct_haplotypes_from_svar2_readbound`, -`hap_diffs_from_svar2_readbound`, `decode_variants_from_svar2_readbound`) are thin -single-hop wrappers whose cost is compiled-code execution time, not Python overhead, and are -out of scope for Python-layer optimization. diff --git a/tmp/svar2_mvp/prof_perf.sh b/tmp/svar2_mvp/prof_perf.sh deleted file mode 100644 index 81755ae8..00000000 --- a/tmp/svar2_mvp/prof_perf.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Native-layer attribution for the live read-bound path via perf (py-spy is -# unusable: ptrace_scope=2; Python 3.10 has no perf trampoline so Python frames -# are opaque -> DSO-level + Rust-symbol self-time is the split). -set -eu -cd "$(git rev-parse --show-toplevel)" -OUT=tmp/svar2_mvp/prof_out/readbound -mkdir -p "$OUT" -PERF=/carter/users/dlaub/.pixi/bin/perf -PY=.pixi/envs/dev/bin/python -FREQ=299 -REPORT="$OUT/native_baseline.md" -echo "# SVAR2 read-bound native baseline ($(date -I))" > "$REPORT" - -probe_K () { # mode cohort -> K sized to ~40s - local per - per=$("$PY" tmp/svar2_mvp/prof_getitem.py "$1" "$2" 5 | sed 's/per_call_s=//') - "$PY" -c "import math;print(max(20,math.ceil(40/max(float('$per'),1e-4))))" -} - -for c in germline somatic; do for m in haplotypes variants; do - tag="${m}_${c}"; K=$(probe_K "$m" "$c") - echo "## $tag (K=$K)" | tee -a "$REPORT" - # instruction-count reference (the Phase-B gate baseline) - echo '### perf stat' >> "$REPORT" - { "$PERF" stat -e instructions,cycles -- "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" ; } \ - 2>> "$REPORT" 1>/dev/null || echo "(perf stat HW counters unavailable)" >> "$REPORT" - # sampling profile - "$PERF" record -g --call-graph fp -F $FREQ -o "$OUT/$tag.data" -- \ - "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" >/dev/null 2>&1 - echo '### DSO split' >> "$REPORT"; echo '```' >> "$REPORT" - "$PERF" report --stdio --sort=dso --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ - | grep -vE '^\s*#|^\s*$' | head -12 >> "$REPORT"; echo '```' >> "$REPORT" - echo '### top Rust/native self-time symbols' >> "$REPORT"; echo '```' >> "$REPORT" - "$PERF" report --stdio --sort=symbol --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ - | grep -vE '^\s*#|^\s*$' | head -20 >> "$REPORT"; echo '```' >> "$REPORT" - echo '### call graph (top)' >> "$REPORT"; echo '```' >> "$REPORT" - "$PERF" report --stdio --sort=overhead,symbol -i "$OUT/$tag.data" 2>/dev/null \ - | grep -vE '^\s*#|^\s*$' | head -45 >> "$REPORT"; echo '```' >> "$REPORT" -done; done -echo "NATIVE_BASELINE_DONE -> $REPORT" diff --git a/tmp/svar2_mvp/prof_python.py b/tmp/svar2_mvp/prof_python.py deleted file mode 100644 index 3c85fbdf..00000000 --- a/tmp/svar2_mvp/prof_python.py +++ /dev/null @@ -1,43 +0,0 @@ -"""cProfile + pyinstrument over the live read (Python-layer attribution). - - python tmp/svar2_mvp/prof_python.py - -cProfile ranks Python functions by cumulative time; pyinstrument gives a -low-overhead statistical wall-clock call tree as a cross-check (cProfile's own -per-call overhead can distort tiny hot loops).""" - -import cProfile -import io -import pstats -import sys - -from prof_getitem import make_call - - -def main(mode, cohort, K): - call = make_call(mode, cohort) - call() # warm - - pr = cProfile.Profile() - pr.enable() - for _ in range(K): - call() - pr.disable() - s = io.StringIO() - pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats(30) - print(f"### cProfile {mode} {cohort} (K={K}), sort=cumulative\n") - print("```\n" + s.getvalue() + "```\n") - - from pyinstrument import Profiler - - p = Profiler(interval=0.0005) - p.start() - for _ in range(K): - call() - p.stop() - print(f"### pyinstrument {mode} {cohort} (K={K})\n") - print("```\n" + p.output_text(unicode=False, color=False, show_all=False) + "```\n") - - -if __name__ == "__main__": - main(sys.argv[1], sys.argv[2], int(sys.argv[3])) diff --git a/tmp/svar2_mvp/split_folded.py b/tmp/svar2_mvp/split_folded.py deleted file mode 100644 index d85c9fe2..00000000 --- a/tmp/svar2_mvp/split_folded.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Split a py-spy --format raw (folded) stack file into Python vs native -self-time by LEAF frame. A leaf frame is Python iff it contains '.py:'. - - python split_folded.py -""" - -import sys -from collections import Counter - - -def is_python(frame: str) -> bool: - return ".py:" in frame or frame.endswith(".py") - - -def main(path): - py = nat = 0 - leaves = Counter() - classed = {} - with open(path) as fh: - for line in fh: - line = line.rstrip("\n") - if not line: - continue - stack, _, cnt = line.rpartition(" ") - try: - n = int(cnt) - except ValueError: - continue - leaf = stack.split(";")[-1] - leaves[leaf] += n - classed[leaf] = "python" if is_python(leaf) else "native" - if is_python(leaf): - py += n - else: - nat += n - tot = py + nat - if tot == 0: - print("no samples parsed") - return - print( - f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}" - ) - print("top-15 leaf frames (self-time):") - for leaf, n in leaves.most_common(15): - print(f" {100 * n / tot:5.1f}% [{classed[leaf]:6s}] {leaf}") - - -if __name__ == "__main__": - main(sys.argv[1]) diff --git a/tmp/svar2_mvp/validate.py b/tmp/svar2_mvp/validate.py deleted file mode 100644 index df32bc47..00000000 --- a/tmp/svar2_mvp/validate.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Spot-check that gvl returns non-empty, sane haplotypes + variants through -both the SVAR1 (gvl Dataset over .svar) and SVAR2 (SparseVar2Source over .svar2) -backends, on a handful of regions x a few samples. Correctness is already proven -by the test suite; this proves the REAL-DATA plumbing works.""" - -import sys -from pathlib import Path - -import numpy as np -import genvarloader as gvl -from genoray import SparseVar2 -from genvarloader._dataset._svar2_source import SparseVar2Source - -REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" - - -def main(prefix: str, chrom: str): - # A few small regions (0-based, half-open) in a variant-dense chr21 window. - regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500)] - - # --- SVAR2 backend (adapter direct) --- - sv2 = SparseVar2(f"{prefix}.svar2") - print(f"[svar2] n_samples={sv2.n_samples} ploidy={sv2.ploidy}") - ref_bytes = _contig_ref(REF, chrom) - src = SparseVar2Source(sv2) - hap = src.reconstruct( - chrom, - regions, - np.frombuffer(ref_bytes, np.uint8), - np.array([0, len(ref_bytes)], np.int64), - pad_char=ord("N"), - shifts=None, - output_length=-1, - ) - lens = np.asarray(hap.offsets) - print( - f"[svar2] hap ragged rows={len(lens) - 1} " - f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}" - ) - var = sv2.decode(chrom, regions) - print(f"[svar2] decode variants: {var}") - - # --- SVAR1 backend (gvl Dataset over .svar) --- - import polars as pl - - bed = pl.DataFrame( - { - "chrom": [chrom] * len(regions), - "chromStart": [s for s, _ in regions], - "chromEnd": [e for _, e in regions], - } - ) - ds_path = f"{prefix}.gvl" - gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) - ds = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") - seqs = ds[: len(regions), :3] # a few regions x first 3 samples - print(f"[svar1] gvl haplotypes sample shape/type: {type(seqs)}") - - -def _contig_ref(fasta: str, chrom: str) -> bytes: - import pysam - - return pysam.FastaFile(fasta).fetch(chrom).encode() - - -if __name__ == "__main__": - main(sys.argv[1], sys.argv[2]) # argv: From 853ad617a84dbd2bc8d77e8bfb25eb5051537006 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 14:03:06 -0700 Subject: [PATCH 105/108] docs(svar2): backfill PR #266 link into Phase 6a roadmap entry Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/roadmaps/rust-migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index e7cde032..2920d995 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -778,7 +778,7 @@ _PR: —_ > 1.38×, variant-windows 4.58×). ### Phase 6a — SVAR2 read-bound dataset wiring (genoray query-only) ✅ -_PR: TBD (branch `svar2-m6b-kernel`)_ +_PR: [#266](https://github.com/mcvickerlab/GenVarLoader/pull/266) (branch `svar2-m6b-kernel`) — ⛔ release-gated on the genoray publish (see below)_ _Specs: `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` (genoray side), `docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md` (this side); earlier design specs `docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md` and From 3ccadd9d2d272543e4cd5833e521d2593d5c559a Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 14:10:46 -0700 Subject: [PATCH 106/108] fix(svar2): raise (not assert) on max_ends packing-width overflow A bare `assert` is stripped under `python -O`, so a pathological >~2Mb deletion footprint could silently corrupt the packed composite key. Raise ValueError so it fails fast regardless of optimization level. (Final whole-branch review nit.) Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_write.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index 2565dc00..035eb827 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1132,9 +1132,10 @@ def _svar2_region_max_ends( # by pos then by end; recover end = pos + ext on unpack. ext_var = 1 - np.minimum(ilen_arr, 0) # small: 1 + deletion length SHIFT = 21 - assert int(ext_var.max(initial=0)) < (1 << SHIFT), ( - "variant footprint exceeds tie-break packing width" - ) + # raise (not assert) so it still fails fast under `python -O`: a pathological + # >~2 Mb deletion footprint would otherwise silently corrupt the packed key. + if int(ext_var.max(initial=0)) >= (1 << SHIFT): + raise ValueError("variant footprint exceeds tie-break packing width") key = (pos_arr << SHIFT) | ext_var key_k = key[keep] region_k = region_of_var[keep] From 9e1f296c8983e0666dec67a4d4a134a4ab62e432 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 15:09:46 -0700 Subject: [PATCH 107/108] docs(svar2): reference issue #267 in the FlankSample fill-seed guard Task 8: the multi-contig FlankSample track-fill divergence was recorded only in a code comment behind a NotImplementedError guard. Reference the tracking issue so the guard can be lifted once the fill-seed index is made global. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/genvarloader/_dataset/_reconstruct.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index a240e5dc..294cf63c 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -409,7 +409,8 @@ def _call_svar2( # Single-contig is exact (local == global); non-seeded fills (Repeat5p # etc.) are exact regardless. Proper fix (follow-up): pass a per-group # global-query-offset into the FFI so the kernel seeds with the global - # row index; for now, guard. + # row index; for now, guard. Tracked in + # https://github.com/mcvickerlab/GenVarLoader/issues/267. n_contig_groups = int(np.unique(regions[:, 0]).size) if n_contig_groups > 1 and bool((strat_ids == FLANK_SAMPLE).any()): raise NotImplementedError( From 649689d1fff4abf8aba6bb9f700ebf5756522104 Mon Sep 17 00:00:00 2001 From: d-laub Date: Mon, 13 Jul 2026 15:36:27 -0700 Subject: [PATCH 108/108] fix(svar2): global fill-seed for multi-contig FlankSample tracks (#267) The read-bound SVAR2 track kernel is called once per contig group, so its `k / ploidy` query index is contig-LOCAL. The FlankSample fill hash `hash4(base_seed, query, hap, out_idx+i)` needs the GLOBAL batch row to match the single fused SVAR1 call, which diverged whenever a contig appeared at more than one global batch row. - `tracks::shift_and_realign_tracks_from_svar2` takes an optional `query_seed` (local -> global row map); `None` seeds with the local index (identity), keeping the single-fused-call/union path byte-for-byte. - The read-bound FFI wrapper threads a new `global_query` arg; the union wrapper passes `None`. - `realign_track_block` passes each contig group's global row indices (`qsel`). - Guard in `_call_svar2` lifted. Guard test converted to a parity test (`test_svar2_flanksample_multicontig_matches_svar1`, verified non-vacuous: it fails when seeded with the local index). Full pytest tree + cargo tests green; SVAR1 path byte-unchanged (additive Option param). Docs/skill/roadmap updated. (Incidental: `pixi lock` refreshed a stale narwhals transitive pin via the pixi-lock pre-commit hook.) Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/roadmaps/rust-migration.md | 25 ++++++- docs/source/faq.md | 2 +- docs/source/format.md | 4 +- pixi.lock | 67 +++++++++---------- python/genvarloader/_dataset/_reconstruct.py | 36 +++------- python/genvarloader/_dataset/_svar2_haps.py | 6 ++ skills/genvarloader/SKILL.md | 5 +- src/ffi/mod.rs | 10 +++ src/tracks/mod.rs | 19 +++++- tests/_oracles/svar2_readbound_inputs.py | 2 + tests/dataset/test_svar2_dataset.py | 70 ++++++++++++++------ 11 files changed, 157 insertions(+), 89 deletions(-) diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index 2920d995..8dd04b5a 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -811,13 +811,20 @@ as a Rust path-dep, the first place gvl's Rust crate depends on genoray's Rust c (kept only as the parity oracle for tests). - [x] Guard matrix (Phase-1 scope; raise `NotImplementedError`, not silent mis-compute): spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, - fixed-length (`int output_length`) haplotype-realigned tracks, `variants`/`variant-windows` + fixed-length (`int output_length`) haplotype-realigned tracks, and `variants`/`variant-windows` output with jitter (write `max_jitter>0` or read `jitter>0` — the readbound variants decode - has no right-clip), `extend_to_length=False` at write time, and multi-contig `FlankSample` - track fills (contig-local vs. global fill-seed query index divergence). **Now supported** + has no right-clip), plus `extend_to_length=False` at write time. **Now supported** (moved off the guard list this pass): `unphased_union` and `"variant-windows"` output for both `"variants"` and `"variant-windows"`, plus `var_fields`-selected store INFO/FORMAT fields — see the field-routing task line below. +- [x] Multi-contig `FlankSample` (seed-dependent) track fills (issue + [#267](https://github.com/mcvickerlab/GenVarLoader/issues/267)): the read-bound track kernel + is called once per contig group, so `k / ploidy` is a contig-LOCAL query index; the fill hash + `hash4(base_seed, query, hap, out_idx+i)` needs the GLOBAL batch row to match the single fused + SVAR1 call. `realign_track_block` now passes each group's global row indices into the FFI + (`global_query`) and `shift_and_realign_tracks_from_svar2` seeds with those (identity `None` + preserves the single-call/union path byte-for-byte). Guard lifted; parity locked by + `test_svar2_dataset.py::test_svar2_flanksample_multicontig_matches_svar1`. - [x] var_fields → .svar2 store INFO/FORMAT field routing (plan 2026-07-12-svar2-info-format-field-routing.md). - [x] Docs/skill audit (this task): `skills/genvarloader/SKILL.md`, `docs/source/{write,format,faq}.md`, @@ -891,6 +898,18 @@ conversion/write paths. ## Notes & decisions log +- 2026-07-13 (Phase 6a — issue [#267](https://github.com/mcvickerlab/GenVarLoader/issues/267) + multi-contig `FlankSample` fill-seed; branch `svar2-m6b-kernel`): lifted the last read-path guard. + `_call_svar2` realigns per contig group, so the kernel's `k / ploidy` query index is contig-LOCAL; + the FlankSample fill hash `hash4(base_seed, query, hap, out_idx+i)` needs the GLOBAL batch row to + match the single fused SVAR1 call, which diverged whenever a contig appeared at >1 global row. + Fix: `shift_and_realign_tracks_from_svar2` takes an optional `query_seed` (local→global row map); + the read-bound FFI wrapper threads a new `global_query` arg (each group's `qsel`) and Python passes + it in `realign_track_block`. `None`/single-fused-call and non-seeded fills (Repeat5p etc.) stay + byte-identical. Guard test converted to parity + (`test_svar2_flanksample_multicontig_matches_svar1`, verified non-vacuous: fails with the local + index). Full tree green; SVAR1 path byte-unchanged (additive `Option` param). + - 2026-07-13 (Phase 6a — final pre-merge hardening pass; branch `svar2-m6b-kernel`): Shipped scope grew past the 2026-07-05 entry below: `unphased_union` (both `"variants"` and `"variant-windows"`), `"variant-windows"` output itself, and `var_fields`-selected `.svar2` diff --git a/docs/source/faq.md b/docs/source/faq.md index 9b505c82..c9ed751d 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -78,7 +78,7 @@ Both are sparse columnar variant archives from [`genoray`](https://github.com/mc - **`.svar`** reconstructs by building an interval search tree over the queried window and a per-read dense union of the overlapping variants. - **`.svar2`** reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges at write time, and `Dataset.__getitem__` gathers directly off that cache and calls all-Rust kernels — it builds **no interval search tree and no dense union per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. -`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter, and multi-contig `FlankSample` track fills) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output, `unphased_union` (for both `"variants"` and `"variant-windows"`), and `var_fields`-selected store INFO/FORMAT fields (also for both, when the `.svar2` was written with them) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. +`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, and `variants`/`variant-windows` output with jitter) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output, `unphased_union` (for both `"variants"` and `"variant-windows"`), and `var_fields`-selected store INFO/FORMAT fields (also for both, when the `.svar2` was written with them) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs (and `FlatVariantWindows.alt`/`.alt_window` for `"variant-windows"`), and only for pure-deletion records. `ref_window` is byte-identical between the two backends. diff --git a/docs/source/format.md b/docs/source/format.md index 9be749a5..dc215243 100644 --- a/docs/source/format.md +++ b/docs/source/format.md @@ -153,7 +153,9 @@ The following combinations are Phase-1 scope and raise `NotImplementedError` (or - `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound decode does not right-clip to the post-jitter window). - `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source. -- `FlankSample` insertion-fill for tracks spanning multiple contigs in one query. + +(Multi-contig `FlankSample` track fills are now supported and byte-identical to the `.svar` +backend — issue #267.) See the `genvarloader` skill's `.svar2` section for the full narrative and `var_fields` semantics. diff --git a/pixi.lock b/pixi.lock index b3831f05..1f39437e 100644 --- a/pixi.lock +++ b/pixi.lock @@ -204,6 +204,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -233,7 +234,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -374,6 +374,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl @@ -411,7 +412,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl dev: channels: @@ -605,6 +605,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -647,7 +648,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl @@ -789,6 +789,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl @@ -824,7 +825,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl docs: channels: @@ -2019,6 +2019,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl @@ -2054,7 +2055,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl @@ -2121,6 +2121,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl @@ -2162,7 +2163,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl @@ -2464,6 +2464,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -2489,7 +2490,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -2696,6 +2696,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl @@ -2726,7 +2727,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py310: channels: @@ -2920,6 +2920,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -2949,7 +2950,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -3090,6 +3090,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl @@ -3127,7 +3128,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py311: channels: @@ -4649,6 +4649,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl @@ -4686,7 +4687,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl @@ -4896,6 +4896,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl @@ -4926,7 +4927,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl packages: - conda: https://conda.anaconda.org/bioconda/linux-64/bcftools-1.23.1-hb2cee57_0.conda @@ -14458,6 +14458,27 @@ packages: version: 1.37.1 sha256: c682bf83f5f352e5e02f5c16c652c48ca40442f07b236f30662b22217320ce76 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl + name: narwhals + version: 2.24.0 + sha256: 42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489 + requires_dist: + - cudf-cu12>=24.10.0 ; sys_platform == 'linux' and extra == 'cudf' + - dask[dataframe]>=2024.8 ; extra == 'dask' + - duckdb>=1.1 ; extra == 'duckdb' + - ibis-framework>=6.0.0 ; extra == 'ibis' + - packaging>=21.3 ; extra == 'ibis' + - pyarrow-hotfix>=0.7 ; extra == 'ibis' + - modin>=0.22.0 ; extra == 'modin' + - pandas>=1.3.4 ; extra == 'pandas' + - polars>=0.20.4 ; extra == 'polars' + - pyarrow>=13.0.0 ; extra == 'pyarrow' + - pyspark>=3.5.0 ; extra == 'pyspark' + - pyspark[connect]>=3.5.0 ; extra == 'pyspark-connect' + - narwhals[duckdb] ; extra == 'sql' + - sqlparse>=0.5.5 ; extra == 'sql' + - sqlframe>=3.22.0,!=3.39.3 ; extra == 'sqlframe' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl name: alabaster version: 1.0.0 @@ -17437,28 +17458,6 @@ packages: name: webencodings version: 0.5.1 sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 -- pypi: https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl - name: narwhals - version: 2.23.0 - sha256: 769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9 - requires_dist: - - cudf-cu12>=24.10.0 ; sys_platform == 'linux' and extra == 'cudf' - - dask[dataframe]>=2024.8 ; extra == 'dask' - - duckdb>=1.1 ; extra == 'duckdb' - - ibis-framework>=6.0.0 ; extra == 'ibis' - - rich>=12.4.4 ; extra == 'ibis' - - packaging>=21.3 ; extra == 'ibis' - - pyarrow-hotfix>=0.7 ; extra == 'ibis' - - modin>=0.22.0 ; extra == 'modin' - - pandas>=1.3.4 ; extra == 'pandas' - - polars>=0.20.4 ; extra == 'polars' - - pyarrow>=13.0.0 ; extra == 'pyarrow' - - pyspark>=3.5.0 ; extra == 'pyspark' - - pyspark[connect]>=3.5.0 ; extra == 'pyspark-connect' - - narwhals[duckdb] ; extra == 'sql' - - sqlparse>=0.5.5 ; extra == 'sql' - - sqlframe>=3.22.0,!=3.39.3 ; extra == 'sqlframe' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl name: pygments version: 2.20.0 diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index 294cf63c..8ca005c0 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -25,7 +25,7 @@ from .._utils import lengths_to_offsets from ._genotypes import _as_starts_stops from ._haps import _H, Haps, ReconstructionRequest, _NewH, _Variants -from ._insertion_fill import FLANK_SAMPLE, Repeat5p +from ._insertion_fill import Repeat5p from ._insertion_fill import lower as _lower_insertion_fills from ._flat_variants import _FlatVariantWindows from ._protocol import Reconstructor @@ -396,31 +396,15 @@ def _call_svar2( ] strat_ids, strat_params = _lower_insertion_fills(strat_list) - # Multi-contig FlankSample track fills seed the fill value off a - # contig-local query index, which diverges from the global fill seed - # across a batch; guarded until the fill-seed index is made global. - # SVAR1 realigns the whole batch - # in ONE fused call, so the fill hash `hash4(base_seed, query, hap, - # out_idx+i)` uses the GLOBAL row `query`. `_call_svar2` calls the - # readbound kernel once PER CONTIG GROUP, where `query = k/ploidy` is - # contig-LOCAL, so a global row landing at a different local position - # gets different fill offsets in inserted regions. base_seed matches - # (both derive from the full idx); only the per-query index diverges. - # Single-contig is exact (local == global); non-seeded fills (Repeat5p - # etc.) are exact regardless. Proper fix (follow-up): pass a per-group - # global-query-offset into the FFI so the kernel seeds with the global - # row index; for now, guard. Tracked in - # https://github.com/mcvickerlab/GenVarLoader/issues/267. - n_contig_groups = int(np.unique(regions[:, 0]).size) - if n_contig_groups > 1 and bool((strat_ids == FLANK_SAMPLE).any()): - raise NotImplementedError( - "svar2 haplotype-realigned tracks with a seed-dependent " - "insertion fill (FlankSample) across multiple contigs are not " - "yet supported: the per-contig-group kernel calls seed the fill " - "with a contig-local query index that diverges from the " - "single-batch SVAR1 path. Use a single-contig query, or a " - "non-seeded insertion fill (e.g. the default Repeat5p)." - ) + # Seed-dependent (FlankSample) fills stay byte-identical to the single + # fused SVAR1 call across a multi-contig batch: SVAR1 realigns the + # whole batch in ONE call, so the fill hash `hash4(base_seed, query, + # hap, out_idx+i)` uses the GLOBAL row `query`. `_call_svar2` calls the + # readbound kernel once PER CONTIG GROUP where `k / ploidy` is + # contig-LOCAL, so `realign_track_block` passes the group's global row + # indices into the FFI (`global_query`) and the kernel seeds with those + # instead of the local index. `base_seed` already matches (both derive + # from the full idx). Fixed in issue #267. # Base seed identical to the SVAR1 path (idx-xor when deterministic). if deterministic: base_seed = np.uint64( diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py index 169da6e2..4b903ba4 100644 --- a/python/genvarloader/_dataset/_svar2_haps.py +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -572,6 +572,12 @@ def realign_track_block( params_c, np.int64(strategy_id), np.uint64(base_seed), + # GLOBAL batch row per local query: this group's queries land at + # positions `qsel` in the full (b, P) batch. The kernel seeds the + # FlankSample fill with this (not the contig-local `k / ploidy`), + # so the per-contig-group split matches the single fused SVAR1 + # call byte-for-byte (issue #267). + np.ascontiguousarray(qsel, np.int64), should_parallelize(g_total * 4), ) cat_data.append(np.asarray(out_data, np.float32)) diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index 23d29ac9..34be2f5c 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -96,9 +96,10 @@ Unlike `.svar` (whose read path builds an interval search tree + a per-read dens - In-kernel reverse-complement (`to_rc`). - Fixed-length (integer `output_length`) haplotype-realigned **track** output (plain haplotype output at a fixed length is fine — only the track kernel is guarded). - `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). -- `FlankSample` insertion-fill for tracks spanning **multiple contigs** in one query (single-contig queries and non-seeded fills like the default `Repeat5p` are exact). - `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source (write-time; raises `NotImplementedError` — `.svar2` sources must use the default `extend_to_length=True`). +(`FlankSample` insertion-fill for tracks spanning multiple contigs in one query is now supported and byte-identical to the `.svar` backend — issue #267.) + **`variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. The same convention carries into `with_seqs("variant-windows")`: `ref_window` is byte-identical between `.svar`/`.svar2`, but `alt`/`alt_window` differ for pure-deletion records for the same reason. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw allele/window bytes differ for pure-deletion records. See `docs/source/faq.md`. Symbolic/breakend variants are rejected the same as `.svar`, but for `.svar2` the rejection happens **upstream, at `.svar2` conversion time** (the store format cannot represent them) — a `.svar2` must be built from an already-filtered source; gvl cannot re-filter a materialized `.svar2` any more than it can a materialized `.svar`. @@ -433,7 +434,7 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). - **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. -- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), `extend_to_length=False` at write time, and multi-contig `FlankSample` track fills all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, and `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and `extend_to_length=False` at write time all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`), and multi-contig `FlankSample` track fills (issue #267) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. - **`.svar2` `variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way; `ref_window` is also byte-identical — only raw ALT/`alt_window` bytes differ for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 278dd6ca..9331e5c0 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1213,6 +1213,7 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( params: PyReadonlyArray1, strategy_id: i64, base_seed: u64, + global_query: PyReadonlyArray1, parallel: bool, ) -> PyResult<(Bound<'py, PyArray1>, Bound<'py, PyArray1>)> { use crate::svar2; @@ -1255,6 +1256,11 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( let tracks_a = tracks.as_array(); let track_offsets_a = track_offsets.as_array(); let params_a = params.as_array(); + // (n_q,) LOCAL query -> GLOBAL batch row map for the FlankSample fill seed; + // the read-bound path calls this kernel once per contig group, so the local + // `k / ploidy` index would otherwise diverge from the single fused SVAR1 call + // (issue #267). + let global_query_a = global_query.as_array(); let (out_data, out_offsets_vec) = py.detach(move || { let rb = genoray_core::query::HapRanges::new( @@ -1335,6 +1341,7 @@ pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( params_a, strategy_id, base_seed, + Some(global_query_a), parallel, ); @@ -1621,6 +1628,9 @@ pub fn shift_and_realign_tracks_from_svar2<'py>( params_a, strategy_id, base_seed, + // Single fused call over the whole batch: `k / ploidy` IS the global + // row, so the FlankSample seed needs no remap (issue #267). + None, parallel, ); diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index 3220124d..c17c1088 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -693,6 +693,13 @@ pub fn shift_and_realign_tracks_sparse( /// - `track_offsets`: (n_q + 1,) offsets into tracks (one track per query) /// - `params`: per-strategy parameter (f64), shape (1,) /// - `strategy_id`, `base_seed`: insertion-fill strategy parameters +/// - `query_seed`: optional (n_q,) map from the LOCAL query index (`k / ploidy`) +/// to the GLOBAL batch row used as the FlankSample fill-seed `query` component. +/// `None` seeds with the local index (identity), which is exact for a +/// single-call/single-group batch. Callers that split one logical batch across +/// several kernel invocations (e.g. the SVAR2 read-bound path's per-contig-group +/// loop) MUST pass the group's global row indices so seed-dependent fills match +/// the single fused SVAR1 call. See GenVarLoader issue #267. /// - `parallel`: if true, use rayon to process work items concurrently #[allow(clippy::too_many_arguments)] pub fn shift_and_realign_tracks_from_svar2( @@ -715,6 +722,7 @@ pub fn shift_and_realign_tracks_from_svar2( params: ndarray::ArrayView1, strategy_id: i64, base_seed: u64, + query_seed: Option>, parallel: bool, ) { let ploidy = shifts.ncols(); @@ -740,6 +748,14 @@ pub fn shift_and_realign_tracks_from_svar2( let query = k / ploidy; let hap = k % ploidy; + // FlankSample fill-seed `query` component: GLOBAL batch row when a + // `query_seed` map is supplied (multi-call/per-group batches), else the + // local index (exact for a single fused call). See issue #267. + let q_seed = match query_seed { + Some(qs) => qs[query] as u64, + None => query as u64, + }; + let t_s = track_offsets[query] as usize; let t_e = track_offsets[query + 1] as usize; let q_track = ndarray::ArrayView1::from(&tracks_flat[t_s..t_e]); @@ -788,7 +804,7 @@ pub fn shift_and_realign_tracks_from_svar2( None, // keep: SVAR2 has no per-haplotype keep mask strategy_id, base_seed, - query as u64, + q_seed, hap as u64, ); }; @@ -2542,6 +2558,7 @@ mod tests { params.view(), REPEAT_5P, 0, + None, // query_seed: single call, local index is the global row false, // serial ); diff --git a/tests/_oracles/svar2_readbound_inputs.py b/tests/_oracles/svar2_readbound_inputs.py index 97cb2a55..67774858 100644 --- a/tests/_oracles/svar2_readbound_inputs.py +++ b/tests/_oracles/svar2_readbound_inputs.py @@ -272,6 +272,8 @@ def build_readbound_tracks( np.ascontiguousarray(params, np.float64), np.int64(strategy_id), np.uint64(base_seed), + # Single FFI call over the whole block: local query == global row (#267). + np.arange(n_q, dtype=np.int64), parallel, ) diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py index f2627ef7..5ceaa724 100644 --- a/tests/dataset/test_svar2_dataset.py +++ b/tests/dataset/test_svar2_dataset.py @@ -308,19 +308,22 @@ def test_svar2_tracks_match_svar1_multicontig( assert np.allclose(ad, bd, equal_nan=True), "track data differ" -def test_svar2_flanksample_multicontig_guard(tmp_path, svar2_fixture2, _src2): - """FlankSample (seed-dependent fill) + MULTI-contig must raise, not silently - diverge from SVAR1. - - ``_call_svar2`` realigns per contig group, seeding the fill with a - contig-LOCAL query index; SVAR1 seeds with the GLOBAL row. For a seed-only - fill (FlankSample) this diverges across >1 contig, so it is guarded. (The - single-contig and default-Repeat5p paths remain exact -- see the parity tests - above, which never set a fill.) +def test_svar2_flanksample_multicontig_matches_svar1( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """FlankSample (seed-dependent fill) + MULTI-contig is byte-identical to SVAR1. + + ``_call_svar2`` realigns per contig group, so ``k / ploidy`` is a contig-LOCAL + query index; SVAR1 realigns the whole batch in one fused call and seeds the + FlankSample fill hash with the GLOBAL row. Issue #267 makes the read-bound + path pass each group's global row indices (``global_query``) into the FFI so + the kernel seeds with the global row too. Without the fix the chr1 group's + queries (which land at global rows 1, 3 in the interleaved bed) would seed off + local indices 0, 1 and diverge in every inserted region. """ import pyBigWig - from genoray import SparseVar2 + from genoray import SparseVar, SparseVar2 from genvarloader._dataset._insertion_fill import FlankSample @@ -341,17 +344,28 @@ def test_svar2_flanksample_multicontig_guard(tmp_path, svar2_fixture2, _src2): paths[s] = str(p) track = gvl.BigWigs("signal", paths) - # Interleaved chr2/chr1 bed -> >1 contig group. + # Interleaved chr2/chr1 bed -> >1 contig group, with each contig appearing at + # multiple GLOBAL rows so local != global for the fill seed. bed = pl.DataFrame( { - "chrom": ["chr2", "chr1"], - "chromStart": [0, 0], - "chromEnd": [40, 40], + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], } ) - d = tmp_path / "fs.gvl" + d1 = tmp_path / "fs1.gvl" + d2 = tmp_path / "fs2.gvl" gvl.write( - d, + d1, + bed, + variants=SparseVar(svar_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), tracks=track, @@ -359,13 +373,27 @@ def test_svar2_flanksample_multicontig_guard(tmp_path, svar2_fixture2, _src2): max_jitter=0, overwrite=True, ) - ds = ( - gvl.Dataset.open(d, reference=ref) + fill = {"signal": FlankSample(flank_width=3)} + ds1 = ( + gvl.Dataset.open(d1, reference=ref) .with_tracks("signal") - .with_insertion_fill({"signal": FlankSample(flank_width=3)}) + .with_insertion_fill(fill) ) - with pytest.raises(NotImplementedError, match="FlankSample"): - ds[:, :] + ds2 = ( + gvl.Dataset.open(d2, reference=ref) + .with_tracks("signal") + .with_insertion_fill(fill) + ) + + _h1, a = ds1[:, :] + _h2, b = ds2[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "FlankSample track data differ" def _assert_ragged_equal(a, b, name: str) -> None: