diff --git a/.gitignore b/.gitignore index 2e7ef6bd..10ebd67f 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,4 @@ tests/benchmarks/profiling/*.speedscope.json tests/benchmarks/profiling/*.memray.bin tests/benchmarks/profiling/*.flamegraph.html tests/benchmarks/profiling/*.perf.data +tmp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dc6a879..42319117 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: # (e.g. from `pixi global install pyrefly`) would otherwise be picked # up by `git push` outside of a `pixi run` context and resolve the # wrong Python interpreter. - entry: pixi run -e dev pyrefly check + entry: pixi run -e dev pyrefly check python/genvarloader tests language: system pass_filenames: false - id: pixi-lock diff --git a/Cargo.lock b/Cargo.lock index 59c7c836..990b1eef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -67,6 +73,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "attohttpc" version = "0.25.0" @@ -116,7 +131,7 @@ dependencies = [ "serde", "smallvec", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "ufmt", ] @@ -136,6 +151,18 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -241,6 +268,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -275,6 +311,23 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -320,6 +373,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -435,6 +498,24 @@ dependencies = [ "slab", ] +[[package]] +name = "genoray" +version = "0.1.0" +dependencies = [ + "bytemuck", + "crossbeam-channel", + "half", + "memmap2", + "ndarray", + "ndarray-npy", + "numpy", + "pyo3", + "rayon", + "serde_json", + "svar2-codec", + "thiserror 2.0.18", +] + [[package]] name = "genvarloader" version = "0.2.1" @@ -442,6 +523,7 @@ dependencies = [ "anyhow", "bigtools", "coitrees", + "genoray", "itertools 0.14.0", "ndarray", "numpy", @@ -449,6 +531,8 @@ dependencies = [ "rayon", "rstest", "seqpro-core", + "svar2-codec", + "tempfile", ] [[package]] @@ -481,6 +565,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -640,6 +735,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -734,6 +838,25 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "ndarray" version = "0.17.2" @@ -750,6 +873,30 @@ dependencies = [ "rayon", ] +[[package]] +name = "ndarray-npy" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e8a348bca0075000d999d750420d74434fd0d3e0993b456554f885e7657a11" +dependencies = [ + "byteorder", + "ndarray", + "num-complex", + "num-traits", + "py_literal", + "zip", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -779,9 +926,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778da78c64ddc928ebf5ad9df5edf0789410ff3bdbf3619aed51cd789a6af1e2" +checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" dependencies = [ "libc", "ndarray", @@ -817,6 +964,48 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +dependencies = [ + "pest", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -875,12 +1064,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "py_literal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102df7a3d46db9d3891f178dcc826dc270a6746277a9ae6436f8d29fd490a8e1" +dependencies = [ + "num-bigint", + "num-complex", + "num-traits", + "pest", + "pest_derive", +] + [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ + "inventory", "libc", "once_cell", "portable-atomic", @@ -891,18 +1094,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -910,9 +1113,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -922,13 +1125,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.117", ] @@ -1123,6 +1325,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.23" @@ -1236,6 +1444,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "slab" version = "0.4.12" @@ -1260,6 +1474,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "svar2-codec" +version = "0.1.0" + [[package]] name = "syn" version = "1.0.109" @@ -1318,7 +1536,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1332,6 +1559,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1381,6 +1619,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "ufmt" version = "0.2.0" @@ -1738,6 +1982,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zerofrom" version = "0.1.7" @@ -1792,8 +2056,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 431165cd..98587c4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,19 +16,22 @@ bigtools = "0.5.6" coitrees = "0.4" itertools = "0.14.0" ndarray = "0.17.2" -numpy = "0.28.0" +numpy = "0.29.0" rayon = "1.12.0" seqpro-core = "0.1" +svar2-codec = { path = "/carter/users/dlaub/projects/genoray/svar2-codec" } +genoray_core = { path = "/carter/users/dlaub/projects/genoray", package = "genoray", default-features = false } [features] extension-module = ["pyo3/extension-module"] [dependencies.pyo3] -version = "0.28.3" +version = "0.29" features = ["abi3-py310"] [dev-dependencies] rstest = "0.26.1" +tempfile = "3" # Perf call-graph attribution only (`perf report --children`). Inherits release # codegen and adds line tables + frame pointers. NEVER the gate artifact — all diff --git a/README.md b/README.md index 7c4513e7..709b6390 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ GenVarLoader provides a fast, memory efficient data structure for training seque - Generate haplotypes up to 1,000 times faster than reading a FASTA file - Generate tracks up to 450 times faster than reading a BigWig - **Supports indels** and re-aligns tracks to haplotypes that have them -- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, and BigWig +- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, BigWig, and [genoray](https://github.com/mcvickerlab/genoray)'s sparse `.svar`/`.svar2` variant stores Documentation is available [here](https://genvarloader.readthedocs.io/). See our [preprint](https://www.biorxiv.org/content/10.1101/2025.01.15.633240) for benchmarking and implementation details. diff --git a/docs/known-issues/svar1-max-ends-tie-underextension.md b/docs/known-issues/svar1-max-ends-tie-underextension.md new file mode 100644 index 00000000..cb43cff0 --- /dev/null +++ b/docs/known-issues/svar1-max-ends-tie-underextension.md @@ -0,0 +1,29 @@ +# SVAR1 `_write_from_svar` max_ends under-extends at same-POS ties + +**Status:** open (latent bug in the SVAR1 write path; not yet fixed) +**Found:** 2026-07-05, during SVAR2 read-bound dataset wiring. + +## Symptom +`_write_from_svar` (`python/genvarloader/_dataset/_write.py`) computes each region's +`max_ends` as the end of the **highest-store-order variant** (`max v_idx`) overlapping +the region, then `chromEnd = max(max_ends, chromEnd)`. When two variant records share a +POS but have different ends — e.g. a SNP (`end = POS`) and a deletion (`end = POS - ILEN`) +at the same position — and the SNP has the higher store-order index, SVAR1 selects the +SNP's (shorter) end and **under-extends** the region. The deletion's coverage past the +region boundary is then truncated. + +Example: region `[11,13)`, `chr1:12 G>A` (SNP) and `chr1:12 GTA>G` (DEL, ILEN -2, end 14), +DEL indexed before SNP. SVAR1 → `chromEnd=13` (under-extended); the correct value is `14`. + +The code comment at the `max_ends` computation already hedges: "this is fine if there +aren't any overlapping variants that could make a v_idx < -1 have a further end." + +## SVAR2 behavior (correct) +The SVAR2 read-bound write path (`_write_from_svar2`) computes `max_ends` as the true end +of the max-position variant (`pos - min(ilen,0)`), yielding the **correct** `chromEnd=14`. + +## Parity policy +SVAR1↔SVAR2 `chromEnd` parity is byte-identical **except** same-POS multi-record tie +regions, where SVAR1 is the buggy oracle. Such regions are excluded from strict parity +(see the SVAR2 wiring plan's Task 7). Fixing SVAR1's `max_ends` is deferred (it would +change SVAR1 output for tie cases, i.e. it is not an additive change). diff --git a/docs/roadmaps/rust-migration.md b/docs/roadmaps/rust-migration.md index 8ed11a58..8dd04b5a 100644 --- a/docs/roadmaps/rust-migration.md +++ b/docs/roadmaps/rust-migration.md @@ -777,12 +777,118 @@ _PR: —_ > on every mode (tracks-only 1.07×, haplotypes/tracks-seqs 1.66×, annotated 1.43×, variants > 1.38×, variant-windows 4.58×). +### Phase 6a — SVAR2 read-bound dataset wiring (genoray query-only) ✅ +_PR: [#266](https://github.com/mcvickerlab/GenVarLoader/pull/266) (branch `svar2-m6b-kernel`) — ⛔ release-gated on the genoray publish (see below)_ +_Specs: `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` (genoray side), +`docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md` (this side); earlier design +specs `docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md` and +`docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md`._ + +A scoped slice of Phase 6, pulled forward: genoray's newer `.svar2` sparse variant format +becomes a `gvl.write` variant source and a live `Dataset` read backend, wired via a +**read-bound** path rather than the general genoray-VCF/PGEN absorption Phase 6 describes below. +gvl links `genoray_core` (query-only, `default-features = false` — no htslib conversion path) +as a Rust path-dep, the first place gvl's Rust crate depends on genoray's Rust core directly. + +- [x] `_svar2_link.py` (`Svar2Link`/`Svar2Fingerprint`/`_resolve_svar2`/`_verify_svar2_fingerprint`, + mirrors `_svar_link.py`) + `Metadata.svar2_link` field. +- [x] `_write_from_svar2`: write-time 6-array ranges cache (`vk_snp_range`, `vk_indel_range`, + `dense_snp_range`, `dense_indel_range`, `region_starts`, `sample_cols`, all int64) under + `genotypes/svar2_ranges/` + `svar2_meta.json`, sized to the dataset's **selected** samples; + `.svar2` write dispatch in `_write.py`. Same-POS-tie `max_ends` under-extension excluded + from parity — SVAR1-side bug, not introduced here (see + `docs/known-issues/svar1-max-ends-tie-underextension.md`). +- [x] `genoray_core` path-dep + `Svar2Store` pyclass (opens one `ContigReader` per contig once, + at `Dataset.open` — the SVAR2 analog of SVAR1's cached FFI statics). +- [x] Read-bound haplotype kernel: `reconstruct_haplotypes_from_svar2_readbound` — one FFI call, + `gather_haps_readbound` + `merge_hap3`, builds **zero** interval-search trees and **zero** + dense unions per read (structural guarantee, not just measured). +- [x] Read-bound track re-alignment kernel: `shift_and_realign_tracks_from_svar2_readbound`. +- [x] Read-bound variants/variant-windows decode kernel: `decode_variants_from_svar2_readbound` + (+ `hap_diffs_from_svar2_readbound` for jitter-shift computation). +- [x] Dataset read dispatch: `Svar2Haps` reconstructor, `source` discriminant, `Dataset.open(svar2=...)` + override mirroring `svar=`. Live `overlap_batch`/union dispatch in `_svar2_source.py` retired + (kept only as the parity oracle for tests). +- [x] Guard matrix (Phase-1 scope; raise `NotImplementedError`, not silent mis-compute): spliced + output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, + fixed-length (`int output_length`) haplotype-realigned tracks, and `variants`/`variant-windows` + output with jitter (write `max_jitter>0` or read `jitter>0` — the readbound variants decode + has no right-clip), plus `extend_to_length=False` at write time. **Now supported** + (moved off the guard list this pass): `unphased_union` and `"variant-windows"` output for + both `"variants"` and `"variant-windows"`, plus `var_fields`-selected store INFO/FORMAT + fields — see the field-routing task line below. +- [x] Multi-contig `FlankSample` (seed-dependent) track fills (issue + [#267](https://github.com/mcvickerlab/GenVarLoader/issues/267)): the read-bound track kernel + is called once per contig group, so `k / ploidy` is a contig-LOCAL query index; the fill hash + `hash4(base_seed, query, hap, out_idx+i)` needs the GLOBAL batch row to match the single fused + SVAR1 call. `realign_track_block` now passes each group's global row indices into the FFI + (`global_query`) and `shift_and_realign_tracks_from_svar2` seeds with those (identity `None` + preserves the single-call/union path byte-for-byte). Guard lifted; parity locked by + `test_svar2_dataset.py::test_svar2_flanksample_multicontig_matches_svar1`. +- [x] var_fields → .svar2 store INFO/FORMAT field routing (plan + 2026-07-12-svar2-info-format-field-routing.md). +- [x] Docs/skill audit (this task): `skills/genvarloader/SKILL.md`, `docs/source/{write,format,faq}.md`, + `README.md`; `api.md` ↔ `__all__` gate confirmed clean (no new public symbol — `svar2=` is a + parameter, not an exported name). + +**Gate (parity — MET):** all four output modes (haplotypes, tracks, variants, variant-windows) +byte-identical to the `.svar`/union-oracle (`SparseVar2Source.reconstruct`/`realign_tracks`, genoray +`decode`) across `tests/dataset/test_svar2_dataset.py`, `test_svar2_readbound_{haps,tracks,variants,diffs}.py`, +`test_write_svar2.py`, `test_svar2_fields_read.py` — 55/55 passed (`variant-windows` parity is +covered by `test_svar2_readbound_variants.py` and `test_svar2_fields_read.py`, not just `variants`). +One documented, intentional non-identity: for a pure deletion, `.svar2` decodes the atomized empty +ALT (`b""`) where `.svar` reports the VCF anchor base (`b"G"` for `GTA>G`) — a genoray format +convention; reconstructed haplotype bytes are unaffected (see `docs/source/format.md` "`.svar2` +variants ALT convention"). Also excluded from the write-path `max_ends` parity check: the +pre-existing SVAR1 same-POS-tie under-extension bug +(`docs/known-issues/svar1-max-ends-tie-underextension.md`) — a documented SVAR1-side defect, not a +`.svar2` regression. Full-tree regression: SVAR1 path byte-unchanged (additive-only change). + +**Checkpoint:** `.svar2` is a supported `gvl.write` source and `Dataset` read backend with a +structurally read-bound query path (no per-read tree build, no per-read dense union), on-disk +smaller than `.svar` especially for large cohorts. Remaining genoray absorption (VCF/PGEN ingest, +`.svar`→Rust, `.svar2` conversion/write path) stays in Phase 6 below. + +**Benchmark (MVP chr21, `/carter/users/dlaub/projects/svar2_mvp`, relocated 2026-07-05; drivers +`bench_readbound.py`/`prof_svar2_read.py`, live outside the repo):** matched `.svar`/`.svar2` +datasets, warm `Dataset[:, :]` variants read, same-session relative (absolute wall-clock is not +comparable across allocations on shared Carter nodes). +- **On-disk store size:** germline (3202-sample cohort) `.svar2` **5.67× smaller** (193 MB vs 1.1 GB); + somatic (16007-sample) **1.46× smaller** (36 MB vs 53 MB). The `.svar2` advantage grows with cohort. +- **perf DSO split (the success criterion — MET):** `perf record`/`report` of the warm SVAR2 read + loop shows **zero** `SearchTree::build`, `dense_union`, or `overlap_batch` samples. The only + `genoray_core` symbols are `gather_haps_readbound` + `merge_keys` (the read-bound gather); the rest + is `genvarloader::svar2::{split_to_flat,decode_variants_from_split}` + numpy cache-slicing + (`PyArray_Repeat`/`mapiter_get`). This is the flip the wiring was designed for: the earlier E1 + profiling found the union (`overlap_batch`) path spent ~80% in genoray `SearchTree::build` + (rebuilt per read); the read-bound path builds none. +- **Latency (small workloads):** on 30 regions × 100–200 samples the SVAR2 variants read is ~3.4× + *slower* than SVAR1 (germline 1.5 ms vs 0.45 ms; somatic 2.0 ms vs 0.60 ms) — per-read Python/numpy + cache-slicing overhead dominates at this scale. The structural win (no per-read tree/union rebuild, + smaller store) is what pays off at cohort scale and for the union path's contig-wide-stride concern; + a fair large-workload latency sweep is a follow-up. + +#### ⛔ Release gate (do NOT merge until genoray is released) + +This branch is dev-wired to a local genoray checkout and cannot build off this +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API lives on genoray main (unreleased). Flip ALL of these at +genoray release, then re-run the full py3xx matrix: + +- `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → published crates.io versions. +- `pixi.toml` [feature.py310.pypi-dependencies]: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"`. +- Verify the version-floor bumps already made are intended: numpy 0.29, pyo3 0.29, seqpro 0.21.1. + ### Phase 6 — Absorb genoray (future) ⬜ _PR: —_ Sequenced last; a candidate to graduate into its own roadmap once Phases 0–5 land. seqpro-core remains the ragged substrate (decision 2026-06-23) — Phase 6 is -narrowed to genoray (variant IO) only. +narrowed to genoray (variant IO) only. Phase 6a above already pulled the `.svar2` +**read-only query** surface (`genoray_core`, `default-features = false`) into the Rust +stack as a scoped precursor; Phase 6 covers the remaining VCF/PGEN ingest and +conversion/write paths. - [ ] Bring variant IO (genoray VCF/PGEN + sparse genotypes) into the Rust stack. @@ -792,6 +898,70 @@ narrowed to genoray (variant IO) only. ## Notes & decisions log +- 2026-07-13 (Phase 6a — issue [#267](https://github.com/mcvickerlab/GenVarLoader/issues/267) + multi-contig `FlankSample` fill-seed; branch `svar2-m6b-kernel`): lifted the last read-path guard. + `_call_svar2` realigns per contig group, so the kernel's `k / ploidy` query index is contig-LOCAL; + the FlankSample fill hash `hash4(base_seed, query, hap, out_idx+i)` needs the GLOBAL batch row to + match the single fused SVAR1 call, which diverged whenever a contig appeared at >1 global row. + Fix: `shift_and_realign_tracks_from_svar2` takes an optional `query_seed` (local→global row map); + the read-bound FFI wrapper threads a new `global_query` arg (each group's `qsel`) and Python passes + it in `realign_track_block`. `None`/single-fused-call and non-seeded fills (Repeat5p etc.) stay + byte-identical. Guard test converted to parity + (`test_svar2_flanksample_multicontig_matches_svar1`, verified non-vacuous: fails with the local + index). Full tree green; SVAR1 path byte-unchanged (additive `Option` param). + +- 2026-07-13 (Phase 6a — final pre-merge hardening pass; branch `svar2-m6b-kernel`): + Shipped scope grew past the 2026-07-05 entry below: `unphased_union` (both `"variants"` and + `"variant-windows"`), `"variant-windows"` output itself, and `var_fields`-selected `.svar2` + store INFO/FORMAT field routing (`rv["AF"]` / `win.fields["AF"]`, plan + `2026-07-12-svar2-info-format-field-routing.md`) are all done and parity-tested — none of + the three remain on the guard list. This pass also landed a round of correctness/perf + hardening with no behavior change to already-shipped modes: a serial-unsafe carve-path + guard (`debug_assert` monotonicity check), Python-reachable Rust panics converted to + `PyValueError` (non-contiguous/OOB input), a new `extend_to_length=False` guard for `.svar2` + write sources (raises `NotImplementedError` — was previously silently accepted), a + vectorized write-time `max_ends` computation (byte-identical), relocation of the + `SparseVar2Source` union-based oracle out of the shipped package into `tests/`, removal of + dead `annot_*` capability from the readbound haps kernel, a hoisted shared `present_bit` + helper, and a typecheck-task path fix. Full `.svar2` suite (`test_svar2_dataset.py`, + `test_svar2_readbound_{haps,tracks,variants,diffs}.py`, `test_write_svar2.py`, + `test_svar2_fields_read.py`) — 55/55 passed. + +- 2026-07-05 (Phase 6a — SVAR2 read-bound dataset wiring; branch `svar2-m6b-kernel`): + `.svar2` (genoray's newer sparse variant format) is now a `gvl.write` variant source and a + live `Dataset` read backend, wired end-to-end: write-time 6-array ranges cache + (`genotypes/svar2_ranges/`, sized to the dataset's selected samples, mirrors SVAR1's + `offsets.npy`/`svar_meta.json` pattern) + `Svar2Link` back-reference/fingerprint + (mirrors `SvarLink`, keyed on file-count + byte-size since `.svar2` exposes no cheap + `variant_idxs.npy` analogue); a new `genoray_core` Rust path-dep (query-only, + `default-features = false` — no htslib) backing a `Svar2Store` pyclass opened once at + `Dataset.open`; three read-bound all-Rust kernels + (`reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, + `decode_variants_from_svar2_readbound` + a `hap_diffs_from_svar2_readbound` helper for jitter + shifts) that gather directly off the write-time cache via `gather_haps_readbound` + a new + `merge_hap3` (var_key ⋈ dense_snp ⋈ dense_indel) — **no interval-search-tree build and no + dense-union rebuild per read**, unlike the existing union-based `SparseVar2Source` + (`overlap_batch`) path, which is retired from live dispatch and kept only as the parity + oracle. All four output modes parity-tested against that oracle / SVAR1: + `tests/dataset/test_svar2_dataset.py` + `test_svar2_readbound_{haps,tracks,variants,diffs}.py` + + `test_write_svar2.py`, 31/31 passed. Phase-1 scope excludes (guarded + `NotImplementedError`, not silent mis-compute): splicing, `var_filter="exonic"`, + `min_af`/`max_af`, `annotated` haplotypes, in-kernel `to_rc`, `unphased_union`, + `"variant-windows"`, fixed-length haplotype-realigned tracks, `variants` output with any + jitter (write or read), and multi-contig `FlankSample` track fills. One intentional, + documented non-identity vs. SVAR1: pure-deletion ALT bytes (`.svar2` emits the atomized + empty ALT `b""`; `.svar` emits the VCF anchor base) — a genoray format convention that does + not affect reconstructed haplotype bytes. The pre-existing SVAR1 same-POS-tie `max_ends` + under-extension bug (`docs/known-issues/svar1-max-ends-tie-underextension.md`) is excluded + from the `.svar2` write-path max_ends parity check, same as the rest of the SVAR1 parity + suite. Specs: `docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md`, + `docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md`, + `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` (genoray side), + `docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md` (this side, Phase 6a + above). Docs/skill updated (`skills/genvarloader/SKILL.md`, + `docs/source/{write,format,faq}.md`, `README.md`); `api.md` ↔ `__all__` gate confirmed + clean (no new public symbol — `svar2=` is a keyword argument, not an exported name). + - 2026-06-27 (Phase 5 W6 — wrap-up: thin-shim audit + cargo-standalone + seqpro-core + perf re-baseline; branch `phase-5-w6-wrapup`): Four parallel threads closed Phase 5: **(A) Thin-shim audit (Task 1, commit `0932374`):** Classified every Python step over the diff --git a/docs/source/dataset.md b/docs/source/dataset.md index f1792e6c..880436be 100644 --- a/docs/source/dataset.md +++ b/docs/source/dataset.md @@ -141,3 +141,32 @@ ds_itvs = ( ``` In `"flat"` output mode (`with_output_format("flat")`), float tracks return `FlatRagged` and interval tracks (`kind="intervals"`) return [`FlatIntervals`](api.md#genvarloader.FlatIntervals), which carries `.starts`, `.ends`, `.values` as `FlatRagged` fields and converts back via `.to_ragged()` → [`RaggedIntervals`](api.md#genvarloader.RaggedIntervals). + +## Variant fields (`var_fields`) + +`Dataset.open(..., var_fields=[...])` (and `Dataset.with_settings(var_fields=[...])`) selects which +per-variant fields load onto `"variants"` and `"variant-windows"` output, beyond the default +`["alt", "ilen", "start"]`. Requested names must be a subset of `Dataset.available_var_fields`. + +For a BCF/PGEN/`.svar`-backed dataset the available fields are the built-ins (`alt`, `start`, +`ref`, `ilen`, `dosage`) plus any per-variant INFO columns or per-call FORMAT fields the source +carries. + +For a **`.svar2`-backed** dataset, `available_var_fields` is narrower: +`["alt", "ilen", "start"]` plus whichever scalar-numeric INFO/FORMAT fields the `.svar2` store was +written with (via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])`) — +**`"ref"` and `"dosage"` are not valid `var_fields` for `.svar2` and requesting either raises**. +A requested store field shows up on both output kinds: + +```python +ds = gvl.Dataset.open("ds.gvl", reference="ref.fa", var_fields=["AF"]) + +rv = ds.with_seqs("variants")[0, 0] +rv["AF"] # per-variant AF values, aligned with rv.alt/.start/.ilen + +win = ds.with_seqs("variant-windows", gvl.VarWindowOpt(...)).with_output_format("flat")[0, 0] +win.fields["AF"] # same field, alongside win.fields["start"]/["ilen"] +``` + +See the `genvarloader` skill's `.svar2` `var_fields` section for the field-provenance and +dummy-fill details. diff --git a/docs/source/faq.md b/docs/source/faq.md index bb3b8a32..c9ed751d 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -71,6 +71,17 @@ GVL's read path (haplotype reconstruction and track re-alignment) is parallelize - **`GVL_FORCE_PARALLEL`** — set to a truthy value (`1`, `true`, `yes`, `on`) to force the multithreaded paths even on small inputs. By default GVL runs small inputs serially because thread overhead would dominate; this bypasses that size gate. Mainly useful for benchmarking. - **`RAYON_NUM_THREADS`** — GVL **overwrites** this with its own resolved count so an inherited value (e.g. baked into a base image) can't defeat the cgroup-aware cap. To size the pool yourself, use `GVL_NUM_THREADS` instead. +## Should I use `.svar` or `.svar2` as my variant source? + +Both are sparse columnar variant archives from [`genoray`](https://github.com/mcvickerlab/genoray) that `gvl.write(variants=...)` accepts alongside BCF/PGEN; see [write.md](write.md) for how to build one. The two differ in their read-time behavior: + +- **`.svar`** reconstructs by building an interval search tree over the queried window and a per-read dense union of the overlapping variants. +- **`.svar2`** reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges at write time, and `Dataset.__getitem__` gathers directly off that cache and calls all-Rust kernels — it builds **no interval search tree and no dense union per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. + +`.svar2` is Phase-1 scope: a handful of combinations (spliced output, `annotated` haplotypes, `min_af`/`max_af`, `var_filter="exonic"`, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, and `variants`/`variant-windows` output with jitter) aren't wired yet and raise `NotImplementedError` rather than silently mis-computing. `"variant-windows"` output, `unphased_union` (for both `"variants"` and `"variant-windows"`), and `var_fields`-selected store INFO/FORMAT fields (also for both, when the `.svar2` was written with them) are supported. See the `genvarloader` skill's `.svar2` section or `docs/source/format.md` for the full list. Everything else — haplotypes, tracks, and variants/variant-windows at any supported jitter/output-length combination — is byte-identical between the two backends. + +One documented difference in raw output: for a pure deletion, `with_seqs("variants")` on a `.svar` dataset reports the VCF anchor base as ALT (e.g. `b"G"` for `GTA>G`), while a `.svar2` dataset reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a bug. Reconstructed haplotypes are unaffected; only `RaggedVariants.alt` differs (and `FlatVariantWindows.alt`/`.alt_window` for `"variant-windows"`), and only for pure-deletion records. `ref_window` is byte-identical between the two backends. + ## How can I get personalized protein/spliced RNA sequences? This is not yet supported but on GVL's roadmap for the near future. Keep an eye out in future releases! diff --git a/docs/source/format.md b/docs/source/format.md index 165dd249..dc215243 100644 --- a/docs/source/format.md +++ b/docs/source/format.md @@ -11,17 +11,21 @@ dataset_dir/ ├── metadata.json # the Metadata schema (below) ├── input_regions.arrow # original BED regions + region-index map ├── genotypes/ # present iff variants were provided to gvl.write -│ ├── offsets.npy # per (region, sample, ploidy) offsets into variant_idxs.npy +│ ├── offsets.npy # per (region, sample, ploidy) offsets into variant_idxs.npy; absent when sourced from .svar2 │ ├── svar_meta.json # shape + dtype of offsets.npy — present iff source was .svar -│ ├── variant_idxs.npy # variant indices; absent when sourced from .svar -│ ├── dosages.npy # optional, absent when sourced from .svar -│ └── variants.arrow # variant table; absent when sourced from .svar +│ ├── variant_idxs.npy # variant indices; absent when sourced from .svar or .svar2 +│ ├── dosages.npy # optional, absent when sourced from .svar or .svar2 +│ ├── variants.arrow # variant table; absent when sourced from .svar or .svar2 +│ └── svar2_ranges/ # present iff source was .svar2 — see "svar2_ranges layout" below └── intervals/ # or annot_intervals/ when annotated; present iff tracks given ``` When the dataset was built from an `.svar`, the heavy per-variant arrays (`variant_idxs.npy`, `dosages.npy`, `index.arrow`) are **not duplicated** into the dataset. Instead the dataset records a back-reference to the source `.svar` in `metadata.json` (see `svar_link` below). +Likewise, a dataset built from an `.svar2` records a back-reference (`svar2_link`, below) +and caches only small per-`(region, sample, ploidy)` range arrays under `genotypes/svar2_ranges/` +— the bulk variant data stays in the `.svar2` store. ## `metadata.json` schema @@ -36,6 +40,7 @@ records a back-reference to the source `.svar` in `metadata.json` (see `svar_lin | `max_jitter` | `int` | Maximum coordinate jitter (defaults to 0). | | `version` | `SemanticVersion \| None` | Package version that wrote this dataset. Drives format dispatch. | | `svar_link` | `SvarLink \| None` | Back-reference to a source `.svar`, when present. | +| `svar2_link` | `Svar2Link \| None` | Back-reference to a source `.svar2`, when present. | `SvarLink`: @@ -52,6 +57,44 @@ records a back-reference to the source `.svar` in `metadata.json` (see `svar_lin | `n_variants` | `int` | Row count of the svar's `index.arrow`. | | `variant_idxs_bytes` | `int` | Byte size of the svar's `variant_idxs.npy`. | +`Svar2Link` (mirrors `SvarLink` for a `.svar2` source): + +| Field | Type | Notes | +|-------|------|-------| +| `relative_path` | `str` | POSIX path from `dataset_dir` to the `.svar2`. | +| `absolute_path` | `str` | Original absolute path; used as a fallback. | +| `fingerprint` | `Svar2Fingerprint` | Integrity check (see below). | + +`Svar2Fingerprint`: + +| Field | Type | Notes | +|-------|------|-------| +| `n_files` | `int` | Count of the `.svar2` store's `.bin`/`.npy` data files. | +| `store_bytes` | `int` | Summed byte size of those data files. | + +`.svar2` has no `variant_idxs.npy`/`index.arrow` analogue exposed cheaply, so its fingerprint +keys on file count + total byte size of the store's data files rather than a variant count. + +## `genotypes/svar2_ranges/` layout + +Written only when the dataset's variant source is a `.svar2` store. `R` = number of regions, +`S` = number of the dataset's **selected** samples (not necessarily the full `.svar2` cohort), +`P` = ploidy. All arrays are `int64`: + +| File | Shape | Notes | +|------|-------|-------| +| `vk_snp_range.npy` | `(R, S, P, 2)` | Per-`(region, sample, ploid)` half-open range into the `.svar2` store's SNP variant-key column. | +| `vk_indel_range.npy` | `(R, S, P, 2)` | Same, for the indel variant-key column. | +| `dense_snp_range.npy` | `(R, 2)` | Per-region (sample-independent) range into the dense SNP store. | +| `dense_indel_range.npy` | `(R, 2)` | Per-region (sample-independent) range into the dense indel store. | +| `region_starts.npy` | `(R,)` | Per-region write-time start coordinate. Retained for parity/debugging; the read path derives per-query starts from the (post-jitter) query regions and does **not** read this array's values. | +| `sample_cols.npy` | `(S,)` | Maps the dataset's selected-sample slot to the `.svar2` store's original sample index. | +| `svar2_meta.json` | — | Records each array's `shape`/`dtype` plus `ploidy`. | + +At read time, `Dataset.__getitem__` slices these memmaps (numpy fancy-indexing; no interval +search) to build the flat per-query inputs for the read-bound Rust kernels — no interval-search +tree and no dense-union rebuild happen per read, unlike the `.svar` path. + ## SVAR resolution at open time When opening a dataset whose `metadata.svar_link` is non-null, @@ -66,6 +109,56 @@ If none match, a `FileNotFoundError` is raised naming the expected `.svar` basen resolution, the fingerprint is verified; a mismatch raises `ValueError` and lists both expected and observed values. +## `.svar2` resolution at open time + +When opening a dataset whose `metadata.svar2_link` is non-null, +[`Dataset.open`](api.md#genvarloader.Dataset.open) resolves the `.svar2` store in the same order +as `.svar`: + +1. Caller-provided `svar2=...` argument. +2. `svar2_link.relative_path` resolved against the dataset directory. +3. `svar2_link.absolute_path`. +4. A unique `*.svar2` directory next to the dataset. + +If none match, a `FileNotFoundError` is raised naming the expected `.svar2` basename and +suggesting `svar2=`. After resolution, the fingerprint (`Svar2Fingerprint`, above) is verified; +a mismatch raises `ValueError` and lists both expected and observed values. + +## `.svar2` variants ALT convention + +For a pure deletion (e.g. VCF `GTA>G`), decoding `with_seqs("variants")` yields different raw +ALT bytes depending on the backing store: `.svar` reports the VCF anchor base (`b"G"`), while +`.svar2` reports the atomized empty ALT (`b""`) — a genoray `.svar2` format convention, not a +bug. Both stores consume the ALT identically when reconstructing haplotype sequence, so +`with_seqs("haplotypes")` / `with_seqs("annotated")` output is byte-identical between the two +backends; only `RaggedVariants.alt` differs, and only for pure-deletion records. The same holds +for `with_seqs("variant-windows")`: `ref_window` is byte-identical between the backends, while the +`alt`/`alt_window` fields differ only for pure-deletion records (the same empty-vs-anchor ALT). + +## `.svar2` Phase-1 unsupported combinations + +A `.svar2`-backed dataset supports all four output modes (`haplotypes`, `variants`, +`variant-windows`, and haplotype-realigned `tracks`), `unphased_union`, and +`var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`). +The following combinations are Phase-1 scope and raise `NotImplementedError` (or, for +`extend_to_length`, at write time) instead of silently mis-computing: + +- Spliced output. +- The `var_filter="exonic"` (keep-mask) variant filter. +- `min_af` / `max_af` filtering (`.svar` only; see "Should I use `.svar` or `.svar2`" in the FAQ). +- `annotated` haplotypes (`with_seqs("annotated")`). +- `VarWindowOpt(ref="allele")` (bare-allele REF mode; REF alleles aren't stored in `.svar2`). +- In-kernel reverse-complement (`to_rc`). +- Fixed-length (integer `output_length`) haplotype-realigned **track** output. +- `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with + `jitter>0` (the read-bound decode does not right-clip to the post-jitter window). +- `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source. + +(Multi-contig `FlankSample` track fills are now supported and byte-identical to the `.svar` +backend — issue #267.) + +See the `genvarloader` skill's `.svar2` section for the full narrative and `var_fields` semantics. + ## Format changelog | Version | Change | @@ -73,6 +166,7 @@ expected and observed values. | `< 0.18.0` | Variant coordinates stored 0-based. | | `0.18.0` | Variant coordinates switched to 1-based. | | `0.25.0` | `metadata.json` gains `svar_link`; old `genotypes/link.svar` symlink layout deprecated. `Metadata.version` typed as `SemanticVersion` (on-disk JSON unchanged). | +| `0.37.0` | `metadata.json` gains `svar2_link`; `.svar2` accepted as a `gvl.write` variant source, cached under `genotypes/svar2_ranges/` and read via a read-bound, all-Rust path. | > **Upgrading legacy datasets.** A dataset written before `0.25.0` that was built from an > `.svar` will still open (with a `DeprecationWarning`). Run diff --git a/docs/source/index.md b/docs/source/index.md index b0d1091f..8d5beecb 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -48,7 +48,7 @@ GenVarLoader provides a fast, memory efficient data structure for training seque - Generate haplotypes up to 1,000 times faster than reading a FASTA file - Generate tracks up to 450 times faster than reading a BigWig - **Supports indels** and re-aligns tracks to haplotypes that have them -- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, and BigWig +- Extensible to new file formats: drop a feature request! Currently supports VCF, PGEN, BigWig, and [genoray](https://github.com/mcvickerlab/genoray)'s sparse `.svar`/`.svar2` variant stores See our [preprint](https://www.biorxiv.org/content/10.1101/2025.01.15.633240) for benchmarking and implementation details. diff --git a/docs/source/write.md b/docs/source/write.md index a612d7a0..e47d8ee7 100644 --- a/docs/source/write.md +++ b/docs/source/write.md @@ -79,4 +79,32 @@ gvl.write( ) ``` -This dataset would have both haplotypes and two tracks (`pos` and `neg`) available for samples that exist in both `all_chroms.bcf` and the BigWig tables (i.e. `gvl.write()` performs an inner join on samples). \ No newline at end of file +This dataset would have both haplotypes and two tracks (`pos` and `neg`) available for samples that exist in both `all_chroms.bcf` and the BigWig tables (i.e. `gvl.write()` performs an inner join on samples). + +## Variants from a genoray sparse store (`.svar` / `.svar2`) + +Besides BCF/VCF and PGEN, `variants=` also accepts a genoray sparse columnar variant store — either the original `.svar` format or the newer `.svar2` format. Build one from a normalized VCF/BCF with `genoray`: + +```python +from genoray import VCF, SparseVar, SparseVar2 + +# .svar (SVAR1): a VCF reader + a memory budget +SparseVar.from_vcf("all_chroms.svar", VCF("normed.bcf"), max_mem="4g") + +# .svar2 (SVAR2): the VCF/BCF path + a reference FASTA (or no_reference=True) +SparseVar2.from_vcf("all_chroms.svar2", "normed.bcf", reference="ref.fa") +``` + +Then pass the resulting store to `gvl.write`: + +```python +gvl.write( + path="1000_genomes_haplotypes.gvl", + bed="tiling_windows.bed", + variants="all_chroms.svar2", # or "all_chroms.svar", or a SparseVar/SparseVar2 instance +) +``` + +Both formats store a back-reference in the dataset's `metadata.json` instead of duplicating per-variant arrays, so the source store must remain accessible when the dataset is later opened with [`gvl.Dataset.open()`](api.md#genvarloader.Dataset.open) (override its location with `svar=`/`svar2=` if it has moved). + +`.svar2` additionally produces a small write-time cache under `/genotypes/svar2_ranges/` and reads back through an all-Rust, read-bound path with no interval-search-tree build and no dense-union rebuild per read — see [the FAQ](faq.md) for the read-path and on-disk-size tradeoffs, and [the format reference](format.md) for the on-disk layout. `.svar2` currently has a Phase-1 scope: a handful of output combinations (splicing, `annotated` haplotypes, `min_af`/`max_af`, etc.) aren't wired yet and raise `NotImplementedError` — see the `genvarloader` skill or `format.md` for the full list. \ No newline at end of file diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md new file mode 100644 index 00000000..61492a4f --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md @@ -0,0 +1,147 @@ +# SVAR2 vs SVAR1 gvl Benchmark (chr21) + +**Date:** 2026-07-03 +**Task:** Plan Task 4 (Task E) — measure SVAR1 (gvl `Dataset` over `.svar`) vs SVAR2 +(`SparseVar2Source` over `.svar2`) on hap latency, variant latency, and store size, for +the germline + somatic chr21 stores built in Task 3. + +**Script:** `tmp/svar2_mvp/benchmark.py`. **Env:** `pixi run -e default`, genoray 2.15.0. + +## Workload (fairness rule) + +Same workload on both backends: **all samples for a fixed region set**, matching genoray's +per-contig/all-samples query granularity. Warm caches (1 warmup), **median of N=5** repeats. + +- **Regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, + `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)` — 3 regions, 2.5 kb total. +- **Samples:** germline 3202, somatic 16007 (all samples, both backends). +- Both backends query the **same 3 regions × all samples**. (`benchmark.py` was corrected + from the plan draft, which opened `validate.py`'s leftover 2-region `.gvl` while the + SVAR2 path queried 3 regions — an unfair mismatch; it now `gvl.write`s the 3-region BED + first so both measure an identical workload.) + +## Results (median seconds; store size = apparent bytes, `du -sb`) + +| Source (samples) | Metric | SVAR1 | SVAR2 | Outcome | +| --- | --- | --- | --- | --- | +| **germline** (3202) | hap latency (s) | 0.0555 | 0.2834 | SVAR1 5.1× faster | +| | var latency (s)† | 0.0280 | 0.2270 | — (different ops, see below) | +| | store size | 1,149,533,941 (1.15 GB) | 202,842,586 (203 MB) | **SVAR2 5.67× smaller** | +| **somatic** (16007) | hap latency (s) | 0.3798 | 0.3797 | **≈ parity** | +| | var latency (s)† | 0.0518 | 0.1088 | — (different ops, see below) | +| | store size | 55,578,073 (55.6 MB) | 38,184,053 (38.2 MB) | **SVAR2 1.46× smaller** | + +Store-size labels are decimal (MB = 10⁶ B, GB = 10⁹ B) computed from the apparent byte +counts; all ratios below are computed from the byte counts, not the rounded labels. + +**† The two "var latency" columns measure different operations and are NOT a like-for-like +decode comparison.** SVAR2 var = `sv2.decode(chrom, regions)` (genoray raw variant-record +decode → `Ragged`); SVAR1 var = `ds.with_seqs("variants")[:regions, :n_s]` (gvl +variant-**sequence** materialization). They are reported side by side only because each is +the natural "variants" call for its backend — do not read them as one backend decoding +"the same thing" faster. + +## Reading the numbers + +**Store size — SVAR2 wins outright, most on high-AF cohorts.** 5.67× smaller on germline +(1000G, many common/high-allele-frequency variants routed to SVAR2's 1-bit dense matrix) +and 1.46× smaller on somatic (rare/near-private mutations stay sparse in both formats). +This is exactly the empirical short-read distribution SVAR2 targets. + +**Latency — SVAR1 leads on the small high-density cohort; the two are at parity on the +large one.** Between the two cohorts: + +| Backend | hap latency germline (3202) → somatic (16007) | +| --- | --- | +| SVAR1 | 0.0555 → 0.3798 s | +| SVAR2 | 0.2834 → 0.3797 s | + +⚠️ **This is NOT a controlled sample-count sweep — the two points are different datasets** +(1000G germline, high-AF/high-density, vs GDC somatic, rare/sparse), differing in allele +frequency, variant density, and source, not just in sample count. So the movement cannot be +attributed to cohort size alone: germline's high variant density is a large part of why +SVAR2's germline hap latency (0.283 s) is high at only 3202 samples. What the data *does* +show is a real, useful fact — **on the large 16007-sample somatic cohort the un-optimized +SVAR2 adapter already matches the mature SVAR1 Dataset on hap latency** (0.3797 vs 0.3798 s; +median of N=5, no dispersion/CI measured, so treat "parity" as "indistinguishable at this +resolution"). The *hypothesis* that SVAR2's latency is less sample-count-sensitive is +plausible and worth testing, but proving it needs a **same-cohort subsampling sweep** +(fix the dataset, vary S) — not this germline-vs-somatic pair. The var-latency columns are +different operations (†) and are not compared here. + +## Caveat — this is two different code paths, NOT profiled + +The latency columns compare the mature **SVAR1 gvl `Dataset`** path against the **SVAR2 +`SparseVar2Source` adapter** — and the SVAR2 path here is **not the Dataset at all**. It is +a new Python adapter that calls genoray `overlap_batch` live and runs the two-source Rust +kernel per call, with no Dataset-level batching/collation/caching (Task B, deferred). So: + +- We make **no claim about *why* the latencies differ.** We have not run `perf` / a Rust + profiler on either path, so attributing the gap to "pre-materialization", "live decode", + memory layout, or Python overhead would be speculation. Any "X is slow" statement is out + of scope until profiled. +- **Prior gvl profiling does not transfer.** Earlier work found gvl `Dataset` latency was + dominated by the Rust core with negligible Python-orchestration overhead — but that was + measured on the `Dataset` path. The benchmarked SVAR2 path is the **raw adapter**, which + is *new Python orchestration* (per-call genoray query + numpy conversions) that has never + been profiled; its Python overhead on a small 3-region workload is unknown and could be a + material fraction of the 0.28–0.38 s. +- **The size table compares variant *stores* (`.svar` vs `.svar2`), not query artifacts.** + SVAR1's low latency is served from an additional pre-materialized `.gvl` Dataset that + `gvl.write` produces (here only the 3 benchmark regions: germline `.gvl` 292 KB, somatic + `.gvl` 1.6 MB — tiny because it holds just those regions, not the contig). The + `.svar`-vs-`.svar2` size comparison is the correct *variant-source* comparison; just note + SVAR1 additionally materializes a `.gvl` to reach its quoted latency, and a fully-wired + SVAR2 (Task B) would likewise gain a Dataset artifact. + +### What we *can* say about layout (checked in source, not profiled) + +The user hypothesized the SVAR2 dense matrix is variant-major `(V, S, P)` — pessimal for +gvl's access pattern (contiguous variant slices for random `(sample, ploid)`). **Checked and +refuted:** the dense genotype/presence matrix is **hap-major**, variant *innermost* — bit +index `hap * n_dense_variants + col` (`genoray:src/query.rs:131-134`, matching +`data-model.md`'s "hap-major `(sample, ploid, variant)`, variant fastest-varying"). During +build the in-memory chunk is variant-major (`BitGrid3(v,s,p)`) and `dense_merge` +bit-**transposes** it to hap-major on disk (roadmap M4). So a *single* hap's dense variants +are contiguous — the good case for per-hap reconstruction. + +A **related, real** concern remains (also unprofiled): `n_dense_variants` is the **whole +contig's** dense count, so a narrow-region × all-samples query reads one short contiguous +run per hap at stride `n_dense_variants` → a separate cache line per haplotype, scattered +across a contig-wide matrix; and presence is read **bit-by-bit** (`get_bit` per column), not +word-parallel. For germline (many common → dense variants → large `n_dense_variants`) that +scatter is worse, which is *directionally consistent* with germline's higher SVAR2 hap +latency — but this is a hypothesis for the profiler, not a conclusion, and it is +**contig-scoped**, so it ties into the split-by-contig question below. + +## Limitations / open questions (before strong claims or Task B) + +1. **Profile before concluding.** Run `perf` + a Rust profiler on both hap paths; separately + measure the SVAR2 adapter's Python overhead vs its Rust kernel. Only then attribute the + latency gap to a cause. +2. **Same-cohort sample sweep.** The germline↔somatic pair is confounded (different AF / + density / source). To claim SVAR2 scales better with S, subsample one cohort and vary S + with the dataset fixed. +3. **Single contig — split-by-contig layout unassessed.** All measurements are chr21 only. + The contig-wide dense stride above and genoray's per-contig on-disk partition (M3) can + only be evaluated multi-contig; defer that analysis. +4. **Build: reading dominates; single-contig underuses threads.** The conversion logs show + the pipeline reserving `1 concurrent chromosome | 3 HTSlib decompression threads (7 of 8 + active, 1 idle)` — for a single-contig job most cores sit idle and VCF read/decompress is + plainly the bottleneck (germline ~11 min, somatic ~2 h). Worth exploring a thread split + that dedicates ~1 thread to the executor+writer and the rest to VCF decompress+read when + few contigs are present (genoray conversion-pipeline tuning, not gvl). + +## Value proposition & Task B signal + +- **SVAR2 is a clear, unconfounded win on disk footprint** (1.46× somatic, 5.67× germline; + larger where variants are common/high-AF). This alone is a strong reason to pursue Task B. +- **On the large 16k-sample cohort, the un-optimized SVAR2 adapter already matches SVAR1 on + hap latency** — so wiring SVAR2 into the Dataset (Task B) does not start from a latency + hole at scale. It is *not* established that SVAR2 "scales better with cohort size" (see + Limitation 2). +- **Recommendation:** pursue Task B for the storage win, but gate any latency claims on the + profiling in Limitation 1 first — a real chance the current SVAR2 adapter latency is + Python-adapter overhead the Dataset path would erase, and an equally real chance the + contig-wide dense access needs a layout fix. Profile, then decide how much layout work + Task B should include. diff --git a/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md new file mode 100644 index 00000000..343e7d18 --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md @@ -0,0 +1,125 @@ +# SVAR2 gvl MVP — Real-Data Validation (chr21) + +**Date:** 2026-07-03 +**Task:** Plan Task 3 (Task D) — prove the real-data plumbing works through both the +SVAR1 (gvl `Dataset` over `.svar`) and SVAR2 (`SparseVar2Source` over `.svar2`) backends +on real germline + somatic chr21 stores. Correctness is already proven by the test suite +(Task 2 e2e oracle); this exercises the real-data path end-to-end. + +**Env:** `pixi run -e default`, genoray 2.15.0 wheel. Reference FASTA +`/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. +**Work dir (outside repo):** `/carter/users/dlaub/repos/for_loukik/svar2_mvp` (`$W`). + +## Resolved contig names (Step 1) + +All three inputs use `chr21` — **no naming mismatch**: + +| File | Contig | +| --- | --- | +| Reference FASTA (`GRCh38.d1.vd1.fa`) | `chr21` | +| `chr21.bcf` (germline, 1000G) | `chr21` | +| `gdc.chr21.bcf` (somatic, GDC) | `chr21` | + +So `` = `` = `chr21` throughout. The germline `.csi` was +already present (no re-index needed). + +## Cohort sizes + +| Source | Samples | Ploidy | Raw records (chr21) | +| --- | --- | --- | --- | +| germline (1000G) | 3202 | 2 | 1,002,753 | +| somatic (GDC) | 16007 | 2 | 4,525,689 | + +## Build pipeline (Steps 2–3) + +Per source: `bcftools norm -m -any --atomize` → **filter symbolic/breakend** → build both +stores (`build_stores.py`: `SparseVar.from_vcf` for `.svar`, `run_conversion_pipeline` for +`.svar2`). Orchestrated by `$W/build_source.sh` (kept outside the repo), submitted via +`sbatch -p carter-compute` (germline 32G, somatic 128G, 8 cpus each). + +### Two environment/data issues found and fixed (API/infra drift) + +1. **Compute-node `libstdc++` / GLIBCXX shadowing.** On the compute nodes the inherited + `LD_LIBRARY_PATH` puts the gcc11 module's `libstdc++.so.6` (which lacks + `GLIBCXX_3.4.30`) ahead of the pixi env's newer one, so importing genoray crashed at + `llvmlite → numba` load: + `OSError: .../gcc11/.../libstdc++.so.6: version 'GLIBCXX_3.4.30' not found (required by + .../libLLVM-14.so)`. The login node is unaffected (its `miniforge3/lib` on + `LD_LIBRARY_PATH` rescues it). **Fix:** prepend the pixi env lib dir to + `LD_LIBRARY_PATH` in the job so the env's own `libstdc++` (which has `GLIBCXX_3.4.30`) + wins the loader search. Baked into `build_source.sh`. + +2. **SVAR2 rejects symbolic/breakend ALTs (short-read only).** genoray's + `normalize::atomize_record` returns `SymbolicAllele` for symbolic (``, + ``, ``, ``, …) and breakend ALTs, and the reader `.expect()`s on + it — so a single SV record **panics the whole conversion** + (`worker thread 'read-chr21' panicked`). `*`/`.` alleles are silently skipped, not + errored. **Fix:** drop symbolic ("other") + breakend ("bnd") types with + `bcftools view -V other,bnd` before building, and — for a **fair** benchmark — build + *both* stores from the identical filtered input. Filtering impact: + + | Source | Records before | Records after filter | Symbolic/bnd dropped | + | --- | --- | --- | --- | + | germline | 1,002,753 | 1,001,385 | **1,368** (702 ``, 260 ``, 195 ``, 174 ``, 17 ``, 12 ``, 7 ``, 1 ``) | + | somatic | 4,525,689 | 4,525,689 | **0** (GDC somatic calls are SNV/indel only) | + + This friction motivated a new roadmap milestone on the genoray side — **M13: opt-in + skip for out-of-scope alleles during conversion** (drop instead of panic), in + `genoray:docs/roadmap/svar-2.md`. + +Build wall-clock: germline ~11 min; somatic ~2h1m (peak RSS ~10.9 GB). + +## Backend validation (Step 4) — `validate.py`, 2 regions × chr21 + +Regions (0-based half-open): `(20_000_000, 20_001_000)` [1000 bp] and +`(30_000_000, 30_000_500)` [500 bp]. `validate.py` ran **as-written — no API drift** in the +`SparseVar2Source.reconstruct` / `sv.decode` / `gvl.write` / `gvl.Dataset.open` signatures. + +### germline (`$W/germline`) + +- **SVAR2** (`SparseVar2Source`): `n_samples=3202 ploidy=2`; hap ragged + `rows=12808` (= 2 regions × 3202 samples × 2 ploidy ✓), `min_len=499 max_len=1000` + (1000 bp window full-length; the 499 is a single-base net deletion) — sane. `decode` + returned a non-empty `seqpro.rag.Ragged`. +- **SVAR1** (gvl `Dataset` over `.svar`): dataset written (3202 samples, 2 regions) and + opened cleanly; `with_seqs("haplotypes")` returned a `seqpro.rag.Ragged`. + +### somatic (`$W/somatic`) + +- **SVAR2**: `n_samples=16007 ploidy=2`; hap ragged `rows=64028` (= 2 × 16007 × 2 ✓), + `min_len=498 max_len=1001` (a −2 DEL and a +1 INS at the extremes) — sane. `decode` + returned a non-empty `Ragged`. +- **SVAR1**: dataset written (16007 samples, 2 regions) and opened; haplotypes `Ragged`. + +Both backends return non-empty, correctly-shaped haplotypes and variants on real data for +both cohorts. + +## Store sizes (Step 5) + +`du` on the four stores: + +Apparent sizes (`du -sb`); decimal labels (MB = 10⁶ B, GB = 10⁹ B): + +| Store | SVAR1 (`.svar`) | SVAR2 (`.svar2`) | SVAR2 / SVAR1 | SVAR2 advantage | +| --- | --- | --- | --- | --- | +| germline (3202 smp) | 1,149,533,941 B (1.15 GB) | 202,842,586 B (203 MB) | 0.176× | **5.67× smaller** | +| somatic (16007 smp) | 55,578,073 B (55.6 MB) | 38,184,053 B (38.2 MB) | 0.687× | **1.46× smaller** | + +SVAR2 wins on size for **both** cohorts. The germline win is dramatic (5.7×): 1000G +carries many **common, high-allele-frequency** variants, which SVAR2's cost model routes +to the 1-bit dense matrix — far cheaper than SVAR1's `u32` pointers. Somatic mutations are +**rare / near-private** (low AF), so they stay sparse in both formats and the gap narrows +to 1.46×. This is exactly the empirical distribution SVAR2 was designed to exploit. + +## Artifacts + +- Store builder: `tmp/svar2_mvp/build_stores.py` (committed) +- Backend validator: `tmp/svar2_mvp/validate.py` (committed) +- Job orchestrator: `$W/build_source.sh` (outside repo — norm/filter/build wrapper with + the `LD_LIBRARY_PATH` fix; not committed) +- Four stores + normalized/filtered BCFs live under `$W` (outside the repo). + +## Feeds Task 4 (benchmark) + +The four stores are the benchmark inputs. Early size signal above; latency (hap/variant, +all-samples-per-region) is measured next. diff --git a/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md b/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md new file mode 100644 index 00000000..6bde899d --- /dev/null +++ b/docs/superpowers/notes/2026-07-03-svar2-profiling-results.md @@ -0,0 +1,20 @@ +# SVAR2 Profiling Results (E1–E4) + +**Date:** 2026-07-03 · **Spec:** `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` +**Node/env baseline:** see `tmp/svar2_mvp/env_baseline.txt`. +**Symbolization:** gvl + genoray rebuilt `--release` with `debug=line-tables-only` + frame pointers. + +## E1 — Query-latency attribution +_(filled by Task 3)_ + +## E2 — Same-cohort sample sweep +_(filled by Task 5)_ + +## E3 — Dense-access layout probe +_(filled by Task 6; conditional on E1)_ + +## E4 — Conversion thread-allocation +_(filled by Task 8)_ + +## Recommendations +_(filled by Task 9)_ diff --git a/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md b/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md new file mode 100644 index 00000000..cab59f9a --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-genoray-search-gather-split.md @@ -0,0 +1,953 @@ +# SVAR2 genoray `find_ranges` / `gather_ranges` / `read_ranges` Split — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split genoray's fused `SparseVar2.overlap_batch` into a *search-only* `find_ranges`, a *tree-free* `gather_ranges`, and a fused `read_ranges` wrapper — so a downstream cache can run the interval search once (at write time) and replay it at read time with no `SearchTree::build`. + +**Architecture:** Refactor the Rust `query::overlap_batch` into two pure functions sharing the region-independent dense union: `find_ranges` (runs every `SearchTree::new` and returns a compact `RangesBundle` of index ranges) and `gather_ranges` (consumes the bundle, does pure slicing + `carried` tests + k-way merge, no trees). Expose all three on `PyContigReader`, then on the Python `SparseVar2` class with `samples=` subsetting and an `out=` streaming path on `find_ranges`. `read_ranges = gather_ranges(find_ranges(...))` is the parity oracle. + +**Tech Stack:** Rust (PyO3, ndarray, numpy crate), Python 3.10+, pixi, maturin, pytest, cargo test. + +**Repo:** `/carter/users/dlaub/projects/genoray` — branch off `svar-2`. This is a **separate deliverable** that must ship a wheel before the gvl wiring plan (`2026-07-03-svar2-gvl-dataset-wiring.md`) can consume it. + +## Global Constraints + +- **Byte-identical parity contract** (verbatim from spec): for any `contig, starts, ends, samples`, + `overlap_batch(...)` ≡ `read_ranges(...)` ≡ `gather_ranges(find_ranges(...))`, and the reconstruction + from any of them ≡ the genoray `decode` oracle, **field-for-field / byte-for-byte**. +- **`samples=None` subset** on all three public Python methods, matching every other `SparseVar` range method (`_find_starts_ends`, `read_ranges`): `None` → all samples; a list restricts which samples' offsets/payload are computed. Unknown samples raise `ValueError`. +- **`out=` streaming** on `find_ranges` only, mirroring `SparseVar._find_starts_ends(..., out=out)` — writes the bundle into caller-preallocated arrays so `gvl.write` can stream straight to a memmap. +- **`gather_ranges` performs ZERO interval search** — no `SearchTree::new` anywhere in its call graph. This is the entire point; a test asserts it. +- **Additive:** `overlap_batch` stays working and byte-unchanged (it may later be deprecated — maintainer's call, out of scope here). All existing genoray tests stay green. +- **Rust vs Python build:** `cargo test` compiles from source and needs no rebuild. Python tests import the compiled extension — **run `pixi run maturin develop --release` before any pytest that exercises new bindings**, or pytest imports the stale `.so`. +- **Conventional commits** (commitizen). Ensure prek hooks are installed before the first commit (`pixi run prek-install`). + +--- + +## File Structure + +**Rust (`src/`):** +- `src/query.rs` — add `RangesBundle` struct, `fn find_ranges`, `fn gather_ranges`, `fn read_ranges`. Refactor `overlap_batch`'s body to share `dense_union` + the inner gather loop with `gather_ranges` (DRY; `overlap_batch` becomes `gather_ranges(&reader, &find_ranges(reader, regions, None))` internally, or keeps its own body — see Task 3). +- `src/py_query_ranges.rs` — **new** `#[pymethods]` block on `PyContigReader` exposing `find_ranges` / `gather_ranges` / `read_ranges` as numpy-dict methods (mirrors `src/py_query_batch.rs`; a separate file keeps M6b's `overlap_batch` binding untouched, per the existing multiple-pymethods convention). +- `src/lib.rs` — register the new module (`mod py_query_ranges;`). + +**Python (`python/genoray/`):** +- `python/genoray/_svar2_batch.py` — add `find_ranges`, `gather_ranges`, `read_ranges` methods to `_BatchQueryMixin` (next to `overlap_batch`), each resolving `samples=` to column indices and delegating to the Rust `PyContigReader`. +- `python/genoray/_svar2.py` — no signature change; `SparseVar2` already mixes in `_BatchQueryMixin`. Confirm the new methods surface. + +**Rust tests (`tests/`):** +- `tests/test_ranges_split.rs` — **new** cargo integration test: `read_ranges` bundle ≡ `overlap_batch` field-for-field; `gather_ranges` is search-free. + +**Python tests (`tests/`):** +- `tests/test_svar2_ranges.py` — **new** pytest: Python `find_ranges`/`gather_ranges`/`read_ranges` parity vs `overlap_batch`, `samples=` subsetting, and `out=` streaming. + +**Docs:** +- `docs/roadmaps/` (genoray's own roadmap) + `CHANGELOG.md` — record the split. + +--- + +### Task 1: `RangesBundle` struct + `find_ranges` (search-only Rust core) + +**Files:** +- Modify: `src/query.rs` (add near `overlap_batch`, ~line 509) +- Test: `tests/test_ranges_split.rs` (create) + +**Interfaces:** +- Consumes: `ContigReader` (existing), `ContigReader::dense_union() -> DenseUnion` (existing, `src/query.rs:351`), `DenseUnion::overlap(qs, qe) -> (usize, usize)` (existing, `:284`), `SearchTree`/`overlap_range` (existing, `src/search.rs`), `ContigReader::vk_snp`/`vk_indel` column accessors (existing, used in `vk_slice` `:296`). +- Produces: + ```rust + pub struct RangesBundle { + pub n_regions: usize, + pub n_samples: usize, // number of SELECTED samples (subset-aware) + pub ploidy: usize, + pub region_starts: Vec, // (R) q_start per region — needed by gather's left-overlap re-check + pub dense_range: Vec<(usize, usize)>, // (R) [s,e) into the shared dense union + pub sample_cols: Vec, // (n_samples) selected slot -> original sample index + pub vk_snp_range: Vec<(usize, usize)>, // (R*H) absolute [start,end) into vk_snp packed positions/keys + pub vk_indel_range: Vec<(usize, usize)>, // (R*H) absolute [start,end) into vk_indel packed positions/keys + } + // H = n_samples * ploidy; row (r*H + h), h = selected_s*ploidy + p. + pub fn find_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, // original sample indices; None = all + ) -> RangesBundle; + ``` + +- [ ] **Step 1: Write the failing cargo test** + +Create `tests/test_ranges_split.rs`: + +```rust +//! SVAR2 search/gather split: find_ranges produces the index ranges that +//! gather_ranges replays into the same BatchResult overlap_batch returns. + +mod common; + +use common::{SynthRecord, build_contig}; +use genoray_core::query::{ContigReader, find_ranges, overlap_batch}; +use tempfile::tempdir; + +fn synth_reader(out: &std::path::Path) -> ContigReader { + let samples = ["S0", "S1"]; + let records = vec![ + SynthRecord { pos: 100, ref_allele: b"A", alts: vec![&b"C"[..]], gt: vec![1, 0, 0, 0] }, + SynthRecord { pos: 200, ref_allele: b"A", alts: vec![&b"AT"[..]], gt: vec![0, 1, 1, 1] }, + SynthRecord { pos: 300, ref_allele: b"AT", alts: vec![&b"A"[..]], gt: vec![1, 1, 0, 1] }, + ]; + build_contig(out, "chr1", &samples, 2, &records); + ContigReader::open(out.to_str().unwrap(), "chr1", 2, 2).unwrap() +} + +#[test] +fn test_find_ranges_dense_range_matches_overlap_batch() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let br = overlap_batch(&reader, ®ions); + let rb = find_ranges(&reader, ®ions, None); + + // Same per-region dense index ranges; H+1 vk_off implies R*H vk sub-ranges. + assert_eq!(rb.dense_range, br.dense_range); + assert_eq!(rb.n_regions, br.n_regions); + assert_eq!(rb.n_samples, br.n_samples); + assert_eq!(rb.ploidy, br.ploidy); + assert_eq!(rb.vk_snp_range.len(), regions.len() * br.n_samples * br.ploidy); + assert_eq!(rb.vk_indel_range.len(), regions.len() * br.n_samples * br.ploidy); + assert_eq!(rb.region_starts, vec![0u32, 250u32]); +} +``` + +- [ ] **Step 2: Run the test to verify it fails to compile** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: FAIL — `cannot find function find_ranges` / `RangesBundle` unresolved. + +- [ ] **Step 3: Implement `RangesBundle` + `find_ranges`** + +In `src/query.rs`, add above `overlap_batch` (adapting the search half of `overlap_batch` at `:509` and the `overlap_range` call from `vk_slice`/`gather_keys`). `find_ranges` runs every `SearchTree::new`; it must **not** gather keys or compute presence bits. + +```rust +/// Search-only half of the batch query: every `SearchTree::new` runs here, and +/// the result is a compact bundle of index ranges that `gather_ranges` replays +/// with no further search. Mirrors `SparseVar::_find_starts_ends`. +pub struct RangesBundle { + pub n_regions: usize, + pub n_samples: usize, + pub ploidy: usize, + pub region_starts: Vec, + pub dense_range: Vec<(usize, usize)>, + pub sample_cols: Vec, + pub vk_snp_range: Vec<(usize, usize)>, + pub vk_indel_range: Vec<(usize, usize)>, +} + +pub fn find_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, +) -> RangesBundle { + let ploidy = reader.ploidy; + let sample_cols: Vec = match samples { + Some(s) => s.to_vec(), + None => (0..reader.n_samples).collect(), + }; + let n_samples = sample_cols.len(); + let n_regions = regions.len(); + let h = n_samples * ploidy; + + // Region-independent union; `overlap` builds one SearchTree per region. + let dense = reader.dense_union(); + let dense_range: Vec<(usize, usize)> = + regions.iter().map(|&(qs, qe)| dense.overlap(qs, qe)).collect(); + let region_starts: Vec = regions.iter().map(|&(qs, _)| qs).collect(); + + let mut vk_snp_range = Vec::with_capacity(n_regions * h); + let mut vk_indel_range = Vec::with_capacity(n_regions * h); + for &(qs, qe) in regions { + for &orig_s in &sample_cols { + for p in 0..ploidy { + let col = orig_s * ploidy + p; + vk_snp_range.push(reader.vk_snp_overlap(col, qs, qe)); + vk_indel_range.push(reader.vk_indel_overlap(col, orig_s, p, qs, qe)); + } + } + } + + RangesBundle { + n_regions, + n_samples, + ploidy, + region_starts, + dense_range, + sample_cols, + vk_snp_range, + vk_indel_range, + } +} +``` + +Then add two search-only helpers on `impl ContigReader` (extract the `overlap_range` calls out of `vk_slice` at `:296`; return **absolute** `[o0+s_idx, o0+e_idx)` indices into the packed column so gather needs no column lookup): + +```rust +/// Absolute [start,end) into vk_snp's packed positions/keys for (col, region). +/// The SNP channel's search half (max_del = 0). No gather. +fn vk_snp_overlap(&self, col: usize, q_start: u32, q_end: u32) -> (usize, usize) { + let (o0, o1) = self.vk_snp.column(col); + let positions = &self.vk_snp.positions()[o0..o1]; + if positions.is_empty() { + return (o0, o0); + } + let v_ends: Vec = positions.iter().map(|&p| p + 1).collect(); + let tree = crate::search::SearchTree::new(positions); + let (s, e) = crate::search::overlap_range(&tree, &v_ends, 0, q_start, q_end); + (o0 + s, o0 + e) +} + +/// Absolute [start,end) into vk_indel's packed positions/keys for (col, region). +/// The indel channel's search half (per-column max_del bound). No gather. +fn vk_indel_overlap(&self, col: usize, sample: usize, p: usize, q_start: u32, q_end: u32) -> (usize, usize) { + let (o0, o1) = self.vk_indel.column(col); + let positions = &self.vk_indel.positions()[o0..o1]; + if positions.is_empty() { + return (o0, o0); + } + let keys = &as_u32(&self.vk_indel.keys)[o0..o1]; + let max_del = self.vk_indel_max_del[[sample, p]]; + let v_ends: Vec = positions + .iter() + .enumerate() + .map(|(i, &pos)| pos + 1 + rvk::deletion_len(keys[i])) + .collect(); + let tree = crate::search::SearchTree::new(positions); + let (s, e) = crate::search::overlap_range(&tree, &v_ends, max_del, q_start, q_end); + (o0 + s, o0 + e) +} +``` + +> Implementation note: `overlap_range` currently lives behind `spine::gather_keys` (`src/spine.rs:48`). Confirm `overlap_range` and `SearchTree` are `pub` in `src/search.rs` (they are used across modules already); if `vk_snp.column`/`.positions()`/`vk_indel.keys` are private to a sibling module, add `pub(crate)` accessors — do not widen further than needed. + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (1 test). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +pixi run prek-install +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add find_ranges search-only query core" +``` + +--- + +### Task 2: `gather_ranges` (tree-free Rust core) + +**Files:** +- Modify: `src/query.rs` +- Test: `tests/test_ranges_split.rs` (extend) + +**Interfaces:** +- Consumes: `RangesBundle` (Task 1), `ContigReader`, `ContigReader::dense_union()`, `spine::merge_keys` (existing, `src/spine.rs:63`), `KeyRef` (existing), `DenseTable::carried(hap, col)` (existing, used in `overlap_batch` at `:548`), `ContigReader::lut_arrays()` (existing, `:260`), `rvk::snp_code_to_key`/`unpack_snp_key_at`/`deletion_len` (existing). +- Produces: + ```rust + pub fn gather_ranges(reader: &ContigReader, rb: &RangesBundle) -> BatchResult; + ``` + Returns the **exact same `BatchResult`** shape `overlap_batch` returns (`vk`, `vk_off`, `dense`, `dense_range`, `dense_present`, `dense_present_off`, `n_regions`, `n_samples`, `ploidy`) — so all downstream numpy conversion and the SVAR2 kernels are unchanged. + +- [ ] **Step 1: Write the failing test (extend `tests/test_ranges_split.rs`)** + +```rust +use genoray_core::query::gather_ranges; + +#[test] +fn test_gather_ranges_reproduces_overlap_batch_field_for_field() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let got = gather_ranges(&reader, &find_ranges(&reader, ®ions, None)); + + assert_eq!(got.n_regions, oracle.n_regions); + assert_eq!(got.n_samples, oracle.n_samples); + assert_eq!(got.ploidy, oracle.ploidy); + assert_eq!(got.vk, oracle.vk); + assert_eq!(got.vk_off, oracle.vk_off); + assert_eq!(got.dense, oracle.dense); + assert_eq!(got.dense_range, oracle.dense_range); + assert_eq!(got.dense_present, oracle.dense_present); + assert_eq!(got.dense_present_off, oracle.dense_present_off); +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_gather_ranges_reproduces_overlap_batch_field_for_field` +Expected: FAIL — `cannot find function gather_ranges`. + +- [ ] **Step 3: Implement `gather_ranges`** + +Adapt the inner triple loop of `overlap_batch` (`src/query.rs:526-567`) to consume `rb` sub-ranges instead of building trees. The **only** change from `overlap_batch`'s body is that `vk_slice`'s two `spine::gather_keys` calls (which each build a `SearchTree`) are replaced by direct slices of the precomputed `rb.vk_snp_range` / `rb.vk_indel_range`, filtered by `carried` + the per-element left-overlap check `q_start < v_end`, then `merge_keys`. The dense-presence loop is copied verbatim (it never built a tree). + +```rust +/// Tree-free gather: replay a `RangesBundle` into the same `BatchResult` that +/// `overlap_batch` produces. Contains NO `SearchTree::new` — the search already +/// happened in `find_ranges`. +pub fn gather_ranges(reader: &ContigReader, rb: &RangesBundle) -> BatchResult { + let ploidy = rb.ploidy; + let n_samples = rb.n_samples; + let n_regions = rb.n_regions; + let hpr = n_samples * ploidy; // haps per region + + let dense = reader.dense_union(); + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_present: Vec = Vec::new(); + let mut dense_present_off: Vec = vec![0]; + + for r in 0..n_regions { + let qs = rb.region_starts[r]; + let (ds, de) = rb.dense_range[r]; + for si in 0..n_samples { + let orig_s = rb.sample_cols[si]; + for p in 0..ploidy { + let col = orig_s * ploidy + p; + let hap = col; + let row = r * hpr + si * ploidy + p; + + // --- var_key gather (no search) --- + let mut snp_run: Vec = Vec::new(); + let (ss, se) = rb.vk_snp_range[row]; + { + let positions = self_positions_snp(reader); + let keys = as_bytes(&reader.vk_snp.keys); + for i in ss..se { + // SNP v_end = pos + 1; left-overlap re-check. + if reader.vk_snp.carried_column_bit(col, i) && qs < positions[i] + 1 { + snp_run.push(KeyRef { + position: positions[i], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, i)), + }); + } + } + } + let mut indel_run: Vec = Vec::new(); + let (is_, ie_) = rb.vk_indel_range[row]; + { + let positions = reader.vk_indel.positions(); + let keys = as_u32(&reader.vk_indel.keys); + for i in is_..ie_ { + let v_end = positions[i] + 1 + rvk::deletion_len(keys[i]); + if reader.vk_indel.carried_column_bit(col, i) && qs < v_end { + indel_run.push(KeyRef { position: positions[i], key: keys[i] }); + } + } + } + let merged = spine::merge_keys(vec![snp_run, indel_run]); + vk.extend_from_slice(&merged); + vk_off.push(vk.len()); + + // --- dense presence bits (verbatim from overlap_batch) --- + let nbits = de - ds; + let bit_base = *dense_present_off.last().unwrap(); + let need_bytes = (bit_base + nbits).div_ceil(8); + if dense_present.len() < need_bytes { + dense_present.resize(need_bytes, 0); + } + for (k, j) in (ds..de).enumerate() { + let (is_indel, dcol) = dense.src[j]; + let carried = if is_indel { + reader.dense_indel.as_ref().expect("indel src implies table").carried(hap, dcol) + } else { + reader.dense_snp.as_ref().expect("snp src implies table").carried(hap, dcol) + }; + if carried && dense.v_ends[j] > qs { + bits::set_bit(&mut dense_present, bit_base + k); + } + } + dense_present_off.push(bit_base + nbits); + } + } + } + + BatchResult { + n_regions, + n_samples, + ploidy, + vk, + vk_off, + dense: dense.refs, + dense_range: rb.dense_range.clone(), + dense_present, + dense_present_off, + } +} +``` + +> Implementation note: the exact per-element `carried` accessor for a **var_key** column (`carried_column_bit`) and the `positions`/`keys` slice accessors are genoray-internal. The existing `vk_slice` (`:296`) reaches them via `spine::gather_keys`'s `carried: impl Fn(usize) -> bool` closure — in the current code that closure is `|_| true` (var_key channel carries every stored key by construction; the presence filter is the dense channel's job). **Verify this**: if `vk_slice` passes `|_| true`, then the var_key gather needs no per-element carried test at all — drop `carried_column_bit` and keep only the `qs < v_end` left-overlap re-check. Let the field-for-field parity test (Step 1) pin the correct behavior; do not guess — match `overlap_batch` byte-for-byte. Replace the `self_positions_snp(reader)` placeholder with the real `reader.vk_snp.positions()` accessor once confirmed `pub(crate)`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (both tests). If the field-for-field test fails, the divergence is in the var_key `carried`/left-overlap handling — reconcile against `vk_slice`/`gather_keys` until byte-identical. + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add gather_ranges tree-free query core" +``` + +--- + +### Task 3: `read_ranges` fused wrapper + search-free assertion + +**Files:** +- Modify: `src/query.rs` +- Test: `tests/test_ranges_split.rs` (extend) + +**Interfaces:** +- Consumes: `find_ranges` (Task 1), `gather_ranges` (Task 2). +- Produces: `pub fn read_ranges(reader: &ContigReader, regions: &[(u32, u32)], samples: Option<&[usize]>) -> BatchResult;` + +- [ ] **Step 1: Write the failing test** + +```rust +use genoray_core::query::read_ranges; + +#[test] +fn test_read_ranges_equals_overlap_batch() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let got = read_ranges(&reader, ®ions, None); + assert_eq!(got.vk, oracle.vk); + assert_eq!(got.vk_off, oracle.vk_off); + assert_eq!(got.dense_present, oracle.dense_present); + assert_eq!(got.dense_present_off, oracle.dense_present_off); + assert_eq!(got.dense_range, oracle.dense_range); +} + +// Subset parity: read_ranges over a sample subset equals the corresponding +// hap-rows of the full overlap_batch. For samples=[1] (original index 1), +// region r's hap rows are r*H + [ploidy .. 2*ploidy) of the full result. +#[test] +fn test_read_ranges_sample_subset_matches_full() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 400u32)]; + + let full = overlap_batch(&reader, ®ions); + let sub = read_ranges(&reader, ®ions, Some(&[1])); + assert_eq!(sub.n_samples, 1); + // hap rows for sample 1 in the full result: h in [1*ploidy, 2*ploidy). + let ploidy = full.ploidy; + for p in 0..ploidy { + let full_h = 1 * ploidy + p; + let sub_h = 0 * ploidy + p; + assert_eq!( + &sub.vk[sub.vk_off[sub_h]..sub.vk_off[sub_h + 1]], + &full.vk[full.vk_off[full_h]..full.vk_off[full_h + 1]], + ); + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_read_ranges` +Expected: FAIL — `cannot find function read_ranges`. + +- [ ] **Step 3: Implement `read_ranges`** + +```rust +/// Fused search+gather: the public/live-query analog of `SparseVar::read_ranges` +/// and the parity oracle for the split. Byte-identical to `overlap_batch` for +/// `samples = None`. +pub fn read_ranges( + reader: &ContigReader, + regions: &[(u32, u32)], + samples: Option<&[usize]>, +) -> BatchResult { + gather_ranges(reader, &find_ranges(reader, regions, samples)) +} +``` + +Optionally (DRY, keep `overlap_batch` byte-identical): leave `overlap_batch` as-is — do **not** re-route it through the split in this task, to avoid perturbing the existing oracle while the split stabilizes. A follow-up may collapse them once parity is locked. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split` +Expected: PASS (all Task 1–3 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/query.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): add read_ranges fused wrapper + subset parity" +``` + +--- + +### Task 4: PyO3 bindings — `find_ranges` / `gather_ranges` / `read_ranges` on `PyContigReader` + +**Files:** +- Create: `src/py_query_ranges.rs` +- Modify: `src/lib.rs` (add `mod py_query_ranges;`) +- Test: `tests/test_ranges_split.rs` (add a binding round-trip test mirroring `tests/test_batch_raw.rs`) + +**Interfaces:** +- Consumes: `PyContigReader` (existing, `src/py_query.rs:12`), `find_ranges`/`gather_ranges`/`read_ranges` (Tasks 1–3), the numpy helpers `u8_to_pyarray`/`u32_to_i32_pyarray`/`usize_to_i64_pyarray` (existing, `src/py_convert.rs`). +- Produces on `PyContigReader`: + - `read_ranges(regions, samples=None) -> PyDict` — the **same key/dtype contract** as `overlap_batch` (`vk_pos`, `vk_key`, `vk_off`, `dense_pos`, `dense_key`, `dense_range`, `dense_present`, `dense_present_off`, `lut_bytes`, `lut_off`, `n_regions`, `n_samples`, `ploidy`). + - `find_ranges(regions, samples=None, out=None) -> PyDict` — the compact bundle: `dense_range (R,2) i32`, `region_starts (R) i32`, `sample_cols (n_samples) i64`, `vk_snp_range (R*H,2) i64`, `vk_indel_range (R*H,2) i64`, `n_regions`, `n_samples`, `ploidy`. `out` (optional dict of preallocated arrays) receives the ranges in place. + - `gather_ranges(bundle: PyDict, samples=None) -> PyDict` — same output contract as `read_ranges`; `bundle` is a `find_ranges` dict. + +- [ ] **Step 1: Write the failing binding test** + +Add to `tests/test_ranges_split.rs` (uses `Python::with_gil`, mirrors `tests/test_batch_raw.rs`): + +```rust +use genoray_core::py_query::PyContigReader; +use numpy::{PyArray1, PyArrayMethods}; +use pyo3::prelude::*; +use pyo3::types::PyDictMethods; + +#[test] +fn test_py_read_ranges_dict_matches_overlap_batch_dict() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let _reader = synth_reader(&out); + let base = out.to_str().unwrap().to_string(); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + Python::with_gil(|py| { + let pr = PyContigReader::new(&base, "chr1", 2, 2).unwrap(); + let d_ob = pr.overlap_batch(py, regions.clone()).unwrap(); + let d_rr = pr.read_ranges(py, regions.clone(), None).unwrap(); + for k in ["vk_pos", "vk_key", "dense_pos", "dense_key"] { + let a = d_ob.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + let b = d_rr.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + assert_eq!(a.as_slice().unwrap(), b.as_slice().unwrap(), "key {k}"); + } + }); +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test --test test_ranges_split test_py_read_ranges_dict_matches_overlap_batch_dict` +Expected: FAIL — no method `read_ranges` on `PyContigReader`. + +- [ ] **Step 3: Implement the bindings** + +Create `src/py_query_ranges.rs` (mirror `src/py_query_batch.rs` for the `read_ranges`/`gather_ranges` output dict; add the compact `find_ranges` dict + `out=` streaming): + +```rust +//! SVAR2 search/gather split: numpy-dict bindings on `PyContigReader`. +//! Separate #[pymethods] block (multiple-pymethods) so the M6b overlap_batch +//! binding in py_query_batch.rs is untouched. + +use ndarray::Array2; +use numpy::{PyArray1, PyArray2, PyArrayMethods, ToPyArray}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::py_convert::{u8_to_pyarray, u32_to_i32_pyarray, usize_to_i64_pyarray}; +use crate::py_query::PyContigReader; +use crate::query::{find_ranges, gather_ranges, read_ranges, BatchResult, RangesBundle}; + +fn batch_result_to_dict<'py>( + py: Python<'py>, + reader_lut: (Vec, Vec), + br: &BatchResult, +) -> PyResult> { + // Identical to py_query_batch.rs::overlap_batch's dict assembly. + let vk_pos: Vec = br.vk.iter().map(|k| k.position).collect(); + let vk_key: Vec = br.vk.iter().map(|k| k.key).collect(); + let dense_pos: Vec = br.dense.iter().map(|k| k.position).collect(); + let dense_key: Vec = br.dense.iter().map(|k| k.key).collect(); + let r = br.dense_range.len(); + let mut dr: Vec = Vec::with_capacity(r * 2); + for &(s, e) in &br.dense_range { dr.push(s as i32); dr.push(e as i32); } + let dense_range = Array2::from_shape_vec((r, 2), dr).expect("dense_range").to_pyarray(py); + let (lut_bytes, lut_off_u64) = reader_lut; + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let d = PyDict::new(py); + d.set_item("vk_pos", u32_to_i32_pyarray(py, &vk_pos))?; + d.set_item("vk_key", u32_to_i32_pyarray(py, &vk_key))?; + d.set_item("vk_off", usize_to_i64_pyarray(py, &br.vk_off))?; + d.set_item("dense_pos", u32_to_i32_pyarray(py, &dense_pos))?; + d.set_item("dense_key", u32_to_i32_pyarray(py, &dense_key))?; + d.set_item("dense_range", dense_range)?; + d.set_item("dense_present", u8_to_pyarray(py, &br.dense_present))?; + d.set_item("dense_present_off", usize_to_i64_pyarray(py, &br.dense_present_off))?; + d.set_item("lut_bytes", u8_to_pyarray(py, &lut_bytes))?; + d.set_item("lut_off", PyArray1::from_slice(py, &lut_off))?; + d.set_item("n_regions", br.n_regions)?; + d.set_item("n_samples", br.n_samples)?; + d.set_item("ploidy", br.ploidy)?; + Ok(d) +} + +fn bundle_to_dict<'py>(py: Python<'py>, rb: &RangesBundle) -> PyResult> { + let pairs2 = |v: &[(usize, usize)]| -> Vec { + let mut o = Vec::with_capacity(v.len() * 2); + for &(a, b) in v { o.push(a as i64); o.push(b as i64); } + o + }; + let dr: Vec = rb.dense_range.iter().flat_map(|&(a, b)| [a as i32, b as i32]).collect(); + let d = PyDict::new(py); + d.set_item("dense_range", Array2::from_shape_vec((rb.n_regions, 2), dr).unwrap().to_pyarray(py))?; + d.set_item("region_starts", u32_to_i32_pyarray(py, &rb.region_starts))?; + d.set_item("sample_cols", usize_to_i64_pyarray(py, &rb.sample_cols))?; + let h = rb.n_samples * rb.ploidy; + d.set_item("vk_snp_range", + Array2::from_shape_vec((rb.n_regions * h, 2), pairs2(&rb.vk_snp_range)).unwrap().to_pyarray(py))?; + d.set_item("vk_indel_range", + Array2::from_shape_vec((rb.n_regions * h, 2), pairs2(&rb.vk_indel_range)).unwrap().to_pyarray(py))?; + d.set_item("n_regions", rb.n_regions)?; + d.set_item("n_samples", rb.n_samples)?; + d.set_item("ploidy", rb.ploidy)?; + Ok(d) +} + +fn bundle_from_dict(py: Python<'_>, d: &Bound<'_, PyDict>) -> RangesBundle { + let get_i64 = |k: &str| -> Vec { + d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly().as_slice().unwrap().to_vec() + }; + let get_i32 = |k: &str| -> Vec { + d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly().as_slice().unwrap().to_vec() + }; + let get_i32_2d = |k: &str| -> Vec<(usize, usize)> { + let a = d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + a.as_array().rows().into_iter().map(|r| (r[0] as usize, r[1] as usize)).collect() + }; + let get_i64_2d = |k: &str| -> Vec<(usize, usize)> { + let a = d.get_item(k).unwrap().unwrap().cast::>().unwrap().readonly(); + a.as_array().rows().into_iter().map(|r| (r[0] as usize, r[1] as usize)).collect() + }; + let n_regions = d.get_item("n_regions").unwrap().unwrap().extract().unwrap(); + let n_samples = d.get_item("n_samples").unwrap().unwrap().extract().unwrap(); + let ploidy = d.get_item("ploidy").unwrap().unwrap().extract().unwrap(); + RangesBundle { + n_regions, n_samples, ploidy, + region_starts: get_i32("region_starts").into_iter().map(|x| x as u32).collect(), + dense_range: get_i32_2d("dense_range"), + sample_cols: get_i64("sample_cols").into_iter().map(|x| x as usize).collect(), + vk_snp_range: get_i64_2d("vk_snp_range"), + vk_indel_range: get_i64_2d("vk_indel_range"), + } +} + +#[pymethods] +impl PyContigReader { + pub fn read_ranges<'py>(&self, py: Python<'py>, regions: Vec<(u32, u32)>, samples: Option>) -> PyResult> { + let br = read_ranges(&self.inner, ®ions, samples.as_deref()); + batch_result_to_dict(py, self.inner.lut_arrays(), &br) + } + pub fn find_ranges<'py>(&self, py: Python<'py>, regions: Vec<(u32, u32)>, samples: Option>) -> PyResult> { + let rb = find_ranges(&self.inner, ®ions, samples.as_deref()); + bundle_to_dict(py, &rb) + } + pub fn gather_ranges<'py>(&self, py: Python<'py>, bundle: Bound<'py, PyDict>) -> PyResult> { + let rb = bundle_from_dict(py, &bundle); + let br = gather_ranges(&self.inner, &rb); + batch_result_to_dict(py, self.inner.lut_arrays(), &br) + } +} +``` + +Add `mod py_query_ranges;` to `src/lib.rs` (next to `mod py_query_batch;`). + +> Implementation note: the `out=` streaming variant is deferred to the Python layer (Task 5) — `find_ranges` returns freshly-allocated numpy arrays here; the Python `SparseVar2.find_ranges(..., out=...)` copies them into a caller memmap. This keeps the Rust binding simple and matches how gvl actually uses it (write once). If profiling later shows the copy matters, push `out=` into Rust then. + +- [ ] **Step 4: Rebuild the extension and run tests** + +Run: +```bash +cd /carter/users/dlaub/projects/genoray +pixi run cargo test --test test_ranges_split +pixi run maturin develop --release +``` +Expected: cargo PASS; maturin builds cleanly. + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add src/py_query_ranges.rs src/lib.rs tests/test_ranges_split.rs +rtk git commit -m "feat(svar2): PyContigReader find/gather/read_ranges bindings" +``` + +--- + +### Task 5: Python `SparseVar2` methods with `samples=` and `out=` + +**Files:** +- Modify: `python/genoray/_svar2_batch.py` +- Test: `tests/test_svar2_ranges.py` (create) + +**Interfaces:** +- Consumes: `SparseVar2._readers[contig]` (existing `PyContigReader` per contig, `python/genoray/_svar2_batch.py:25`), `SparseVar2.samples`/`.ploidy`/`.n_samples` (existing, `_svar2.py:44-46`), the new `PyContigReader.read_ranges`/`find_ranges`/`gather_ranges` (Task 4). +- Produces on `SparseVar2` (via `_BatchQueryMixin`): + ```python + def find_ranges(self, contig, starts, ends, samples=None, out=None) -> dict[str, np.ndarray] + def gather_ranges(self, contig, ranges, samples=None) -> dict[str, np.ndarray] + def read_ranges(self, contig, starts, ends, samples=None) -> dict[str, np.ndarray] + ``` + `ranges` is a `find_ranges` dict. `read_ranges`'s output dict is the **exact same contract** as `overlap_batch`. + +- [ ] **Step 1: Write the failing pytest** + +Create `tests/test_svar2_ranges.py` (reuse the `svar2_store` fixture that `tests/test_svar2_batch.py` uses): + +```python +import numpy as np +from genoray import SparseVar2 + + +def _assert_dicts_equal(a: dict, b: dict, keys): + for k in keys: + np.testing.assert_array_equal(np.asarray(a[k]), np.asarray(b[k]), err_msg=k) + + +PAYLOAD_KEYS = [ + "vk_pos", "vk_key", "vk_off", "dense_pos", "dense_key", "dense_range", + "dense_present", "dense_present_off", "lut_bytes", "lut_off", +] + + +def test_read_ranges_matches_overlap_batch(svar2_store): + sv = SparseVar2(svar2_store) + starts, ends = [0, 5], [40, 20] + ob = sv.overlap_batch("chr1", list(zip(starts, ends))) + rr = sv.read_ranges("chr1", starts, ends) + _assert_dicts_equal(ob, rr, PAYLOAD_KEYS) + assert int(rr["n_regions"]) == 2 + + +def test_gather_of_find_matches_read(svar2_store): + sv = SparseVar2(svar2_store) + starts, ends = [0], [40] + ranges = sv.find_ranges("chr1", starts, ends) + gathered = sv.gather_ranges("chr1", ranges) + read = sv.read_ranges("chr1", starts, ends) + _assert_dicts_equal(read, gathered, PAYLOAD_KEYS) + + +def test_read_ranges_sample_subset(svar2_store): + sv = SparseVar2(svar2_store) + full = sv.overlap_batch("chr1", [(0, 40)]) + sub = sv.read_ranges("chr1", [0], [40], samples=[sv.samples[1]]) + assert int(sub["n_samples"]) == 1 + ploidy = sv.ploidy + for p in range(ploidy): + fh = 1 * ploidy + p + sh = 0 * ploidy + p + np.testing.assert_array_equal( + full["vk_pos"][full["vk_off"][fh]:full["vk_off"][fh + 1]], + sub["vk_pos"][sub["vk_off"][sh]:sub["vk_off"][sh + 1]], + ) + + +def test_find_ranges_out_streaming(svar2_store): + sv = SparseVar2(svar2_store) + ranges = sv.find_ranges("chr1", [0], [40]) + # Pre-allocate matching-shape buffers and stream into them. + out = {k: np.empty_like(np.asarray(ranges[k])) for k in + ("dense_range", "region_starts", "sample_cols", "vk_snp_range", "vk_indel_range")} + ranges2 = sv.find_ranges("chr1", [0], [40], out=out) + for k in out: + np.testing.assert_array_equal(np.asarray(ranges2[k]), np.asarray(ranges[k])) + # out= wrote in place: returned array shares the buffer. + assert np.asarray(ranges2[k]).base is out[k] or ranges2[k] is out[k] +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run pytest tests/test_svar2_ranges.py -q` +Expected: FAIL — `SparseVar2` has no attribute `read_ranges`. + +- [ ] **Step 3: Implement the Python methods** + +Add to `_BatchQueryMixin` in `python/genoray/_svar2_batch.py`. Resolve `samples=` names → original integer indices via `self.samples.index(...)`; validate membership. + +```python + def _sample_idxs(self, samples): + if samples is None: + return None + idxs = [] + for s in np.atleast_1d(np.asarray(samples)).tolist(): + if s not in self.samples: + raise ValueError(f"Sample {s!r} not found in the dataset.") + idxs.append(self.samples.index(s)) + return idxs + + def read_ranges(self, contig, starts, ends, samples=None): + """Fused search+gather query (byte-identical to ``overlap_batch`` for + ``samples=None``). See ``overlap_batch`` for the returned dict contract.""" + reg = self._regions(starts, ends) + return self._readers[contig].read_ranges(reg, self._sample_idxs(samples)) + + def find_ranges(self, contig, starts, ends, samples=None, out=None): + """Search-only step: returns the compact ranges bundle to be replayed by + ``gather_ranges``. When ``out`` is a dict of preallocated arrays keyed by + the bundle field names, the ranges are written into it in place.""" + reg = self._regions(starts, ends) + d = self._readers[contig].find_ranges(reg, self._sample_idxs(samples)) + if out is not None: + for k, buf in out.items(): + np.asarray(buf)[...] = np.asarray(d[k]) + d[k] = buf + return d + + def gather_ranges(self, contig, ranges, samples=None): + """Tree-free gather step: replay a ``find_ranges`` bundle into the full + ``overlap_batch`` payload dict. ``samples`` is accepted for symmetry but + the subset is already fixed by the bundle; passing a different subset is + a ValueError.""" + return self._readers[contig].gather_ranges(ranges) +``` + +Add the `_regions` helper (shared with `overlap_batch`, which currently inlines `[(int(s), int(e)) for s, e in regions]`): + +```python + @staticmethod + def _regions(starts, ends): + s = np.atleast_1d(np.asarray(starts)) + e = np.atleast_1d(np.asarray(ends)) + return [(int(a), int(b)) for a, b in zip(s, e)] +``` + +Import numpy at module top (`import numpy as np`) — currently only imported under `TYPE_CHECKING`; move it to a real import since the methods use it at runtime. + +- [ ] **Step 4: Rebuild + run** + +Run: +```bash +cd /carter/users/dlaub/projects/genoray +pixi run maturin develop --release +pixi run pytest tests/test_svar2_ranges.py -q +``` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add python/genoray/_svar2_batch.py tests/test_svar2_ranges.py +rtk git commit -m "feat(svar2): SparseVar2 find/gather/read_ranges with samples= and out=" +``` + +--- + +### Task 6: Reconstruction parity vs the `decode` oracle + +**Files:** +- Test: `tests/test_svar2_ranges.py` (extend) + +**Interfaces:** +- Consumes: `SparseVar2.decode(...)` (existing oracle, `_svar2.py`), the M6b→numpy contract, and the split methods (Task 5). This task adds no production code — it hardens the byte-identical contract end-to-end so the gvl plan can depend on it. + +- [ ] **Step 1: Write the failing/again-green oracle test** + +Extend `tests/test_svar2_ranges.py`. Mirror whatever reconstruction check `tests/test_svar2_batch.py` / `tests/test_svar2_decode.py` already use against `decode`; assert the `read_ranges` and `gather_ranges(find_ranges)` payloads reconstruct to the identical `decode` output. If `tests/test_svar2_decode.py` exposes a helper that turns an `overlap_batch` dict into per-hap calls, reuse it verbatim on the three payloads. + +```python +def test_split_reconstructs_like_decode_oracle(svar2_store): + from tests.test_svar2_decode import decode_from_payload # reuse existing helper + sv = SparseVar2(svar2_store) + starts, ends = [0], [40] + ob = sv.overlap_batch("chr1", list(zip(starts, ends))) + rr = sv.read_ranges("chr1", starts, ends) + gr = sv.gather_ranges("chr1", sv.find_ranges("chr1", starts, ends)) + oracle = sv.decode("chr1", starts[0], ends[0]) # adapt to real decode signature + for payload in (ob, rr, gr): + assert decode_from_payload(sv, payload) == oracle +``` + +> If no reusable `decode_from_payload` helper exists, this test degrades to the field-for-field payload equality already covered in Task 5 plus the Rust `test_gather_ranges_reproduces_overlap_batch_field_for_field` — in that case, delete this task's test and rely on those, since `overlap_batch` is itself the decode-validated reference (per `tests/test_batch_raw.rs`'s header comment). **Check `tests/test_svar2_decode.py` first**; do not invent a helper name. + +- [ ] **Step 2: Run** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run pytest tests/test_svar2_ranges.py tests/test_svar2_batch.py tests/test_svar2_decode.py -q` +Expected: PASS. + +- [ ] **Step 3: Full regression** + +Run: `cd /carter/users/dlaub/projects/genoray && pixi run cargo test && pixi run test` +Expected: all green (no existing test regressed by the additive methods). + +- [ ] **Step 4: Commit** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add tests/test_svar2_ranges.py +rtk git commit -m "test(svar2): search/gather split reconstructs like decode oracle" +``` + +--- + +### Task 7: Docs, roadmap, and wheel release + +**Files:** +- Modify: `CHANGELOG.md` (or let commitizen generate it) +- Modify: genoray roadmap doc (whichever tracks SVAR2 / M6b — grep `docs/` for `overlap_batch` / `M6b`) +- Modify: `python/genoray/_svar2.py` / `_svar2_batch.py` docstrings (already added in Task 5) + +**Interfaces:** +- Produces: a released genoray wheel/version that the gvl plan pins. + +- [ ] **Step 1: Update the roadmap** + +Find the genoray SVAR2 roadmap section (`rtk grep "overlap_batch" docs/` and `rtk grep "M6b" docs/`) and record: the search/gather split shipped (`find_ranges`/`gather_ranges`/`read_ranges` with `samples=`, `out=` on `find_ranges`), and note the **open question** the spec flags — whether to convert the read path to fully read-bound (dense union caching) — as a follow-up. + +- [ ] **Step 2: Verify the public API doc / API stubs** + +If genoray publishes a Sphinx `api.md`/autodoc or `.pyi` stubs for `SparseVar2`, add `find_ranges`/`gather_ranges`/`read_ranges`. Run `cd /carter/users/dlaub/projects/genoray && pixi run -e doc doc` if a docs env exists and confirm it builds. + +- [ ] **Step 3: Bump version + build the wheel** + +```bash +cd /carter/users/dlaub/projects/genoray +pixi run bump-dry # preview the version bump +# then perform the real bump per genoray's release process (commitizen) +pixi run maturin build --release +``` +Record the released version string — the gvl plan's Global Constraints pin `genoray >= `. + +- [ ] **Step 4: Final commit / tag** + +```bash +cd /carter/users/dlaub/projects/genoray +rtk git add -A +rtk git commit -m "docs(svar2): record find/gather/read_ranges split + roadmap follow-up" +``` + +--- + +## Self-Review + +- **Spec coverage (Component A):** `find_ranges` (Task 1), `gather_ranges` (Task 2), `read_ranges` (Task 3), `samples=` on all three (Tasks 1/3/5), `out=` on `find_ranges` (Tasks 4-note/5), PyO3 bindings (Task 4), Python surface (Task 5), byte-identical parity vs `overlap_batch` and `decode` (Tasks 2/3/6), docs+roadmap+wheel (Task 7). ✅ +- **Deferred correctly:** dense-union caching / fully read-bound conversion is logged as a genoray follow-up (Task 7), matching the spec's open question. The var_key `carried` semantics are pinned by the field-for-field parity test rather than guessed (Tasks 2-3). +- **Type consistency:** `RangesBundle` fields, `H = n_samples * ploidy`, and the row index `r*H + selected_s*ploidy + p` are used identically across `find_ranges`, `gather_ranges`, and both binding dicts. diff --git a/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md b/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md new file mode 100644 index 00000000..d200cb9c --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-gvl-dataset-wiring.md @@ -0,0 +1,941 @@ +# SVAR2 gvl Dataset Wiring — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the SVAR2 format into the gvl `Dataset` the way SVAR1 is — cache genoray's interval-search result at `gvl.write` time and replay it at read time via `gather_ranges` + the existing SVAR2 kernels — so SVAR2 haplotype/track reads stop paying a per-query `SearchTree::build`. + +**Architecture:** Mirror the SVAR1 write/read wiring. `_write_from_svar2` streams `SparseVar2.find_ranges(..., out=...)` into a compact per-region/per-hap ranges cache under `genotypes/`, plus a `Svar2Link` back-reference in `metadata.json`. On read, a new `HapsSvar2` reconstructor loads the cached ranges for the requested `(region, sample)` block, calls `SparseVar2.gather_ranges` (tree-free), and feeds the payload to `reconstruct_haplotypes_from_svar2` / `shift_and_realign_tracks_from_svar2`. A source discriminant (`svar2_link` present) selects `HapsSvar2` over the SVAR1 `Haps`. + +**Tech Stack:** Python 3.10+, numpy, polars, pydantic, seqpro `Ragged`, genoray `SparseVar2` (from the genoray split plan), pixi, maturin, pytest. + +**Repo:** `/carter/users/dlaub/projects/GenVarLoader` worktree `svar2-m6b-kernel`. **Depends on** the shipped genoray wheel from `2026-07-03-svar2-genoray-search-gather-split.md` (`find_ranges`/`gather_ranges`/`read_ranges` with `samples=`/`out=`). + +## Global Constraints + +- **Depends on genoray >= ``.** Before starting, confirm `pixi run -e dev python -c "from genoray import SparseVar2; SparseVar2.find_ranges; SparseVar2.gather_ranges"` succeeds; if not, bump the genoray pin in `pixi.toml` / `pyproject.toml` and `pixi run -e dev install`. +- **Byte-identical parity contract** (verbatim from spec): cached-path reconstruct ≡ live `read_ranges`/`overlap_batch` reconstruct ≡ `decode` oracle, on the M6b matrix (SNP/INS/DEL × samples × ploids) + real chr21 germline & somatic stores. Track re-alignment matched the same way. +- **Additive:** the SVAR1 path is **byte-unchanged**. The full SVAR1 regression suite stays green (`pixi run -e dev pytest tests -q`; `pixi run -e dev cargo-test` for kernels). Follows the rust-migration byte-identical parity contract and the numba-oracle-bug policy (if the cached path and a numba oracle disagree, check whether numba is the buggy one before "fixing" the new path). +- **Scope:** haplotypes + tracks **only**. SVAR2 `variants` and `annotated` output modes are out of scope (raise a clear `NotImplementedError`); the same cache extends to them later. +- **REBUILD RUST BEFORE PYTHON TESTS:** these changes are pure-Python (no `src/` edits), so `maturin develop` is *not* required for this plan — but if any parity test imports the SVAR2 kernels and you touched `src/`, run `pixi run -e dev maturin develop --release` first. +- **Docs gates:** any public-API change updates `skills/genvarloader/SKILL.md`; `api.md` stays in sync with `__all__`; user-facing docs audited (`README.md`, `docs/source/{api,write,format,faq}.md`). See CLAUDE.md's skill-maintenance + docs-audit rules. +- **Conventional commits** (commitizen). Ensure prek hooks installed before committing. + +--- + +## File Structure + +- `python/genvarloader/_dataset/_svar2_link.py` — **new**. `Svar2Fingerprint`, `Svar2Link` pydantic models, `_resolve_svar2`, `_verify_fingerprint2`. Mirrors `_svar_link.py`. +- `python/genvarloader/_dataset/_write.py` — **modify**. Add `.svar2` detection in the variant-source dispatch (~`:225`), a `SparseVar2` branch in the genotype-writing dispatch (~`:325`), and a new `_write_from_svar2(...)` (mirrors `_write_from_svar` at `:961`). Add `svar2_link` to the `Metadata` model (`:86`). +- `python/genvarloader/_dataset/_svar2_source.py` — **modify**. Add a cache-load + `gather_ranges` path; keep the live `overlap_batch` path as the parity oracle. +- `python/genvarloader/_dataset/_haps.py` — **modify**. Add `HapsSvar2` reconstructor (haplotypes-only, `Reconstructor[RaggedSeqs]`) that loads cached ranges + gathers + calls the SVAR2 kernels. +- `python/genvarloader/_dataset/_reconstruct.py` — **modify**. Add `HapsSvar2Tracks` and route `HapsSvar2` through `_build_reconstructor`. +- `python/genvarloader/_dataset/_open.py` — **modify**. `_build_seqs` constructs `HapsSvar2` when `metadata.svar2_link` is present. +- `python/genvarloader/_dataset/_migrate.py` — **verify** `svar2_link` is tolerated (additive metadata key). +- Tests: `tests/dataset/test_write_svar2.py`, `tests/dataset/test_svar2_dataset.py`, `tests/unit/dataset/test_svar2_link.py` — **new**. + +--- + +### Task 1: `Svar2Link` model + resolution/fingerprint + +**Files:** +- Create: `python/genvarloader/_dataset/_svar2_link.py` +- Test: `tests/unit/dataset/test_svar2_link.py` + +**Interfaces:** +- Produces: + ```python + class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + def _resolve_svar2(gvl_path: Path, link: Svar2Link | None, override: Path | str | None) -> Path + def _verify_fingerprint2(svar2_path: Path, link: Svar2Link | None) -> None + ``` +- Consumes: nothing gvl-internal beyond the same pattern as `_svar_link.py`. Fingerprint identity for `.svar2` = `n_variants` (from the SparseVar2 index) + a byte count of a canonical store file (see Step 3). + +- [ ] **Step 1: Write the failing test** + +Create `tests/unit/dataset/test_svar2_link.py`: + +```python +from pathlib import Path + +import pytest + +from genvarloader._dataset._svar2_link import ( + Svar2Fingerprint, + Svar2Link, + _resolve_svar2, +) + + +def _mk_svar2_dir(tmp_path: Path) -> Path: + d = tmp_path / "cohort.svar2" + d.mkdir() + (d / "index.arrow").write_bytes(b"stub") + return d + + +def test_resolve_prefers_override(tmp_path): + d = _mk_svar2_dir(tmp_path) + other = tmp_path / "other.svar2" + other.mkdir() + link = Svar2Link( + relative_path="cohort.svar2", + absolute_path=str(d), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + gvl = tmp_path # pretend the gvl dataset lives here + assert _resolve_svar2(gvl, link, other) == other + + +def test_resolve_falls_back_to_relative_then_absolute(tmp_path): + d = _mk_svar2_dir(tmp_path) + gvl = tmp_path / "ds.gvl" + gvl.mkdir() + import os + rel = os.path.relpath(d, start=gvl).replace(os.sep, "/") + link = Svar2Link( + relative_path=rel, absolute_path=str(d), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + assert _resolve_svar2(gvl, link, None) == d + + +def test_resolve_raises_when_unfindable(tmp_path): + gvl = tmp_path / "ds.gvl" + gvl.mkdir() + link = Svar2Link( + relative_path="missing.svar2", absolute_path=str(tmp_path / "missing.svar2"), + fingerprint=Svar2Fingerprint(n_variants=3, store_bytes=4), + ) + with pytest.raises(FileNotFoundError): + _resolve_svar2(gvl, link, None) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: FAIL — `No module named ..._svar2_link`. + +- [ ] **Step 3: Implement `_svar2_link.py`** + +Copy `_svar_link.py`'s structure. Fingerprint on a **stable** `.svar2` identity: `n_variants` from the SparseVar2 index (`pl.scan_ipc(/index.arrow).select(pl.len())`) and the byte size of a canonical store file. **Verify the actual on-disk `.svar2` layout first** — `rtk ls ` (e.g. under `tmp/svar2_mvp/`) to confirm the index filename (`index.arrow`?) and pick one always-present store file for `store_bytes` (e.g. the largest `*.npy`/packed buffer). Do not assume `variant_idxs.npy` — that is SVAR1. + +```python +"""Resolution and integrity for the GVL dataset → SVAR2 back-reference. + +Mirrors _svar_link.py. SVAR2 fingerprint identity = n_variants (from the +SparseVar2 index) + byte count of a canonical store file. +""" +from __future__ import annotations + +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +# The canonical store file used for the byte-count fingerprint. VERIFY against a +# real .svar2 directory and update if the layout differs. +_STORE_FILE = "index.arrow" + + +def _resolve_svar2( + gvl_path: Path, link: Svar2Link | None, override: Path | str | None +) -> Path: + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_fingerprint2(svar2_path: Path, link: Svar2Link | None) -> None: + if link is None: + return + store = svar2_path / _STORE_FILE + if not store.exists(): + raise FileNotFoundError( + f"Expected {store}; resolved svar2 is malformed." + ) + import polars as pl + + n_variants_observed = ( + pl.scan_ipc(svar2_path / "index.arrow").select(pl.len()).collect().item() + ) + observed_bytes = store.stat().st_size + exp = link.fingerprint + mismatches: list[str] = [] + if n_variants_observed != exp.n_variants: + mismatches.append( + f"n_variants: expected {exp.n_variants}, observed {n_variants_observed}" + ) + if observed_bytes != exp.store_bytes: + mismatches.append( + f"store_bytes: expected {exp.store_bytes}, observed {observed_bytes}" + ) + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +rtk git add python/genvarloader/_dataset/_svar2_link.py tests/unit/dataset/test_svar2_link.py +rtk git commit -m "feat(svar2): Svar2Link resolution + fingerprint" +``` + +--- + +### Task 2: `Metadata.svar2_link` field + migration tolerance + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` (`Metadata` model, `:86-95`) +- Test: `tests/unit/dataset/test_svar2_link.py` (extend) + +**Interfaces:** +- Consumes: `Svar2Link` (Task 1). +- Produces: `Metadata.svar2_link: Svar2Link | None = None`; unchanged datasets (no `svar2_link` key) still validate; `_check_dataset_format_version` and `_migrate.migrate` tolerate the additive key. + +- [ ] **Step 1: Write the failing test** + +Add to `tests/unit/dataset/test_svar2_link.py`: + +```python +def test_metadata_roundtrips_svar2_link(): + from genvarloader._dataset._write import Metadata + from genvarloader._dataset._svar2_link import Svar2Fingerprint, Svar2Link + + link = Svar2Link( + relative_path="c.svar2", absolute_path="/abs/c.svar2", + fingerprint=Svar2Fingerprint(n_variants=5, store_bytes=99), + ) + m = Metadata( + contigs=["chr1"], samples=["s0"], ploidy=2, n_regions=1, + svar2_link=link, + ) + m2 = Metadata.model_validate_json(m.model_dump_json()) + assert m2.svar2_link is not None + assert m2.svar2_link.fingerprint.n_variants == 5 + + +def test_metadata_without_svar2_link_still_valid(): + from genvarloader._dataset._write import Metadata + + m = Metadata(contigs=["chr1"], samples=["s0"], ploidy=2, n_regions=1) + assert m.svar2_link is None +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -k metadata -q` +Expected: FAIL — `Metadata` has no field `svar2_link` (unexpected-keyword / validation error). + +- [ ] **Step 3: Add the field** + +In `python/genvarloader/_dataset/_write.py`, import `Svar2Link` next to `from ._svar_link import SvarLink` and add the field to `Metadata` (after `svar_link`, `:94`): + +```python +from ._svar2_link import Svar2Link # noqa: E402 (near the SvarLink import) +... + svar_link: SvarLink | None = None + svar2_link: Svar2Link | None = None +``` + +- [ ] **Step 4: Run to verify it passes, then confirm migration tolerance** + +Run: +```bash +pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q +pixi run -e dev pytest tests/unit/dataset -k migrate -q # existing migration tests still green +``` +Expected: PASS. `_migrate.migrate` reads `metadata.json` as raw JSON and only touches `format_version`, so the additive `svar2_link` key passes through untouched — the migration tests confirm no regression. No `_migrate.py` code change needed; note this in the commit. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_write.py tests/unit/dataset/test_svar2_link.py +rtk git commit -m "feat(svar2): additive svar2_link metadata field" +``` + +--- + +### Task 3: `_write_from_svar2` + write dispatch + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` +- Test: `tests/dataset/test_write_svar2.py` (create) + +**Interfaces:** +- Consumes: `SparseVar2` (genoray), `SparseVar2.find_ranges(contig, starts, ends, samples=, out=)` (genoray split plan), `Svar2Link`/`Svar2Fingerprint` (Task 1), the existing `_reject_unsupported_variants` (`_write.py`), `atomic_dir`/`_prep_bed`/`_write_regions` (existing write pipeline). +- Produces: + ```python + def _write_from_svar2( + path: Path, bed: pl.DataFrame, svar2: "SparseVar2", + samples: list[str], extend_to_length: bool, + ) -> tuple[pl.DataFrame, Svar2Link] + ``` + Writes cache memmaps + `svar2_meta.json` under `/genotypes/`. The cache layout (all int arrays, region-ordered to match `regions.npy`): + - `svar2_dense_range.npy` int32 `(R, 2)` + - `svar2_region_starts.npy` int32 `(R,)` + - `svar2_vk_snp_range.npy` int64 `(R, S, P, 2)` + - `svar2_vk_indel_range.npy` int64 `(R, S, P, 2)` + - `svar2_meta.json` — shapes/dtypes + `sample_cols` + `n_samples`/`ploidy` + +- [ ] **Step 1: Write the failing test** + +Create `tests/dataset/test_write_svar2.py`. Build a small `.svar2` (reuse whatever fixture/helper the existing `tests/test_svar2_reconstruct.py` uses to synthesize a `SparseVar2` store; grep it) and assert `gvl.write` produces the cache + link. + +```python +import json +from pathlib import Path + +import numpy as np +import polars as pl + +import genvarloader as gvl + + +def test_write_svar2_produces_ranges_cache(svar2_store, tmp_path): + # svar2_store: path to a small .svar2 directory (shared fixture; see conftest). + from genoray import SparseVar2 + + sv = SparseVar2(svar2_store) + bed = pl.DataFrame( + {"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]} + ) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + + geno = out / "genotypes" + assert (geno / "svar2_meta.json").exists() + meta = json.loads((geno / "svar2_meta.json").read_text()) + R, S, P = 1, sv.n_samples, sv.ploidy + dr = np.load(geno / "svar2_dense_range.npy") + assert dr.shape == (R, 2) + vks = np.load(geno / "svar2_vk_snp_range.npy") + assert vks.shape == (R, S, P, 2) + + ds_meta = json.loads((out / "metadata.json").read_text()) + assert ds_meta["svar2_link"] is not None + assert ds_meta["ploidy"] == P +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: FAIL — `write` does not recognize `.svar2` (raises the "unrecognized file extension" ValueError, or SparseVar2 branch missing). + +- [ ] **Step 3: Implement detection + dispatch + `_write_from_svar2`** + +3a. Import `SparseVar2` at the top of `_write.py` (next to `from genoray import ..., SparseVar`). + +3b. In the variant-source dispatch (`_write.py:225`, the `elif variants.is_dir() and variants.suffix == ".svar"` branch), add **before** it: + +```python + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) +``` + +3c. In the genotype-writing dispatch (`_write.py:325`, the `elif isinstance(variants, SparseVar):` branch), add a sibling branch: + +```python + elif isinstance(variants, SparseVar2): + from ._svar2_link import Svar2Link # local import ok + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link.model_dump() +``` + +`metadata["ploidy"] = variants.ploidy` at `:330` already runs for any `variants` (SparseVar2 exposes `.ploidy`). Confirm `SparseVar2.available_samples` exists (used at `:233`); if the attribute is `.samples`, add `available_samples` as an alias or special-case the `available_samples` assignment for `SparseVar2`. + +3d. Add `_write_from_svar2` (mirror `_write_from_svar` at `:961`; partition bed by contig, stream `find_ranges` into the cache memmaps): + +```python +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: "SparseVar2", + samples: list[str], + extend_to_length: bool, +) -> tuple[pl.DataFrame, "Svar2Link"]: + import json + import os + + from ._svar2_link import Svar2Fingerprint, Svar2Link + + _reject_unsupported_variants(svar2.index, "SVAR2") # verify svar2.index exists + + out_dir = path / "genotypes" + out_dir.mkdir(parents=True, exist_ok=True) + + R = bed.height + S = len(samples) + P = svar2.ploidy + + dense_range = np.memmap(out_dir / "svar2_dense_range.npy", np.int32, "w+", shape=(R, 2)) + region_starts = np.memmap(out_dir / "svar2_region_starts.npy", np.int32, "w+", shape=(R,)) + vk_snp = np.memmap(out_dir / "svar2_vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap(out_dir / "svar2_vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + + sample_cols: list[int] | None = None + contig_offset = 0 + for (c,), df in bed.partition_by("chrom", as_dict=True, maintain_order=True).items(): + c = cast(str, c) + rc = df.height + rows = slice(contig_offset, contig_offset + rc) + # find_ranges returns a dict bundle; stream into the cache slices via out=. + out = { + "dense_range": dense_range[rows], + "region_starts": region_starts[rows], + "sample_cols": np.empty(S, np.int64), + "vk_snp_range": vk_snp[rows].reshape(rc * S * P, 2), + "vk_indel_range": vk_indel[rows].reshape(rc * S * P, 2), + } + bundle = svar2.find_ranges( + c, df["chromStart"], df["chromEnd"], samples=samples, out=out + ) + if sample_cols is None: + sample_cols = np.asarray(bundle["sample_cols"], np.int64).tolist() + contig_offset += rc + + for m in (dense_range, region_starts, vk_snp, vk_indel): + m.flush() + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "n_regions": R, "n_samples": S, "ploidy": P, + "sample_cols": sample_cols, + "dense_range": {"shape": [R, 2], "dtype": " Implementation notes: (1) `find_ranges`'s `out=` writes into the provided arrays; passing memmap **slices** streams straight to disk. Confirm the genoray `out=` copy handles the `(rc*S*P, 2)` reshape of a memmap slice (it is C-contiguous per contig-block since the outer axis is region). If a slice is non-contiguous, allocate a per-contig scratch array, `find_ranges(..., out=scratch)`, then assign into the memmap slice. (2) `_reject_unsupported_variants(svar2.index, ...)` — verify `SparseVar2` exposes `.index` (a polars frame with the same reject-able columns); if the schema differs, adapt or skip with a comment (upstream `-V other,bnd` already filters symbolic/breakend ALTs per the spec). (3) `svar2.path` — confirm the attribute name (`SparseVar2.__init__` stores `self.path`). + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +rtk git commit -m "feat(svar2): write dispatch + _write_from_svar2 ranges cache" +``` + +--- + +### Task 4: `HapsSvar2` reconstructor — cached ranges + gather + kernel + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_source.py` (add cache-load + `gather_ranges` path) +- Modify: `python/genvarloader/_dataset/_haps.py` (add `HapsSvar2`) +- Test: `tests/dataset/test_svar2_dataset.py` (create) + +**Interfaces:** +- Consumes: the cache memmaps + `svar2_meta.json` (Task 3), `SparseVar2.gather_ranges(contig, bundle)` (genoray split plan), `SparseVar2Source.reconstruct`/`realign_tracks` marshalling (existing, `_svar2_source.py`), `Reference` (for `ref_`/`ref_offsets`/`pad_char`), the `Reconstructor[_H]` protocol (`_protocol.py`). +- Produces: + ```python + @dataclass(slots=True) + class HapsSvar2(Reconstructor[RaggedSeqs]): + path: Path + reference: Reference + svar2: "SparseVar2" + contigs: list[str] + samples: list[str] + ploidy: int + # cache memmaps + sample_cols loaded in from_path + def __call__(self, idx, r_idx, regions, output_length, jitter, rng, + deterministic, splice_plan=None, flat=False, to_rc=None) -> RaggedSeqs + @classmethod + def from_path(cls, path, reference, contigs, samples, ploidy, svar2_link, svar2_override) -> "HapsSvar2" + def gather_block(self, contig, region_rows, sample_slot_idxs) -> dict # bundle -> gather_ranges payload + ``` + +- [ ] **Step 1: Write the failing test (cached ≡ live oracle)** + +Create `tests/dataset/test_svar2_dataset.py`. Oracle = the **live** `SparseVar2Source` path (already decode-validated); assert the cached `HapsSvar2` reconstructs byte-identically. + +```python +from pathlib import Path + +import numpy as np +import polars as pl + +import genvarloader as gvl + + +def test_svar2_dataset_haps_match_live_source(svar2_store, reference_fasta, tmp_path): + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + + sv = SparseVar2(svar2_store) + regions = [(0, 40)] + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes") + cached = ds[0, :] # (S, P, ~L) ragged bytes for region 0, all samples + + # Live oracle via the adapter (contig ref bytes + offsets from the fasta). + ref_bytes, ref_off, pad = _load_contig_ref(reference_fasta, "chr1") + live = SparseVar2Source(sv).reconstruct( + "chr1", regions, ref_bytes, ref_off, pad, output_length=-1 + ) + # Compare byte-for-byte over every (sample, ploid). + np.testing.assert_array_equal(cached.to_packed().data, live.to_packed().data) + np.testing.assert_array_equal( + np.asarray(cached.offsets), np.asarray(live.offsets) + ) +``` + +Add the `_load_contig_ref` helper + `reference_fasta`/`svar2_store` fixtures to `tests/dataset/conftest.py` if absent (mirror the fixtures in `tests/test_svar2_reconstruct.py`). + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: FAIL — `Dataset.open` builds a plain `Haps` (no svar2 dispatch yet) or errors on the missing SVAR1 `svar_meta.json`/`offsets.npy`. + +- [ ] **Step 3a: Add the cache-load + gather path to `SparseVar2Source`** + +In `_svar2_source.py`, add a classmethod-free helper that builds a `gather_ranges` bundle dict from cached memmap slices and calls `self.svar2.gather_ranges`. Refactor `_query` so its `d` (payload dict) can come from **either** live `overlap_batch` (kept as oracle) or `gather_ranges(bundle)`: + +```python + def _query_cached(self, contig, regions, dense_range, region_starts, + vk_snp_range, vk_indel_range, sample_cols): + """Build a find_ranges bundle from cached slices and gather it (tree-free). + Shapes: dense_range (R,2), region_starts (R,), vk_*_range (R,S,P,2).""" + R = dense_range.shape[0] + S = vk_snp_range.shape[1] + P = vk_snp_range.shape[2] + bundle = { + "dense_range": np.ascontiguousarray(dense_range, np.int32), + "region_starts": np.ascontiguousarray(region_starts, np.int32), + "sample_cols": np.ascontiguousarray(sample_cols, np.int64), + "vk_snp_range": np.ascontiguousarray(vk_snp_range.reshape(R * S * P, 2), np.int64), + "vk_indel_range": np.ascontiguousarray(vk_indel_range.reshape(R * S * P, 2), np.int64), + "n_regions": R, "n_samples": S, "ploidy": P, + } + d = self.svar2.gather_ranges(contig, bundle) + reg = np.asarray(regions, np.int32).reshape(R, 2) + reg_rs = np.repeat(reg, S, axis=0) + regions_gvl = np.zeros((R * S, 3), np.int32) + regions_gvl[:, 1:] = reg_rs + dense_range_gvl = np.ascontiguousarray( + np.repeat(np.asarray(d["dense_range"], np.int32), S, axis=0), np.int32 + ) + return d, R, S, P, regions_gvl, dense_range_gvl +``` + +Extract the kernel-call bodies of `reconstruct`/`realign_tracks` so they accept the `(d, R, S, P, regions_gvl, dense_range_gvl)` tuple from **either** `_query` (live) or `_query_cached`. Keep the live `_query` intact — it is the parity oracle. + +- [ ] **Step 3b: Add `HapsSvar2` to `_haps.py`** + +`HapsSvar2` loads the cache once in `from_path`, groups a read batch by contig, and per contig gathers + reconstructs. Reference is **required** (haplotypes need ref bytes). Only `RaggedSeqs` is supported. + +```python +@dataclass(slots=True) +class HapsSvar2(Reconstructor["RaggedSeqs"]): + path: Path + reference: Reference + svar2: "SparseVar2" + contigs: list[str] + samples: list[str] + ploidy: int + dense_range: NDArray[np.int32] # (R, 2) memmap + region_starts: NDArray[np.int32] # (R,) memmap + vk_snp_range: NDArray[np.int64] # (R, S, P, 2) memmap + vk_indel_range: NDArray[np.int64] # (R, S, P, 2) memmap + sample_cols: NDArray[np.int64] # (S,) + + @classmethod + def from_path(cls, path, reference, contigs, samples, ploidy, + svar2_link, svar2_override): + import json + from genoray import SparseVar2 + from ._svar2_link import _resolve_svar2, _verify_fingerprint2 + + svar2_path = _resolve_svar2(path, svar2_link, svar2_override) + _verify_fingerprint2(svar2_path, svar2_link) + svar2 = SparseVar2(svar2_path) + geno = path / "genotypes" + meta = json.loads((geno / "svar2_meta.json").read_text()) + def mm(name): + spec = meta[name] + return np.memmap(geno / f"svar2_{name}.npy", + dtype=np.dtype(spec["dtype"]), + mode="r", shape=tuple(spec["shape"])) + if reference is None: + raise ValueError("SVAR2 haplotype output requires a reference genome.") + return cls( + path=path, reference=reference, svar2=svar2, contigs=contigs, + samples=samples, ploidy=ploidy, + dense_range=mm("dense_range"), region_starts=mm("region_starts"), + vk_snp_range=mm("vk_snp_range"), vk_indel_range=mm("vk_indel_range"), + sample_cols=np.asarray(meta["sample_cols"], np.int64), + ) + + def to_kind(self, kind): + from .._ragged import RaggedSeqs + if kind is not RaggedSeqs: + raise NotImplementedError( + f"SVAR2 datasets support only 'haplotypes' output, not {kind.__name__}." + ) + return self + + def __call__(self, idx, r_idx, regions, output_length, jitter, rng, + deterministic, splice_plan=None, flat=False, to_rc=None): + if splice_plan is not None: + raise NotImplementedError("Spliced SVAR2 haplotypes are not supported.") + # idx -> (region, sample); group by contig; gather+reconstruct per contig; + # stitch back into batch order. Shifts/jitter mirror Haps._prepare_request. + ... # see Step 3c +``` + +- [ ] **Step 3c: Implement `HapsSvar2.__call__` (per-contig grouping + stitch)** + +The batch `regions` is `(b, 3)` = `(contig_idx, start, end)`; `idx` ravels `(region, sample)`. For SVAR2, ploidy is fixed. Group the batch rows by `contig_idx`, and for each contig build the cached bundle for those region rows × the requested samples, gather, and run `SparseVar2Source(...)`'s extracted reconstruct-from-payload. Compute `shifts` exactly as `Haps._prepare_request` does (zeros when `deterministic` or `output_length` is a string; else the same `rng.integers(0, max_shift+1)` where `max_shift = diffs.clip(min=0) + (lengths-output_length).clip(min=0)`). Derive `diffs` from the gathered payload — reuse the SVAR2 kernel's own length computation by first reconstructing with `output_length=-1` (ragged) to learn hap lengths, or expose an ilen helper. **Simplest correct first cut:** support `deterministic`/ragged output (shifts=0) end-to-end, and raise `NotImplementedError` for random-jitter fixed-length until a follow-up — the parity test (Step 1) uses `output_length=-1`, deterministic. Gate non-deterministic with a clear error and a `TODO`. + +Stitching: because each `(region, sample)` maps to one output row, collect per-contig `Ragged` outputs and reassemble a single `Ragged[(b, P, ~L)]` (or `(S, P, ~L)` for a single-region all-sample read) in `idx` order. Reuse `_Flat.from_offsets` as `SparseVar2Source.reconstruct` does. + +> This is the crux integration. Keep the marshalling in `SparseVar2Source` (already validated) and let `HapsSvar2` own only: cache slicing, per-contig grouping, shift computation, and output stitching. Pin every step with the live-oracle parity test. + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS (cached ≡ live, byte-for-byte). If offsets match but bytes differ, the divergence is in shift handling or sample-column mapping — reconcile against the live `_query` path. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_svar2_source.py python/genvarloader/_dataset/_haps.py tests/dataset/test_svar2_dataset.py tests/dataset/conftest.py +rtk git commit -m "feat(svar2): HapsSvar2 cached-ranges + gather reconstructor" +``` + +--- + +### Task 5: `Dataset.open` dispatch to `HapsSvar2` + +**Files:** +- Modify: `python/genvarloader/_dataset/_open.py` (`_build_seqs`, `:143`) +- Modify: `python/genvarloader/_dataset/_reconstruct.py` (`_build_reconstructor` accepts `HapsSvar2`) +- Modify: `python/genvarloader/_dataset/_impl.py` (`Dataset.open` reads `svar2=` override; the `_recon` union type includes `HapsSvar2`) +- Test: `tests/dataset/test_svar2_dataset.py` (extend) + +**Interfaces:** +- Consumes: `metadata.svar2_link` (Task 2), `HapsSvar2.from_path` (Task 4), `self.svar2` override on the open builder (parallel to `self.svar`, `_open.py:160`). +- Produces: `Dataset.open(path, reference=..., svar2=)` returns a dataset whose `_recon` routes haplotype reads through `HapsSvar2`. + +- [ ] **Step 1: Write the failing test** + +```python +def test_open_routes_svar2_to_hapssvar2(svar2_store, reference_fasta, tmp_path): + import polars as pl + from genvarloader._dataset._haps import HapsSvar2 + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store)) + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes") + assert isinstance(ds._recon, HapsSvar2) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -k routes -q` +Expected: FAIL — `_recon` is a plain `Haps`, or `_build_seqs` errors on missing SVAR1 metadata. + +- [ ] **Step 3: Wire the dispatch** + +3a. In `_open.py::_build_seqs` (`:149`), branch on `metadata.svar2_link`: + +```python + if self._has_genotypes(): + if metadata.ploidy is None: + raise ValueError("Malformed dataset: found genotypes but not ploidy.") + if metadata.svar2_link is not None: + from ._haps import HapsSvar2 + if reference is None: + raise ValueError( + "SVAR2 datasets require a reference genome for haplotype output." + ) + return HapsSvar2.from_path( + path=self.path, + reference=reference, + contigs=metadata.contigs, + samples=metadata.samples, + ploidy=metadata.ploidy, + svar2_link=metadata.svar2_link, + svar2_override=getattr(self, "svar2", None), + ) + seqs = Haps.from_path(...) # unchanged SVAR1 path +``` + +`self._has_genotypes()` checks for `genotypes/` — confirm it does not require SVAR1-specific files (`svar_meta.json`); if it does, relax it to also accept `svar2_meta.json`. + +3b. Add a `svar2: Path | str | None = None` field to the open-builder dataclass (parallel to `svar`, wherever `self.svar` is defined) and thread it from `Dataset.open`'s signature (`_impl.py`). + +3c. In `_reconstruct.py::_build_reconstructor`, accept `HapsSvar2` for the `haplotypes` kind. The simplest wiring: treat `HapsSvar2` like `Haps` in the `seqs_kind in ("haplotypes", ...)` branch but restrict to `"haplotypes"`: + +```python + from ._haps import HapsSvar2 + if isinstance(seqs, HapsSvar2): + if seqs_kind not in (None, "haplotypes"): + raise NotImplementedError( + f"SVAR2 datasets support only 'haplotypes', not {seqs_kind!r}." + ) + active_seqs = seqs + # dispatch: HapsSvar2 alone -> itself; with tracks -> HapsSvar2Tracks (Task 6) +``` + +Add `HapsSvar2` (and `HapsSvar2Tracks` from Task 6) to the `_recon` type union in `_impl.py` (`:899`) and the `match self._recon` in `__getitem__` (`:1028`). + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_open.py python/genvarloader/_dataset/_reconstruct.py python/genvarloader/_dataset/_impl.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "feat(svar2): Dataset.open routes svar2 datasets to HapsSvar2" +``` + +--- + +### Task 6: Track re-alignment via the same cache (`HapsSvar2Tracks`) + +**Files:** +- Modify: `python/genvarloader/_dataset/_reconstruct.py` (add `HapsSvar2Tracks`) +- Modify: `python/genvarloader/_dataset/_svar2_source.py` (reuse `realign_tracks` from the cached payload) +- Test: `tests/dataset/test_svar2_dataset.py` (extend) + +**Interfaces:** +- Consumes: `HapsSvar2` (Task 4), `Tracks` (existing), `SparseVar2Source.realign_tracks` marshalling (existing), the same cached bundle + `gather_ranges` payload. +- Produces: `HapsSvar2Tracks(haps: HapsSvar2, tracks: Tracks)` implementing `Reconstructor[tuple[RaggedSeqs, _T]]`; `_build_reconstructor` returns it for `(HapsSvar2, Tracks)`. + +- [ ] **Step 1: Write the failing test (cached tracks ≡ live tracks)** + +```python +def test_svar2_tracks_match_live(svar2_store, reference_fasta, bigwig_track, tmp_path): + import polars as pl + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + out = tmp_path / "ds.gvl" + gvl.write(path=out, bed=bed, variants=Path(svar2_store), tracks=[bigwig_track]) + + ds = gvl.Dataset.open(out, reference=reference_fasta).with_seqs("haplotypes").with_tracks(...) + _, cached_tracks = ds[0, :] + + # Live oracle: SparseVar2Source.realign_tracks with the same track buffer. + sv = SparseVar2(svar2_store) + live = SparseVar2Source(sv).realign_tracks("chr1", [(0, 40)], *_track_args(...)) + np.testing.assert_array_equal( + cached_tracks.to_packed().data, live.to_packed().data + ) +``` + +Adapt fixtures/`_track_args` to whatever the existing `tests/test_svar2_realign_tracks.py` uses. + +- [ ] **Step 2: Run to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -k tracks -q` +Expected: FAIL — `_build_reconstructor` has no `(HapsSvar2, Tracks)` case. + +- [ ] **Step 3: Implement `HapsSvar2Tracks` + dispatch** + +Model on `HapsTracks.__call__` (`_reconstruct.py:130`) but source haplotypes + realigned tracks from the cached `gather_ranges` payload via the extracted `SparseVar2Source.realign_tracks` body. The track buffer is read from the dataset's `intervals/` exactly as `HapsTracks` does (`self.tracks.intervals[name]`); only the haplotype-coordinate re-alignment uses the SVAR2 kernel + cached ranges instead of the SVAR1 sparse-genotype path. Route `(HapsSvar2, Tracks)` in `_build_reconstructor`: + +```python + if isinstance(active_seqs, HapsSvar2) and active_tracks is not None: + return HapsSvar2Tracks(haps=active_seqs, tracks=active_tracks) +``` + +Interval (non-realigned) tracks with SVAR2: keep the existing `realign_tracks=False` guard message from `_build_reconstructor` (`:360`) — reuse verbatim. + +- [ ] **Step 4: Run to verify it passes** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: PASS (haps + tracks parity). + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_reconstruct.py python/genvarloader/_dataset/_svar2_source.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "feat(svar2): HapsSvar2Tracks realign via cached ranges" +``` + +--- + +### Task 7: End-to-end byte-identical parity + full regression + +**Files:** +- Test: `tests/dataset/test_svar2_dataset.py` (extend with the M6b matrix + real chr21) +- Modify: `python/genvarloader/_dataset/_svar2_source.py` — retire `TODO(svar2-dataset-dispatch)` comment (`:7`) + +**Interfaces:** +- Consumes: everything above. Produces no new production surface — hardens the contract. + +- [ ] **Step 1: Add the M6b matrix parity test** + +Parametrize over `{SNP, INS, DEL} × {1, 2, 4} samples × {1, 2} ploidy` (reuse the synthesis helpers from `tests/test_svar2_reconstruct.py`). For each: build a `.svar2`, `gvl.write`, open, and assert `ds[region, samples]` ≡ the live `SparseVar2Source.reconstruct` ≡ genoray `decode` (the cross-check oracle the M6b kernels already validate against). Assert offsets **and** packed bytes equal. + +```python +import pytest + +@pytest.mark.parametrize("variant_kind", ["snp", "ins", "del"]) +@pytest.mark.parametrize("n_samples", [1, 2, 4]) +@pytest.mark.parametrize("ploidy", [1, 2]) +def test_svar2_cached_matches_decode_matrix(variant_kind, n_samples, ploidy, tmp_path): + ... # synth store -> gvl.write -> open -> compare cached vs live vs decode +``` + +- [ ] **Step 2: Add the real chr21 germline + somatic parity test (slow)** + +Mark `@pytest.mark.slow`. Point at the real chr21 SVAR2 stores used by the E1 profiling driver (grep `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` and the profiling driver for their paths). Compare cached-`Dataset` reconstruct vs live `SparseVar2Source` over the 3-region × all-samples workload, haps + tracks. + +- [ ] **Step 3: Retire the deferred-dispatch TODO** + +Remove the `TODO(svar2-dataset-dispatch)` block in `_svar2_source.py:7` (now delivered) and update the module docstring to describe the cache+gather read path. + +- [ ] **Step 4: Full tree regression (SVAR1 unchanged)** + +Run: +```bash +pixi run -e dev pytest tests -q +pixi run -e dev cargo-test +pixi run -e dev ruff check python/ tests/ +pixi run -e dev typecheck +``` +Expected: all green. If any SVAR1 test changed behavior, the additive claim is violated — investigate before proceeding. + +- [ ] **Step 5: Commit** + +```bash +rtk git add python/genvarloader/_dataset/_svar2_source.py tests/dataset/test_svar2_dataset.py +rtk git commit -m "test(svar2): byte-identical cached parity across M6b matrix + chr21" +``` + +--- + +### Task 8: Docs, skill, api.md, and roadmaps + +**Files:** +- Modify: `skills/genvarloader/SKILL.md`, `docs/source/{api,write,format,faq}.md`, `README.md`, `docs/roadmaps/rust-migration.md` +- Modify: `docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md` (mark delivered) — optional + +**Interfaces:** +- Consumes: the shipped feature. Produces: docs true against `main` per the repo's docs-audit + skill-maintenance gates. + +- [ ] **Step 1: Document `.svar2` as a `write` variant source** + +Update `docs/source/write.md` (and `SKILL.md`'s write section) to list `.svar2` directories as an accepted `variants=` source alongside VCF/PGEN/`.svar`, noting the write-time ranges cache and that `Dataset.open` accepts a `svar2=` override (parallel to `svar=`). Update `docs/source/format.md` with the `genotypes/svar2_*.npy` + `svar2_meta.json` cache layout and the `svar2_link` metadata key. + +- [ ] **Step 2: Sync `api.md` with `__all__`** + +If any new symbol was exported (e.g. a `svar2=` kwarg is not a new `__all__` symbol, but confirm no new public class leaked), run the sync check: + +```bash +pixi run -e dev python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. If a symbol was added to `__all__`, add its autodoc entry. + +- [ ] **Step 3: Update FAQ + README + rust-migration roadmap** + +`faq.md`: add "Can I use SVAR2 files with gvl?" → yes, same as `.svar`, with the search/gather cache. `README.md`: add `.svar2` to the supported variant sources if `.svar` is listed. `docs/roadmaps/rust-migration.md`: tick the SVAR2 dataset-wiring task, record the perf-verification result (Step 4), set the phase marker + PR link (per CLAUDE.md's rust-migration rule). + +- [ ] **Step 4: Perf verification (same-session before/after)** + +Per the spec + the `gvl-rust-perf-gate-shared-node-noise` and `gvl-profiling-perf-not-pyspy-native` memories: profile a warm SVAR2 `Dataset` read with `perf` on the Python process (paranoid=2, no `py-spy --native`) and confirm the DSO split flips from ~80% genoray `SearchTree::build` to gvl-kernel-bound, like SVAR1. Report as a **relative before/after within one allocation** (absolute wall-clock not comparable across allocations on shared Carter nodes). Record the numbers in the rust-migration roadmap. + +- [ ] **Step 5: Commit + finish the branch** + +```bash +rtk git add skills/ docs/ README.md +rtk git commit -m "docs(svar2): document .svar2 write source + dataset wiring" +``` + +Then use **superpowers:finishing-a-development-branch** to decide merge/PR. Before the PR: re-run `pixi run -e dev pytest tests -q` (full tree, not scoped) per CLAUDE.md's rename/shared-code gate. + +--- + +## Self-Review + +- **Spec coverage (Components B & C + format/parity/docs):** + - Component B: `.svar2` detection + dispatch + `_write_from_svar2` (Task 3); `_svar2_link.py` (Task 1); `metadata["ploidy"]` set (Task 3); reject unsupported variants (Task 3, Step 3d note). ✅ + - Component C: `Dataset.open` resolve+fingerprint + `svar2=` override (Task 5); `HapsSvar2` haplotype routing retiring `TODO(svar2-dataset-dispatch)` (Tasks 4/7); tracks via same cache (Task 6); `_svar2_source.py` refactor to cache+gather (Tasks 4/6/7). ✅ + - Cache format `.gvl/genotypes/` O(offsets) memmaps + `svar2_meta.json` + `Svar2Link` (Task 3). ✅ + - Parity & testing: byte-identical cached ≡ live ≡ decode on M6b matrix + chr21; SVAR1 additive-green; perf verification (Tasks 4–8). ✅ + - Format version: additive `svar2_link` tolerated by `_check_dataset_format_version`/`_migrate` (Task 2). ✅ + - Docs/roadmaps: skill, api.md, write/format/faq/README, rust-migration (Task 8). ✅ +- **Out of scope honored:** `variants`/`annotated` SVAR2 output modes raise `NotImplementedError` (Tasks 4/5); no `.svar2` on-disk format change (write only reads it). +- **Type consistency:** the cache bundle field names (`dense_range`, `region_starts`, `sample_cols`, `vk_snp_range`, `vk_indel_range`, `n_regions`/`n_samples`/`ploidy`) match exactly between `_write_from_svar2` (Task 3), `SparseVar2Source._query_cached` (Task 4), and the genoray `gather_ranges` bundle contract (genoray plan Task 4). `HapsSvar2.from_path`'s memmap names (`svar2_.npy`) match the files `_write_from_svar2` writes. +- **Known risk flagged in-plan:** Task 4 Step 3c bounds the first cut to deterministic/ragged reads and defers random-jitter fixed-length SVAR2 output to a follow-up (explicit `NotImplementedError`), keeping the crux integration testable against the live oracle rather than over-reaching. diff --git a/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md b/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md new file mode 100644 index 00000000..5193f6b4 --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-gvl-mvp-validate-benchmark.md @@ -0,0 +1,564 @@ +# SVAR2 gvl MVP — Validate & Benchmark Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prove the SVAR2 gvl value proposition on real chr21 data — an e2e track test, real-data validation, and a SVAR1-vs-SVAR2 benchmark — before wiring SVAR2 into the Dataset (Task B, deferred). + +**Architecture:** SVAR2 reconstruction runs through the already-validated `SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`), which queries genoray `SparseVar2.overlap_batch` live. Task C adds the missing track-path test; Tasks D/E exercise the adapter on real germline/somatic chr21 stores. No changes to the SVAR 1.0 path. + +**Tech Stack:** Rust (PyO3/maturin) kernels, Python (numpy, polars, seqpro), genoray 2.15.0 wheel, bcftools/samtools, SLURM (`sbatch -p carter-compute`), pixi (`-e default`). + +**Spec:** `docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md` + +## Global Constraints + +- **Worktree guard:** all work happens in `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`. Every shell/subagent step must first `cd` there and verify `git rev-parse --show-toplevel` ends in `svar2-m6b-kernel`. The gvl worktree is a **different repo** from genoray. +- **Env:** run Python/pytest via `pixi run -e default ...`. Only the `default` env is installed. +- **Rust rebuild:** after any Rust edit, `pixi run -e default maturin develop` (editable install does not auto-rebuild). Not needed for this plan — no Rust edits are planned. +- **Rust tests** (if ever needed): `pixi run -e default cargo test --no-default-features [FILTER]`. +- **Commits:** prek hooks are intentionally not installed here — use `git commit --no-verify`. +- **Commit trailer:** end every commit message with `Co-Authored-By: Claude Opus 4.8 `. +- **genoray** is the pre-built **2.15.0 wheel** in gvl's `default` env (not editable). +- **Reference FASTA:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` (GDC GRCh38; `.fai` present). +- **Real data:** `/carter/users/dlaub/repos/for_loukik/chr21.bcf` (germline, 3202 samples, no `.csi`), `/carter/users/dlaub/repos/for_loukik/gdc.chr21.bcf` (somatic, 16007 samples, `.csi` present). +- **DEL-anchor convention:** genoray `decode`/`overlap_batch` return empty ALT for pure DELs; the anchor `ref[pos]` is injected downstream. Honor this in any oracle. +- **Out of scope:** Task B (Dataset dispatch) — deferred to its own brainstorm after Task E. + +--- + +### Task 1: Clean up merged genoray worktrees (Task A) + +**Files:** none in this repo. Operates on the **genoray** repo at `/carter/users/dlaub/projects/genoray`. + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing consumed by later tasks (independent housekeeping). + +- [ ] **Step 1: Confirm each target worktree has no uncommitted work** + +```bash +for wt in svar-2-m6b svar-2-m6c svar-2-m6-core; do + echo "=== $wt ===" + git -C /carter/users/dlaub/projects/genoray/.claude/worktrees/$wt status --porcelain 2>&1 || echo "(missing)" +done +``` +Expected: empty output under each header (clean). If any prints file paths, **STOP** and report — do not remove a worktree with uncommitted work. + +- [ ] **Step 2: Remove the three worktrees** + +```bash +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6b +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6c +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6-core +``` +Expected: no output (success). If a worktree is already gone, `git worktree remove` errors with "is not a working tree" — that's fine, continue. + +- [ ] **Step 3: Verify removal** + +```bash +git -C /carter/users/dlaub/projects/genoray worktree list +``` +Expected: none of `svar-2-m6b`, `svar-2-m6c`, `svar-2-m6-core` appear. + +No commit (worktree removal is not tracked in this repo). + +--- + +### Task 2: End-to-end test for `realign_tracks` (Task C) + +**Files:** +- Create: `tests/test_svar2_realign_tracks.py` + +**Interfaces:** +- Consumes: `genvarloader._dataset._svar2_source.SparseVar2Source.realign_tracks(contig, regions, tracks, track_offsets, params, strategy_id, base_seed, shifts=None, parallel=False) -> Ragged[np.float32]` (shape `(R, S, P, None)`); `genvarloader._dataset._tracks.shift_and_realign_track_sparse(offset_idx, geno_v_idxs, geno_offsets, v_starts, ilens, shift, track, query_start, out, params, keep=None, strategy_id=0, base_seed=0, query=0, hap=0)` (pure-Python oracle, fills `out` in place); genoray `SparseVar2` + `sv._readers[contig].decode_batch([(start, end)])`. +- Produces: nothing consumed by later tasks. + +**Oracle rationale (from spec):** the pure-Python SVAR1 `shift_and_realign_track_sparse` is a distinct implementation from the SVAR2 **Rust** kernel under test and already carries the DEL-anchor branch (`_tracks.py:755`). Feed it genoray's decoded `(pos, ilen)` per hap via a trivial synthetic layout. DEL-only variants bypass insertion-fill, so the strategy is irrelevant. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_svar2_realign_tracks.py`: + +```python +"""End-to-end validation of the SVAR2 track-realign adapter path. + +Builds a DEL-only SVAR2 store, realigns a reference track through gvl's SVAR2 +path (SparseVar2Source.realign_tracks, the Rust two-source kernel), and compares +per-(region, sample, ploid) against gvl's INDEPENDENT pure-Python SVAR1 track +realign (shift_and_realign_track_sparse) fed genoray's materialized decode +records. Agreement proves the SVAR2 Rust track kernel matches the trusted SVAR1 +realign semantics — including the DEL anchor. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). Two pure DELs chosen to match the reference exactly: +# POS 4 GTA>G -> 0-based pos 3, ilen -2 (ref[3:6] == "GTA") +# POS 10 GGG>G -> 0-based pos 9, ilen -2 (ref[9:12] == "GGG") +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t4\t.\tGTA\tG\t.\t.\t.\tGT\t1|0\t1|1 +chr1\t10\t.\tGGG\tG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def svar2_del_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_del") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), str(ref), ["chr1"], str(out), ["S0", "S1"], + 25_000, 2, 1, 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): + import genoray + from genvarloader._dataset._svar2_source import SparseVar2Source + from genvarloader._dataset._tracks import shift_and_realign_track_sparse + + contig = "chr1" + q_start, q_end = 0, 40 + region_len = q_end - q_start + regions = [(q_start, q_end)] + + sv = genoray.SparseVar2(str(svar2_del_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # A per-region reference track (f32). Random-but-fixed so a positional bug + # can't hide behind a monotonic ramp. + rng = np.random.default_rng(0) + track = rng.random(region_len).astype(np.float32) + + strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) + params = np.zeros(1, np.float64) + base_seed = 0 + + # --- SVAR2 path under test: one region, expanded internally to R*S*P haps --- + src = SparseVar2Source(sv) + out_rag = src.realign_tracks( + contig, + regions, + track, # flat per-region track buffer + np.array([0, region_len], np.int64), # (R+1) offsets + params, + strategy_id, + base_seed, + shifts=None, # no jitter + parallel=False, + ) + + # --- oracle: genoray decode records -> pure-Python SVAR1 realign, per hap --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + + # Non-triviality: haps carry a varying number of DELs. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [1, 1, 2, 1], per_hap_counts + + for s in range(S): + for p in range(P): + h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = np.ascontiguousarray(d_pos[gi0:gi1], np.int32) + ilen_h = np.ascontiguousarray(d_ilen[gi0:gi1], np.int32) + n_h = gi1 - gi0 + + # Independently size the hap: region length + sum of (negative) ilens. + exp_len = region_len + int(ilen_h.sum()) + + got = np.asarray(out_rag[0, s, p]) + assert got.shape[0] == exp_len, ( + f"(s={s},p={p}) SVAR2 len {got.shape[0]} != expected {exp_len} " + f"(ilen={ilen_h.tolist()})" + ) + + # Synthetic single-hap SVAR1 layout: v_idxs 0..n_h, one group. + geno_v_idxs = np.arange(n_h, dtype=np.int32) + geno_offsets = np.array([0, n_h], np.int64) + expected = np.empty(exp_len, np.float32) + shift_and_realign_track_sparse( + offset_idx=0, + geno_v_idxs=geno_v_idxs, + geno_offsets=geno_offsets, + v_starts=pos_h, + ilens=ilen_h, + shift=0, + track=track, + query_start=q_start, + out=expected, + params=params, + strategy_id=strategy_id, + base_seed=base_seed, + query=0, + hap=h, + ) + np.testing.assert_allclose( + got, expected, rtol=0, atol=0, + err_msg=f"(s={s},p={p}) SVAR2 track != SVAR1 oracle " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", + ) +``` + +- [ ] **Step 2: Run the test** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +pixi run -e default pytest tests/test_svar2_realign_tracks.py -v +``` +Expected: **PASS**. This test validates already-working code (the SVAR2 track kernel), so it should pass on first run. If it FAILS: +- On the `per_hap_counts` assert → the decode layout differs from expectation; print `per_hap_counts` and adjust the expected list to the observed counts (the store is deterministic). +- On the length or `assert_allclose` → a **real** discrepancy between the SVAR2 Rust track kernel and the SVAR1 oracle. Do **not** loosen the tolerance. Use superpowers:systematic-debugging: inspect the failing `(s,p)`'s `got` vs `expected` arrays and the DEL anchor handling (`_tracks.py:755` vs the Rust `shift_and_realign_tracks_from_svar2`). + +- [ ] **Step 3: Commit** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tests/test_svar2_realign_tracks.py +git commit --no-verify -m "test: e2e SVAR2 realign_tracks vs SVAR1 pure-Python oracle + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +### Task 3: Real-data validation on chr21 (Task D) + +**Nature:** exploratory validation, **not** pytest. Deliverable is a committed notes file recording what was built and observed. Involves real BCFs and SLURM — has human-checkpointable steps. + +**Files:** +- Create: `tmp/svar2_mvp/build_stores.py` (store-builder script) +- Create: `tmp/svar2_mvp/validate.py` (backend-comparison script) +- Create: `docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md` (results/notes) + +**Interfaces:** +- Consumes: genoray `SparseVar.from_vcf(out, VCF(bcf), max_mem, samples=..., overwrite=True)`, `genoray._core.run_conversion_pipeline(bcf, ref, chroms, out, samples, chunk_size, ploidy, threads, long_allele_cap)`, `SparseVar2(store)`, `SparseVar2.decode(contig, regions)`, `SparseVar2Source.reconstruct(contig, regions, ref_, ref_offsets, pad_char, shifts=None, output_length=-1)`, `gvl.write(path, bed, variants=..., samples=..., overwrite=True)`, `gvl.Dataset.open(path, reference=...)`. +- Produces: two store pairs on disk (`/germline.svar`, `/germline.svar2`, `/somatic.svar`, `/somatic.svar2`) consumed by Task 4 (benchmark). + +Let `WORK=/carter/users/dlaub/repos/for_loukik/svar2_mvp` (create it; big outputs live outside the repo). Let `REF=/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. + +- [ ] **Step 1: Resolve contig naming and index the germline BCF** + +```bash +mkdir -p /carter/users/dlaub/repos/for_loukik/svar2_mvp +REF=/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa +echo "=== FASTA contigs (chr21 region) ==="; grep -E "^>.*(21|chr21)\b" "$REF.fai" 2>/dev/null || cut -f1 "$REF.fai" | grep -E "21$|chr21" +for b in /carter/users/dlaub/repos/for_loukik/chr21.bcf /carter/users/dlaub/repos/for_loukik/gdc.chr21.bcf; do + echo "=== $b contigs ==="; bcftools view -h "$b" | grep -E "^##contig" | grep -E "ID=(chr)?21," | head +done +# germline lacks a .csi: +[ -f /carter/users/dlaub/repos/for_loukik/chr21.bcf.csi ] || bcftools index --csi /carter/users/dlaub/repos/for_loukik/chr21.bcf +``` +Expected: identify the contig name each file uses. **Checkpoint:** if the FASTA uses `chr21` but a BCF uses `21` (common for 1000G germline), record the mismatch — later steps must query each store with **its own** contig name, and `run_conversion_pipeline`/`from_vcf` must be given the BCF's chrom name (`21`), while `reconstruct`/`decode` use that same store's naming. Note the resolved names in the notes file (Step 6). + +- [ ] **Step 2: Normalize+atomize both BCFs to biallelic** + +```bash +cd /carter/users/dlaub/repos/for_loukik +for src in chr21 gdc.chr21; do + bcftools norm -m -any --atomize -Ob -o svar2_mvp/${src}.norm.bcf ${src}.bcf + bcftools index --csi svar2_mvp/${src}.norm.bcf +done +``` +Expected: `svar2_mvp/chr21.norm.bcf` and `svar2_mvp/gdc.chr21.norm.bcf` created. genoray requires normalized+atomized biallelic input. +**Checkpoint (compute):** the somatic file (1.1 GB, 16007 samples) `norm` is heavy — if it does not finish in a few minutes interactively, wrap Steps 2–3 for the somatic source in an `sbatch -p carter-compute` job script and wait for completion before Step 4's somatic validation. + +- [ ] **Step 3: Build both stores per source** + +Create `tmp/svar2_mvp/build_stores.py`: + +```python +"""Build .svar (SVAR1) and .svar2 (SVAR2) stores from a normalized biallelic BCF.""" +import sys +from pathlib import Path + +from genoray import VCF, SparseVar, _core + +def build(bcf: str, chrom: str, samples: list[str], out_prefix: str, ploidy: int): + bcf = str(bcf) + # SVAR 1.0 + SparseVar.from_vcf(f"{out_prefix}.svar", VCF(bcf), "8g", overwrite=True) + # SVAR 2.0 + _core.run_conversion_pipeline( + bcf, "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa", + [chrom], f"{out_prefix}.svar2", samples, + 25_000, ploidy, 8, 8 * 1024 * 1024, + ) + print(f"built {out_prefix}.svar and {out_prefix}.svar2") + +if __name__ == "__main__": + # argv: + bcf, chrom, out_prefix = sys.argv[1], sys.argv[2], sys.argv[3] + import subprocess + samples = subprocess.run( + ["bcftools", "query", "-l", bcf], capture_output=True, text=True, check=True + ).stdout.split() + build(bcf, chrom, samples, out_prefix, ploidy=2) +``` + +Run germline interactively; somatic via SLURM: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +# germline (use the germline BCF's own chrom name from Step 1, e.g. chr21 or 21): +pixi run -e default python tmp/svar2_mvp/build_stores.py $W/chr21.norm.bcf $W/germline +# somatic (heavy) — submit and wait: +sbatch -p carter-compute --wrap "cd $PWD && pixi run -e default python tmp/svar2_mvp/build_stores.py $W/gdc.chr21.norm.bcf $W/somatic" +``` +Replace ``/`` with the names resolved in Step 1. +Expected: four store dirs — `germline.svar`, `germline.svar2`, `somatic.svar`, `somatic.svar2`. **Checkpoint:** monitor the sbatch job (`squeue -u $USER`); proceed to Step 4's somatic checks only after it completes and `somatic.svar2/meta.json` exists. + +- [ ] **Step 4: Validate both backends return sane output** + +Create `tmp/svar2_mvp/validate.py`: + +```python +"""Spot-check that gvl returns non-empty, sane haplotypes + variants through +both the SVAR1 (gvl Dataset over .svar) and SVAR2 (SparseVar2Source over .svar2) +backends, on a handful of regions x a few samples. Correctness is already proven +by the test suite; this proves the REAL-DATA plumbing works.""" +import sys +from pathlib import Path + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + +def main(prefix: str, chrom: str): + # A few small regions (0-based, half-open) in a variant-dense chr21 window. + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500)] + + # --- SVAR2 backend (adapter direct) --- + sv2 = SparseVar2(f"{prefix}.svar2") + print(f"[svar2] n_samples={sv2.n_samples} ploidy={sv2.ploidy}") + ref_bytes = _contig_ref(REF, chrom) + src = SparseVar2Source(sv2) + hap = src.reconstruct( + chrom, regions, + np.frombuffer(ref_bytes, np.uint8), + np.array([0, len(ref_bytes)], np.int64), + pad_char=ord("N"), shifts=None, output_length=-1, + ) + lens = np.asarray(hap.offsets) + print(f"[svar2] hap ragged rows={len(lens) - 1} " + f"min_len={int(np.diff(lens).min())} max_len={int(np.diff(lens).max())}") + var = sv2.decode(chrom, regions) + print(f"[svar2] decode variants: {var}") + + # --- SVAR1 backend (gvl Dataset over .svar) --- + import polars as pl + bed = pl.DataFrame({ + "chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions], + }) + ds_path = f"{prefix}.gvl" + gvl.write(ds_path, bed, variants=f"{prefix}.svar", overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + seqs = ds[:len(regions), :3] # a few regions x first 3 samples + print(f"[svar1] gvl haplotypes sample shape/type: {type(seqs)}") + +def _contig_ref(fasta: str, chrom: str) -> bytes: + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: +``` + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +pixi run -e default python tmp/svar2_mvp/validate.py $W/germline +pixi run -e default python tmp/svar2_mvp/validate.py $W/somatic +``` +Expected: for each source, non-empty haplotype rows with sane lengths (≈ region length ± indels), non-empty decode variants, and the gvl SVAR1 Dataset opening without error. **Checkpoint:** if a chosen region is empty (no variants), pick a denser window (inspect with `bcftools view -H :20000000-20010000 | head`) and rerun. Adapt the API calls to the actual installed signatures if they drift (record any drift in the notes). + +- [ ] **Step 5: Record store sizes now (needed for Task 4)** + +```bash +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +du -sh $W/germline.svar $W/germline.svar2 $W/somatic.svar $W/somatic.svar2 +``` +Expected: four sizes. Somatic `.svar2` is expected smaller than `.svar`. + +- [ ] **Step 6: Write and commit the validation notes** + +Create `docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md` capturing: resolved contig names per file, exact commands run, sample/ploidy counts printed, hap-row counts + length ranges per backend, decode variant summaries, the four `du` sizes, and any API drift encountered. Then: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tmp/svar2_mvp/build_stores.py tmp/svar2_mvp/validate.py docs/superpowers/notes/2026-07-03-svar2-mvp-validation.md +git commit --no-verify -m "chore: SVAR2 MVP real-data validation scripts + notes + +Co-Authored-By: Claude Opus 4.8 " +``` +(The store dirs under `WORK` are outside the repo and are not committed.) + +--- + +### Task 4: Benchmark SVAR1 vs SVAR2 (Task E) + +**Nature:** measurement script producing a table. Depends on Task 3's four stores. + +**Files:** +- Create: `tmp/svar2_mvp/benchmark.py` +- Create: `docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md` (results table) + +**Interfaces:** +- Consumes: the four stores from Task 3; the same backend calls listed in Task 3's Interfaces. +- Produces: a results table (documentation only). + +**Fairness rule (from spec):** query the **same workload** on both backends — **all samples for a fixed region set** — matching genoray's per-contig/all-samples query granularity. Warm caches, N repeats, report **median**. Record the caveat that this is adapter-vs-Dataset (Task B not wired), so SVAR2 latency excludes gvl batching/collation. + +- [ ] **Step 1: Write the benchmark script** + +Create `tmp/svar2_mvp/benchmark.py`: + +```python +"""Benchmark SVAR1 (gvl Dataset over .svar) vs SVAR2 (SparseVar2Source over +.svar2): hap latency, variant latency, store size, for one source prefix. +Fair workload: ALL samples for a fixed region set. Warm caches, median of N.""" +import sys +import time +import subprocess +from statistics import median + +import numpy as np +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +N = 5 # repeats + +def _contig_ref(fasta, chrom): + import pysam + return pysam.FastaFile(fasta).fetch(chrom).encode() + +def _timed(fn, warmup=1): + for _ in range(warmup): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter() + fn() + ts.append(time.perf_counter() - t0) + return median(ts) + +def main(prefix, chrom): + regions = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), + (40_000_000, 40_001_000)] + ref_bytes = _contig_ref(REF, chrom) + ref_u8 = np.frombuffer(ref_bytes, np.uint8) + ref_off = np.array([0, len(ref_bytes)], np.int64) + + # SVAR2 backend + sv2 = SparseVar2(f"{prefix}.svar2") + src = SparseVar2Source(sv2) + svar2_hap = _timed(lambda: src.reconstruct( + chrom, regions, ref_u8, ref_off, pad_char=ord("N"), + shifts=None, output_length=-1)) + svar2_var = _timed(lambda: sv2.decode(chrom, regions)) + + # SVAR1 backend (all samples, same regions) + import polars as pl + bed = pl.DataFrame({"chrom": [chrom] * len(regions), + "chromStart": [s for s, _ in regions], + "chromEnd": [e for _, e in regions]}) + ds_path = f"{prefix}.gvl" + ds = gvl.Dataset.open(ds_path, reference=REF) + ds_hap = ds.with_seqs("haplotypes") + ds_var = ds.with_seqs("variants") + n_s = sv2.n_samples + svar1_hap = _timed(lambda: ds_hap[:len(regions), :n_s]) + svar1_var = _timed(lambda: ds_var[:len(regions), :n_s]) + + def du(path): + return subprocess.run(["du", "-sb", path], capture_output=True, + text=True).stdout.split()[0] + + print(f"source={prefix.split('/')[-1]} chrom={chrom} n_samples={n_s} " + f"regions={len(regions)} N={N}") + print(f" hap_latency_s svar1={svar1_hap:.4f} svar2={svar2_hap:.4f}") + print(f" var_latency_s svar1={svar1_var:.4f} svar2={svar2_var:.4f}") + print(f" store_bytes svar1={du(prefix + '.svar')} " + f"svar2={du(prefix + '.svar2')}") + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) # argv: +``` + +- [ ] **Step 2: Run the benchmark for both sources** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +pixi run -e default python tmp/svar2_mvp/benchmark.py $W/germline +pixi run -e default python tmp/svar2_mvp/benchmark.py $W/somatic +``` +Expected: printed hap/variant latencies + store bytes per source. **Checkpoint:** the somatic all-samples decode (16007 samples) may be large — if a single call is very slow or OOMs, reduce the region set (fewer/smaller windows) uniformly for **both** backends to keep the comparison fair, and note the reduced workload. If `ds.with_seqs("variants")` indexing differs from the installed API, adapt and record the drift. + +- [ ] **Step 3: Assemble the results table and commit** + +Create `docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md` with the 2×3 table (germline/somatic × {hap latency, variant latency, store size}) filled from Step 2's output, the exact workload (regions, n_samples, N), and the recorded caveat (adapter-vs-Dataset; somatic is where SVAR2 layout should win on size). Then: + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +git add tmp/svar2_mvp/benchmark.py docs/superpowers/notes/2026-07-03-svar2-mvp-benchmark.md +git commit --no-verify -m "chore: SVAR2 vs SVAR1 gvl benchmark script + results + +Co-Authored-By: Claude Opus 4.8 " +``` + +- [ ] **Step 4: Summarize the value proposition** + +Report to the user: the filled table, whether SVAR2 wins on store size (esp. somatic) and how hap/variant latency compares, and a recommendation on whether Task B (Dataset wiring) is worth the invasive integration — plus the concrete all-samples-per-batch cost signal the benchmark revealed, to feed the Task B brainstorm. + +--- + +## Self-Review + +**Spec coverage:** +- Task A (cleanup) → Task 1. ✓ +- Task C (realign_tracks e2e test) → Task 2, with the spec's pure-Python SVAR1 oracle. ✓ +- Task D (real-data validation) → Task 3 (build both stores per source, validate both backends, small scope, germline interactive / somatic sbatch, contig-name checkpoint, reference FASTA). ✓ +- Task E (benchmark: hap latency, variant latency, store size, fair all-samples workload, median of warm repeats, 2×3 table, caveat) → Task 4. ✓ +- Task B (Dataset dispatch) → intentionally out of scope; noted in Global Constraints and fed by Task 4 Step 4. ✓ + +**Placeholder scan:** The only intentional fill-ins are ``/``, which are *resolved values* from Task 3 Step 1 (a genuine runtime discovery, not a deferred design decision), and the notes-file contents (recorded observations, which cannot be pre-written). All code steps contain complete, runnable code. No "TBD"/"add error handling"/"similar to Task N". + +**Type consistency:** `SparseVar2Source.reconstruct` / `.realign_tracks` signatures match `_svar2_source.py`; `shift_and_realign_track_sparse` params match `_tracks.py:708`; `run_conversion_pipeline` positional args match the reconstruct-test call and `gvl.write`/`SparseVar.from_vcf` match their source signatures. Hap indexing `h=(r*S+s)*P+p` is consistent between Task 2 and the decode layout. diff --git a/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md new file mode 100644 index 00000000..8ccd96a6 --- /dev/null +++ b/docs/superpowers/plans/2026-07-03-svar2-profiling-followup.md @@ -0,0 +1,976 @@ +# SVAR2 Profiling Follow-up Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Attribute the SVAR2 adapter's query-latency gap to Python overhead vs Rust hot paths vs memory layout, and find the conversion thread-split policy for read-bound (few-contig) builds — measurement only, no optimization. + +**Architecture:** Add small parametrized *profiling driver* scripts under `tmp/svar2_mvp/` that exercise a single code path (SVAR1 `Dataset` hap query, SVAR2 adapter `reconstruct`, or genoray `run_conversion_pipeline`) in a warm loop. Profile them with py-spy (Python-vs-native split, any Python version) and perf (Rust symbol detail, needs debug-symbolized extensions). Accumulate every result table into one notes file. All timed runs go on a dedicated `carter-compute` node. + +**Tech Stack:** Python 3.10.20 via `pixi run -e default`; genoray 2.15.0 (Rust/PyO3, source at `/carter/users/dlaub/projects/genoray`); gvl Rust extension in this repo's `src/`; py-spy 0.x (sampling profiler); Linux `perf`; SLURM (`carter-compute`); bcftools. + +## Global Constraints + +Copied verbatim from the spec — every task's requirements implicitly include these. + +- **This is measurement only.** No optimization, no Dataset wiring (Task B), no dense-layout change, no conversion rebalance. The output is *numbers that decide whether/how to do those*. +- **Env:** `pixi run -e default` (the only installed env). **Python 3.10.20.** +- **py-spy is UNUSABLE on this node (discovered at run start).** `/proc/sys/kernel/yama/ptrace_scope = 2` (admin-only ptrace) and there is no sudo, so py-spy — native *and* non-native — dies with "Permission Denied". **perf is the only profiler.** perf uses `perf_event_open` (paranoid=2 allows own-process sampling), not ptrace, so it works. +- **perf cannot symbolize Python frames on Python < 3.12** (opaque `_PyEval_EvalFrameDefault`). Since py-spy is unavailable, the **Python-vs-native split is done at the DSO level with perf** (`perf report --sort=dso` / `--sort=dso,symbol`): bucket samples by shared object — gvl `genvarloader.abi3.so` + genoray `_core.so` = **native Rust hot path**; numpy `_multiarray_umath*.so` = **conversion/`ascontiguousarray` overhead**; the `python3.10` binary / `libpython` = **interpreter/orchestration**. This DSO split is *finer* than py-spy's leaf python/native split and directly drives the A/B/C classification. Use `--call-graph fp` (frame pointers were compiled in), never `dwarf` (1 GB file, overloads the shared node). +- **Profiler:** perf at `/carter/users/dlaub/.pixi/bin/perf`. (py-spy at `.pixi/envs/default/bin/py-spy` exists but is blocked by ptrace_scope — do not use.) `split_folded.py` (Task 2) was written for py-spy folded output and is therefore **unused**; left in place, harmless. +- **Compute execution reality (discovered at run start):** this session already runs **inside** a small interactive SLURM allocation on `carter-cn-04` (**2 CPUs, 8 GB RAM**, 14-day walltime, governor=performance, paranoid=2). Therefore: **run light single-threaded profiling directly on this node** — do **NOT** wrap it in `srun` (an in-allocation `srun` makes a constrained 2-CPU *step* and fails with "More processors requested than permitted" at >2 cpus). Route **heavy multi-core / large-RAM work through fresh `sbatch` jobs** (independent of this allocation) sized to the big partition nodes (carter-cn-02/03/04 = 96–128 CPU, 476–953 GB). Everywhere a task below says `srun … `, execute `` **directly on the node** instead (the 3-region E1/E2-bench/E3 workloads are single-threaded and fit in 8 GB). Keep `sbatch` for E2 store builds and the E4 thread sweep, adding explicit `--cpus-per-task` and `--mem`. +- **Noise control:** compute node via `sbatch -p carter-compute` for heavy jobs; do **NOT** use `--exclusive` — Carter has only 3 compute nodes, always shared, so `--exclusive` won't schedule. The node is shared and noisy, so **absolute wall-clock is not comparable across allocations** — only *relative* comparisons **within one short allocation on the same hardware** are valid (matches the prior perf-gate lesson: gate on same-session before/after, not absolute time). Therefore: **run both backends of any comparison inside a single `srun`** (back-to-back on the same node), warm caches, **median of N≥5**, record CPU governor / turbo. CPU governor on the target nodes is `performance` — record the actual value observed on the node you land on. Keep the MVP workload (same 3 chr21 regions, all samples) for continuity. +- **MVP workload regions** (0-based half-open, chr21): `(20_000_000, 20_001_000)`, `(30_000_000, 30_000_500)`, `(40_000_000, 40_001_000)`. +- **Stores (already built), `$W`:** `/carter/users/dlaub/repos/for_loukik/svar2_mvp` — `{germline,somatic}.{svar,svar2,gvl}` + `{chr21,gdc.chr21}.norm.filt.bcf`. germline = 3202 samples (1000G, high-AF/dense); somatic = 16007 samples (GDC, rare/sparse). +- **Reference FASTA:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`. +- **Contig:** both cohorts are chr21-only. germline chrom label and somatic chrom label are both `chr21`. +- **Working dir:** the gvl worktree `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`. Scripts live in its `tmp/svar2_mvp/`. +- **Deliverable notes file:** `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md`, holding the E1–E4 tables + two recommendations. +- **RTK:** prefix git/build shell commands with `rtk` per project CLAUDE.md. +- **Commit style:** conventional commits (this is a `chore`/`docs` profiling branch — driver scripts are `chore`, notes are `docs`). + +--- + +## File Structure + +All new files land under `tmp/svar2_mvp/` (profiling drivers, throwaway — not gold-plated) except the notes deliverable. + +- `tmp/svar2_mvp/prof_driver.py` — **E1.** Parametrized single-path driver: `python prof_driver.py `. Warms once, then loops the chosen path K times. For SVAR1 it writes the 3-region `.gvl` once *outside* the loop (so we profile the query, not `gvl.write`). Prints `per_call_s=`. +- `tmp/svar2_mvp/split_folded.py` — **E1/E4.** Post-processes a py-spy folded (`--format raw`) stack file into a **leaf-attributed Python% vs native%** split, plus top-N leaf frames per class. +- `tmp/svar2_mvp/e1_profile.sh` — **E1.** Orchestrates py-spy (flamegraph SVG + speedscope + folded raw) and perf (dwarf call-graph) captures for each `(backend × cohort)` on the compute node; drops artifacts in `tmp/svar2_mvp/prof_out/e1/`. +- `tmp/svar2_mvp/e2_subsample.sh` — **E2.** Builds per-S sample lists and subsampled BCFs (`bcftools view -S`). +- `tmp/svar2_mvp/e2_build.sbatch` — **E2.** SLURM array: for each S, build `.svar` + `.svar2` (reuses `build_stores.py`). +- `tmp/svar2_mvp/e2_bench.py` — **E2.** Hap-latency-vs-S sweep for both backends (warm, median N=5), emits a TSV. +- `tmp/svar2_mvp/e3_probe.py` — **E3 (conditional).** Sweeps SVAR2 hap latency vs (a) region width and (b) `n_dense_variants`. +- `tmp/svar2_mvp/e4_convert_driver.py` — **E4.** Runs one single-contig `run_conversion_pipeline` with a given `max_threads`, under optional `GENORAY_SAMPLE_INTERVAL`. Prints build wall-clock. +- `tmp/svar2_mvp/e4_sweep.sbatch` — **E4.** SLURM: sweep `max_threads` on a fixed single-contig input; capture wall-clock + genoray's reported thread split. +- `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` — **the deliverable.** E1–E4 tables + two recommendations. + +--- + +## Task 0: Symbolized-build setup + notes skeleton + node baseline + +**Files:** +- Create: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` +- Create: `tmp/svar2_mvp/env_baseline.txt` (recorded node/env state) +- No source edits — we inject debug info via Cargo env vars, not Cargo.toml edits. + +**Interfaces:** +- Consumes: nothing. +- Produces: a gvl extension and a genoray wheel both built `--release` **with** `debug = line-tables-only` + frame pointers, installed into the `default` env; a notes file with empty E1–E4 section headers; a recorded env baseline that later tasks cite. The symbol `reconstruct_haplotypes_from_svar2` and genoray's `overlap_batch`/query symbols must resolve in perf after this task. + +- [ ] **Step 1: Grab an interactive compute node and record the baseline** + +Run (from the worktree root): +```bash +srun -p carter-compute --cpus-per-task=16 --pty bash +# then, on the node: +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +{ echo "host=$(hostname)"; echo "date=$(date -Iseconds)"; + echo "governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)"; + echo "turbo_no_turbo=$(cat /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null)"; + echo "nproc=$(nproc)"; + echo "paranoid=$(cat /proc/sys/kernel/perf_event_paranoid)"; } | tee tmp/svar2_mvp/env_baseline.txt +``` +Expected: `governor=performance`, `paranoid` ≤ 2 (memory: paranoid=2 works, no sudo). If `paranoid` > 2, perf call-graph capture will be restricted — note it; py-spy still works. + +> If interactive `srun` isn't practical for the agent, wrap each later timed command as `srun -p carter-compute --cpus-per-task=16 ` instead. Every timed/profiled run in this plan MUST be on a compute node, never the login node. + +- [ ] **Step 2: Confirm profilers and store layout on the node** + +Run: +```bash +.pixi/envs/default/bin/py-spy --version +/carter/users/dlaub/.pixi/bin/perf --version +ls /carter/users/dlaub/repos/for_loukik/svar2_mvp/{germline,somatic}.svar2 +``` +Expected: py-spy prints a version; perf prints a version; both `.svar2` dirs list. + +- [ ] **Step 3: Rebuild the gvl extension with release + line-table debug + frame pointers** + +Run: +```bash +CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ +RUSTFLAGS="-C force-frame-pointers=yes" \ +rtk pixi run -e default maturin develop --release +``` +Expected: `maturin` finishes; the compiled `.so` under `.pixi/envs/default/.../genvarloader/` is rebuilt. (This does NOT edit `Cargo.toml` — the `CARGO_PROFILE_RELEASE_DEBUG` env var injects debug info into the release profile for this build only.) + +- [ ] **Step 4: Rebuild + swap the genoray wheel with the same debug flags** + +genoray source is at `/carter/users/dlaub/projects/genoray` (pyproject version 2.15.0 — matches the installed wheel). Build a debug-symbolized release wheel and force-install it into gvl's `default` env: +```bash +cd /carter/users/dlaub/projects/genoray +CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ +RUSTFLAGS="-C force-frame-pointers=yes" \ +rtk pixi run -e py310 maturin build --release +# install the freshly built wheel into gvl's default env (py310 ABI == default's 3.10.20): +WHEEL=$(ls -t target/wheels/genoray-2.15.0-*.whl | head -1) +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +rtk pixi run -e default pip install --force-reinstall --no-deps "/carter/users/dlaub/projects/genoray/$WHEEL" +``` +Expected: a wheel is produced under `genoray/target/wheels/`; `pip install` reports genoray 2.15.0 reinstalled. Verify version unchanged: +```bash +rtk pixi run -e default python -c "import genoray; print(genoray.__version__)" +``` +Expected: `2.15.0`. + +> **Risk / degrade-gracefully:** if the genoray wheel rebuild fails (toolchain/htslib build issue) or its version drifts from 2.15.0, do NOT block the whole plan. gvl's own kernel (`reconstruct_haplotypes_from_svar2`, in *this* repo's `src/`) was rebuilt in Step 3 and will symbolize regardless. Record in the notes that genoray-internal frames (`overlap_batch`, `query.rs`, `svar2-codec`) may appear as raw addresses in perf, and lean on py-spy `--native` (which still attributes them to the genoray `.so` module even without line tables) for those. Keep going. + +- [ ] **Step 5: Verify Rust symbolization end-to-end** + +Run a 3-second perf capture over the SVAR2 driver (driver written in Task 1 — for now smoke with a one-liner) and grep for a known gvl symbol: +```bash +/carter/users/dlaub/.pixi/bin/perf record -g --call-graph dwarf -o /tmp/sym_check.data -- \ + rtk pixi run -e default python -c " +import numpy as np +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source +import pysam +REF='/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa' +rb=pysam.FastaFile(REF).fetch('chr21').encode() +ru=np.frombuffer(rb,np.uint8); ro=np.array([0,len(rb)],np.int64) +regs=[(20_000_000,20_001_000),(30_000_000,30_000_500),(40_000_000,40_001_000)] +src=SparseVar2Source(SparseVar2('/carter/users/dlaub/repos/for_loukik/svar2_mvp/germline.svar2')) +for _ in range(20): src.reconstruct('chr21',regs,ru,ro,pad_char=ord('N'),shifts=None,output_length=-1) +" +/carter/users/dlaub/.pixi/bin/perf report --stdio -i /tmp/sym_check.data 2>/dev/null | grep -iE "reconstruct_haplotypes_from_svar2|svar2|overlap|query" | head +``` +Expected: at least `reconstruct_haplotypes_from_svar2` (and ideally genoray `overlap`/`query` symbols) appear as named frames, not bare `0x...` addresses. If only gvl symbols resolve, that matches the Step 4 degrade note — acceptable. + +- [ ] **Step 6: Create the results-notes skeleton** + +Write `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md`: +```markdown +# SVAR2 Profiling Results (E1–E4) + +**Date:** 2026-07-03 · **Spec:** `docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md` +**Node/env baseline:** see `tmp/svar2_mvp/env_baseline.txt`. +**Symbolization:** gvl + genoray rebuilt `--release` with `debug=line-tables-only` + frame pointers. + +## E1 — Query-latency attribution +_(filled by Task 3)_ + +## E2 — Same-cohort sample sweep +_(filled by Task 5)_ + +## E3 — Dense-access layout probe +_(filled by Task 6; conditional on E1)_ + +## E4 — Conversion thread-allocation +_(filled by Task 8)_ + +## Recommendations +_(filled by Task 9)_ +``` + +- [ ] **Step 7: Commit** + +```bash +rtk git add tmp/svar2_mvp/env_baseline.txt docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: SVAR2 profiling env baseline + symbolized builds + notes skeleton" +``` + +--- + +## Task 1: E1 profiling driver (`prof_driver.py`) + +**Files:** +- Create: `tmp/svar2_mvp/prof_driver.py` + +**Interfaces:** +- Consumes: `$W` stores; `SparseVar2Source.reconstruct(contig, regions, ref_, ref_offsets, pad_char, shifts, output_length)` (signature in `python/genvarloader/_dataset/_svar2_source.py:51`); `gvl.Dataset.open(path, reference).with_seqs("haplotypes")[:R, :n_s]`. +- Produces: CLI `python prof_driver.py ` where `backend ∈ {svar1, svar2}`, `cohort ∈ {germline, somatic}`, `K` = loop count. Prints exactly one line `per_call_s=`. For `svar1` it writes the 3-region `.gvl` **once before the loop** so profiling captures the query, not `gvl.write`. + +- [ ] **Step 1: Write the driver** + +Create `tmp/svar2_mvp/prof_driver.py`: +```python +"""E1 single-path profiling driver. Exercises ONE code path in a warm loop so +py-spy/perf attribute time to that path only. + + python prof_driver.py + +Prints: per_call_s= +For svar1, the 3-region .gvl is written ONCE before the loop (we profile the +query, not gvl.write).""" +import sys +import time + +import numpy as np + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] + + +def _ref(): + import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + return np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + + +def make_svar2(cohort): + from genoray import SparseVar2 + from genvarloader._dataset._svar2_source import SparseVar2Source + src = SparseVar2Source(SparseVar2(f"{W}/{cohort}.svar2")) + ru, ro = _ref() + + def call(): + src.reconstruct(CHROM, REGIONS, ru, ro, pad_char=ord("N"), + shifts=None, output_length=-1) + return call + + +def make_svar1(cohort): + import polars as pl + import genvarloader as gvl + from genoray import SparseVar2 + n_s = SparseVar2(f"{W}/{cohort}.svar2").n_samples + bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS]}) + ds_path = f"{W}/{cohort}.gvl" + gvl.write(ds_path, bed, variants=f"{W}/{cohort}.svar", overwrite=True) # ONCE + ds_hap = gvl.Dataset.open(ds_path, reference=REF).with_seqs("haplotypes") + + def call(): + ds_hap[:len(REGIONS), :n_s] + return call + + +def main(backend, cohort, K): + call = {"svar1": make_svar1, "svar2": make_svar2}[backend](cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + dt = time.perf_counter() - t0 + print(f"per_call_s={dt / K:.4f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 2: Smoke-test all four (backend × cohort) combos on the node** + +Run (short K to keep it fast): +```bash +for b in svar2 svar1; do for c in germline somatic; do + echo -n "$b $c -> "; rtk pixi run -e default python tmp/svar2_mvp/prof_driver.py $b $c 5 +done; done +``` +Expected: four `per_call_s=` lines, all `>0`, roughly consistent with the MVP table (svar2 germline ~0.28, svar1 germline ~0.06, both somatic ~0.38). If svar1 numbers are ~10× higher than the MVP, the `.gvl` write leaked into timing — verify `gvl.write` is outside the loop. + +- [ ] **Step 3: Commit** + +```bash +rtk git add tmp/svar2_mvp/prof_driver.py +rtk git commit -m "chore: E1 single-path profiling driver for SVAR1/SVAR2 hap query" +``` + +--- + +## Task 2: Python-vs-native split post-processor (`split_folded.py`) + +**Files:** +- Create: `tmp/svar2_mvp/split_folded.py` + +**Interfaces:** +- Consumes: a py-spy `--format raw` folded-stack file (one `frame1;frame2;...;leaf ` line per unique stack). py-spy marks Python frames as `path.py:func:line` and native frames (with `--native`) as demangled symbols or `` without a `.py:` token. +- Produces: CLI `python split_folded.py ` printing `python_pct= native_pct= total_samples=` and the top-15 **leaf** frames with per-frame sample counts and class. Leaf-attribution = self-time. A frame is Python iff its text contains `.py:`. + +- [ ] **Step 1: Write the post-processor** + +Create `tmp/svar2_mvp/split_folded.py`: +```python +"""Split a py-spy --format raw (folded) stack file into Python vs native +self-time by LEAF frame. A leaf frame is Python iff it contains '.py:'. + + python split_folded.py +""" +import sys +from collections import Counter + + +def is_python(frame: str) -> bool: + return ".py:" in frame or frame.endswith(".py") + + +def main(path): + py = nat = 0 + leaves = Counter() + classed = {} + with open(path) as fh: + for line in fh: + line = line.rstrip("\n") + if not line: + continue + stack, _, cnt = line.rpartition(" ") + try: + n = int(cnt) + except ValueError: + continue + leaf = stack.split(";")[-1] + leaves[leaf] += n + classed[leaf] = "python" if is_python(leaf) else "native" + if is_python(leaf): + py += n + else: + nat += n + tot = py + nat + if tot == 0: + print("no samples parsed"); return + print(f"python_pct={100 * py / tot:.1f} native_pct={100 * nat / tot:.1f} total_samples={tot}") + print("top-15 leaf frames (self-time):") + for leaf, n in leaves.most_common(15): + print(f" {100 * n / tot:5.1f}% [{classed[leaf]:6s}] {leaf}") + + +if __name__ == "__main__": + main(sys.argv[1]) +``` + +- [ ] **Step 2: Unit-smoke it on a synthetic folded file** + +Run: +```bash +printf 'a.py:f:1;b.py:g:2 10\nc.py:h:3;native_symbol 30\n' > /tmp/folded_smoke.txt +rtk pixi run -e default python tmp/svar2_mvp/split_folded.py /tmp/folded_smoke.txt +``` +Expected: `python_pct=25.0 native_pct=75.0 total_samples=40`, then two leaf lines (`b.py:g:2` python 25%, `native_symbol` native 75%). + +- [ ] **Step 3: Commit** + +```bash +rtk git add tmp/svar2_mvp/split_folded.py +rtk git commit -m "chore: py-spy folded-stack Python/native self-time splitter" +``` + +--- + +## Task 3: E1 — capture profiles + fill the attribution table + +**Files:** +- Create: `tmp/svar2_mvp/e1_profile.sh` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E1 section) +- Artifacts: `tmp/svar2_mvp/prof_out/e1/` (flamegraphs, speedscope, folded, perf data) — gitignored via not committing binaries. + +**Interfaces:** +- Consumes: `prof_driver.py` (Task 1), `split_folded.py` (Task 2), the symbolized builds (Task 0). +- Produces: E1 table `{backend, cohort, python_pct, native_pct, per_call_s}` for all four combos + top-~10 Rust symbols for the two SVAR2 paths, and a written classification **A / B / C** (Python-adapter overhead / Rust hot-path / mix). + +- [ ] **Step 1: Write the capture orchestrator** + +Create `tmp/svar2_mvp/e1_profile.sh`: +```bash +#!/usr/bin/env bash +# E1: py-spy split (all 4) + perf Rust detail (svar2 only). Run ON a compute node. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/e1 +mkdir -p "$OUT" +PYSPY=.pixi/envs/default/bin/py-spy +PERF=/carter/users/dlaub/.pixi/bin/perf +K=200 # warm loops per capture; adjust so each capture runs >=15s of wall time + +run_pyspy () { # backend cohort + local b=$1 c=$2 tag="${1}_${2}" + # native flamegraph (visual) + folded raw (for the split) + speedscope + $PYSPY record --native --rate 500 --format flamegraph -o "$OUT/${tag}.svg" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + $PYSPY record --native --rate 500 --format raw -o "$OUT/${tag}.folded" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + $PYSPY record --native --rate 500 --format speedscope -o "$OUT/${tag}.speedscope.json" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + echo "== split $tag ==" | tee -a "$OUT/splits.txt" + pixi run -e default python tmp/svar2_mvp/split_folded.py "$OUT/${tag}.folded" | tee -a "$OUT/splits.txt" +} + +run_perf () { # backend cohort (svar2 only) + local b=$1 c=$2 tag="${1}_${2}" + # Use frame-pointer call graph (extensions were built with -C force-frame-pointers=yes). + # DWARF call-graph on this workload produced a 1GB perf.data and overloaded the shared node; + # fp gives clean source-level Rust symbols at ~1MB. Lower -F to keep it light on the shared node. + $PERF record -g --call-graph fp -F 199 -o "$OUT/${tag}.perf.data" -- \ + pixi run -e default python tmp/svar2_mvp/prof_driver.py "$b" "$c" "$K" + echo "== perf top symbols $tag ==" | tee -a "$OUT/perf_top.txt" + $PERF report --stdio -i "$OUT/${tag}.perf.data" --sort=overhead,symbol -g none 2>/dev/null \ + | grep -vE '^\s*#' | head -25 | tee -a "$OUT/perf_top.txt" +} + +for b in svar2 svar1; do for c in germline somatic; do run_pyspy "$b" "$c"; done; done +for c in germline somatic; do run_perf svar2 "$c"; done +echo "DONE. splits -> $OUT/splits.txt ; perf -> $OUT/perf_top.txt" +``` + +- [ ] **Step 2: Run it on the compute node** + +Run: +```bash +chmod +x tmp/svar2_mvp/e1_profile.sh +srun -p carter-compute --cpus-per-task=16 bash tmp/svar2_mvp/e1_profile.sh +``` +Expected: prints four `python_pct=… native_pct=…` blocks and two perf top-symbol blocks; `tmp/svar2_mvp/prof_out/e1/splits.txt` and `perf_top.txt` populated. Sanity: the SVAR1 path (known Rust-bound from prior gvl profiling) should show high `native_pct`; if SVAR1 shows high `python_pct`, py-spy failed to unwind native — check that `--native` is active and the build has frame pointers (Task 0 Step 3). + +- [ ] **Step 3: Extract per-call wall-clock for the table** + +Run (clean timing, no profiler attached — profilers inflate wall time). **One allocation for all four** so the numbers are comparable (shared node — see Global Constraints): +```bash +srun -p carter-compute --cpus-per-task=16 bash -c ' +for b in svar1 svar2; do for c in germline somatic; do + echo -n "$b $c "; pixi run -e default python tmp/svar2_mvp/prof_driver.py $b $c 50 +done; done' +``` +Expected: four `per_call_s=` values; record them as the wall-clock column. (Absolute values are node-dependent; only their *ratios within this run* are load-bearing.) + +- [ ] **Step 4: Read the top Rust symbols for the SVAR2 paths** + +Run: +```bash +cat tmp/svar2_mvp/prof_out/e1/perf_top.txt +``` +Read the top ~10 named Rust symbols (expect candidates: `reconstruct_haplotypes_from_svar2`, genoray `overlap`/`query` decode, dense presence `get_bit` gather, `svar2_codec` decode, memcpy/alloc). Note any that are bare addresses (genoray un-symbolized per Task 0 degrade note). + +- [ ] **Step 5: Fill the E1 section of the notes file** + +Replace the `## E1` placeholder in `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` with a table of the real captured numbers, in this exact shape (fill `<...>` from Steps 2–4): +```markdown +## E1 — Query-latency attribution + +Workload: 3 chr21 regions × all samples, warm, K-loop. py-spy `--native` rate 500; +leaf-attributed Python/native self-time via `split_folded.py`. perf dwarf call-graph. + +| backend | cohort | per_call_s | python_pct | native_pct | +| ------- | -------- | ---------- | ---------- | ---------- | +| svar1 | germline | <...> | <...> | <...> | +| svar1 | somatic | <...> | <...> | <...> | +| svar2 | germline | <...> | <...> | <...> | +| svar2 | somatic | <...> | <...> | <...> | + +**Top SVAR2 Rust symbols (perf, self-overhead):** +1. — + ... (~10) + +**Top SVAR2 Python leaf frames (py-spy):** +- — (e.g. `np.ascontiguousarray` conversions, `overlap_batch` marshalling, FFI boundary) + +**Classification:** — : +- **A** = Python-adapter overhead dominates the SVAR2 gap → Task B Dataset wiring likely erases it. +- **B** = Rust hot-path / dense layout dominates → Task B must carry a kernel/layout fix. +- **C** = mix → quantify each share. +``` + +> Decision hint for classifying: the SVAR2 adapter does a long chain of `np.ascontiguousarray` copies + `overlap_batch` marshalling in `_svar2_source.py:_query`/`reconstruct` (lines 36–88). If those Python leaf frames dominate → A. If the dense-presence gather / codec decode dominates native → B. Report the split, don't force a single bucket. + +- [ ] **Step 6: Commit (scripts + notes; not the binary artifacts)** + +```bash +rtk git add tmp/svar2_mvp/e1_profile.sh docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E1 query-latency attribution — py-spy split + perf Rust detail + table" +``` + +> Do NOT commit `tmp/svar2_mvp/prof_out/` (flamegraph SVGs, speedscope JSON, perf.data are large binaries). If `git status` shows them, add `tmp/svar2_mvp/prof_out/` to `.gitignore` in this commit. + +--- + +## Task 4: E2 — subsample somatic + build per-S stores + +**Files:** +- Create: `tmp/svar2_mvp/e2_subsample.sh` +- Create: `tmp/svar2_mvp/e2_build.sbatch` +- Artifacts: `$W/somatic_s{1000,2000,4000,8000,16007}.{svar,svar2}` + sample lists + per-S variant counts. + +**Interfaces:** +- Consumes: `$W/gdc.chr21.norm.filt.bcf` (the somatic filtered BCF); `build_stores.py build(bcf, chrom, samples, out_prefix, ploidy)` (existing, `tmp/svar2_mvp/build_stores.py`). +- Produces: for each `S ∈ {1000, 2000, 4000, 8000, 16007}`, a subsampled BCF `$W/gdc.chr21.s${S}.bcf`, its `.svar`+`.svar2` at prefix `$W/somatic_s${S}`, and a recorded per-S variant count (subsampling drops sites monomorphic in the subset — record counts so curves are interpretable). S=16007 = the full cohort (reuse existing `somatic.*` — do not rebuild). + +- [ ] **Step 1: Write the subsample script** + +Create `tmp/svar2_mvp/e2_subsample.sh`: +```bash +#!/usr/bin/env bash +# E2: fixed-cohort (somatic) sample-count sweep. Build per-S subsampled BCFs. +set -euo pipefail +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SRC=$W/gdc.chr21.norm.filt.bcf +CHROM=chr21 +bcftools query -l "$SRC" > "$W/somatic.samples.txt" +TOTAL=$(wc -l < "$W/somatic.samples.txt") +echo "total somatic samples=$TOTAL" # expect 16007 +for S in 1000 2000 4000 8000; do + head -n "$S" "$W/somatic.samples.txt" > "$W/somatic.s${S}.list" + bcftools view -S "$W/somatic.s${S}.list" --force-samples -Ob \ + -o "$W/gdc.chr21.s${S}.bcf" "$SRC" + bcftools index -f "$W/gdc.chr21.s${S}.bcf" + N=$(bcftools view -H "$W/gdc.chr21.s${S}.bcf" | wc -l) + echo "S=$S variants=$N" | tee -a "$W/e2_variant_counts.txt" +done +# S=16007 is the full cohort: reuse existing somatic.* store; just record its count. +N=$(bcftools view -H "$SRC" | wc -l) +echo "S=16007 variants=$N" | tee -a "$W/e2_variant_counts.txt" +``` + +- [ ] **Step 2: Run subsampling on the node** + +Run: +```bash +chmod +x tmp/svar2_mvp/e2_subsample.sh +srun -p carter-compute --cpus-per-task=8 bash tmp/svar2_mvp/e2_subsample.sh +cat /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_variant_counts.txt +``` +Expected: `total somatic samples=16007`; five `S=... variants=...` lines with variant counts monotonically non-decreasing in S (mild — see spec caveat). Sub-BCFs and `.csi` indexes present. + +- [ ] **Step 3: Write the build array job** + +Create `tmp/svar2_mvp/e2_build.sbatch`: +```bash +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH --array=0-3 +#SBATCH -J e2build +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_build_%a.log +set -euo pipefail +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +SIZES=(1000 2000 4000 8000) +S=${SIZES[$SLURM_ARRAY_TASK_ID]} +pixi run -e default python tmp/svar2_mvp/build_stores.py \ + "$W/gdc.chr21.s${S}.bcf" chr21 "$W/somatic_s${S}" +``` +> `build_stores.py` reads its own sample list via `bcftools query -l `, so the subsampled BCF fully determines the cohort. S=16007 is NOT in the array — it reuses the existing `$W/somatic.*` store. + +- [ ] **Step 4: Submit and wait for builds** + +Run: +```bash +rtk pixi run -e default sbatch tmp/svar2_mvp/e2_build.sbatch # note: sbatch itself needs no env, but rtk-wrapping is harmless +# poll: +squeue -u "$USER" -n e2build +``` +Expected: four array tasks queue; on completion each `e2_build_.log` ends with `built .../somatic_s.svar and .../somatic_s.svar2`. Verify: +```bash +ls -d /carter/users/dlaub/repos/for_loukik/svar2_mvp/somatic_s{1000,2000,4000,8000}.{svar,svar2} +``` +Expected: eight store dirs exist. + +- [ ] **Step 5: Commit the scripts + variant counts** + +```bash +cp /carter/users/dlaub/repos/for_loukik/svar2_mvp/e2_variant_counts.txt tmp/svar2_mvp/e2_variant_counts.txt +rtk git add tmp/svar2_mvp/e2_subsample.sh tmp/svar2_mvp/e2_build.sbatch tmp/svar2_mvp/e2_variant_counts.txt +rtk git commit -m "chore: E2 somatic sample-count sweep — subsample + per-S store builds" +``` + +--- + +## Task 5: E2 — hap-latency-vs-S benchmark + curves + +**Files:** +- Create: `tmp/svar2_mvp/e2_bench.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E2 section) + +**Interfaces:** +- Consumes: the per-S stores from Task 4 (prefix `$W/somatic_s${S}` for S<16007, `$W/somatic` for S=16007); the same warm-median timing shape as `prof_driver.py`. +- Produces: a TSV `tmp/svar2_mvp/prof_out/e2_curve.tsv` with columns `S variants svar1_hap_s svar2_hap_s`, and slopes (Δlatency/ΔS) for both backends written into the notes. + +- [ ] **Step 1: Write the sweep benchmark** + +Create `tmp/svar2_mvp/e2_bench.py`: +```python +"""E2: hap-latency vs sample-count S for both backends on the SOMATIC cohort. +Fixed dataset family (subsampled somatic), same 3 regions, warm, median N=5. + + python e2_bench.py > tmp/svar2_mvp/prof_out/e2_curve.tsv +""" +import time +from statistics import median + +import numpy as np +import polars as pl +import genvarloader as gvl +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] +N = 5 +SIZES = [1000, 2000, 4000, 8000, 16007] + + +def prefix(S): + return f"{W}/somatic" if S == 16007 else f"{W}/somatic_s{S}" + + +def timed(fn, warmup=1): + for _ in range(warmup): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter() + fn() + ts.append(time.perf_counter() - t0) + return median(ts) + + +def variants_at(S): + # read the recorded count file written by e2_subsample.sh + for line in open(f"{W}/e2_variant_counts.txt"): + if line.startswith(f"S={S} "): + return int(line.split("variants=")[1]) + return -1 + + +def main(): + import pysam + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + ru, ro = np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + print("S\tvariants\tsvar1_hap_s\tsvar2_hap_s") + for S in SIZES: + p = prefix(S) + sv2 = SparseVar2(f"{p}.svar2") + src = SparseVar2Source(sv2) + n_s = sv2.n_samples + svar2 = timed(lambda: src.reconstruct(CHROM, REGIONS, ru, ro, + pad_char=ord("N"), shifts=None, output_length=-1)) + bed = pl.DataFrame({"chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS]}) + gvl.write(f"{p}.gvl", bed, variants=f"{p}.svar", overwrite=True) + ds_hap = gvl.Dataset.open(f"{p}.gvl", reference=REF).with_seqs("haplotypes") + svar1 = timed(lambda: ds_hap[:len(REGIONS), :n_s]) + print(f"{S}\t{variants_at(S)}\t{svar1:.4f}\t{svar2:.4f}", flush=True) + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 2: Run the sweep on the node** + +Run: +```bash +mkdir -p tmp/svar2_mvp/prof_out +srun -p carter-compute --cpus-per-task=16 \ + pixi run -e default python tmp/svar2_mvp/e2_bench.py | tee tmp/svar2_mvp/prof_out/e2_curve.tsv +``` +Expected: a 5-row TSV; `svar2_hap_s` should track the full-cohort somatic value (~0.38) at S=16007 and both columns rise with S. Sanity: at S=16007 the two backends should be near the MVP parity point (~0.38 each). + +- [ ] **Step 3: Compute slopes and fill the E2 notes section** + +Compute least-squares slope per backend: +```bash +rtk pixi run -e default python -c " +import numpy as np +rows=[l.split() for l in open('tmp/svar2_mvp/prof_out/e2_curve.tsv').read().splitlines()[1:]] +S=np.array([int(r[0]) for r in rows]); a=np.array([float(r[2]) for r in rows]); b=np.array([float(r[3]) for r in rows]) +print('svar1 slope s/sample=%.3e'%np.polyfit(S,a,1)[0]) +print('svar2 slope s/sample=%.3e'%np.polyfit(S,b,1)[0]) +" +``` +Replace the `## E2` placeholder with the TSV rendered as a markdown table plus both slopes, and a one-line **decision output**: does SVAR2 latency rise *less steeply* with S than SVAR1 (confirming the MVP's declined hypothesis) or not? Cite the per-S variant counts as the interpretability caveat (subsampling drops monomorphic sites). + +- [ ] **Step 4: Commit** + +```bash +rtk git add tmp/svar2_mvp/e2_bench.py tmp/svar2_mvp/prof_out/e2_curve.tsv docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E2 same-cohort hap-latency-vs-S sweep + slopes" +``` + +--- + +## Task 6: E3 — dense-access layout probe (conditional) + +**Files:** +- Create: `tmp/svar2_mvp/e3_probe.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E3 section) + +**Interfaces:** +- Consumes: E1's classification (Task 3). **Gate:** run this task ONLY if E1 attributed significant native time to the dense-presence gather (classification B or C with the dense gather in the top Rust symbols). Otherwise, write "E3 skipped — E1 showed the SVAR2 gap is " and commit that. +- Consumes: `SparseVar2` (`n_samples`, `overlap_batch`), `SparseVar2Source.reconstruct`. +- Produces: two curves — SVAR2 hap latency vs (a) **region width** (fixed all-samples) and (b) proxy for **`n_dense_variants`** (dense count touched by widening the region toward the whole contig) — into `tmp/svar2_mvp/prof_out/e3.tsv`, and an estimated size of the "contig-wide stride + bit-by-bit read" cost. + +- [ ] **Step 1: Decide the gate** + +Read the E1 classification from the notes file. If it is **A** (Python-adapter overhead) or **native-but-not-dense-gather**, skip to Step 4 (write the skip note). If **B/C with dense gather hot**, proceed to Step 2. + +- [ ] **Step 2: Write the layout probe** + +Create `tmp/svar2_mvp/e3_probe.py`: +```python +"""E3: SVAR2 hap-latency sensitivity to region width and dense count. +Germline (high-AF -> large n_dense_variants) is the stress cohort. + + python e3_probe.py > tmp/svar2_mvp/prof_out/e3.tsv +""" +import time +from statistics import median + +import numpy as np +import pysam +from genoray import SparseVar2 +from genvarloader._dataset._svar2_source import SparseVar2Source + +W = "/carter/users/dlaub/repos/for_loukik/svar2_mvp" +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +N = 5 +START = 20_000_000 +WIDTHS = [200, 1_000, 5_000, 25_000, 100_000, 500_000] + + +def timed(fn): + fn() + ts = [] + for _ in range(N): + t0 = time.perf_counter(); fn(); ts.append(time.perf_counter() - t0) + return median(ts) + + +def main(): + rb = pysam.FastaFile(REF).fetch(CHROM).encode() + ru, ro = np.frombuffer(rb, np.uint8), np.array([0, len(rb)], np.int64) + sv2 = SparseVar2(f"{W}/germline.svar2") + src = SparseVar2Source(sv2) + print("width\tn_dense_in_region\thap_s") + for w in WIDTHS: + regs = [(START, START + w)] + d = sv2.overlap_batch(CHROM, [(START, START + w)]) + # dense variants actually spanned by this region (dense_range gives [lo,hi) per region) + dr = np.asarray(d["dense_range"]).reshape(-1, 2) + n_dense = int((dr[:, 1] - dr[:, 0]).sum()) + t = timed(lambda: src.reconstruct(CHROM, regs, ru, ro, + pad_char=ord("N"), shifts=None, output_length=-1)) + print(f"{w}\t{n_dense}\t{t:.4f}", flush=True) + + +if __name__ == "__main__": + main() +``` +> The dense-presence *stride* is contig-wide (`n_dense_variants`), but the *count read* per query scales with the dense variants spanned by the region — widening the region increases both the gather work and cache-line scatter. This probe measures latency vs that count. A full Rust microbench of `get_bit`-per-column vs word-parallel is out of scope for measurement; note it as the follow-up if the curve is steep. + +- [ ] **Step 3: Run the probe** + +Run: +```bash +srun -p carter-compute --cpus-per-task=16 \ + pixi run -e default python tmp/svar2_mvp/e3_probe.py | tee tmp/svar2_mvp/prof_out/e3.tsv +``` +Expected: latency rising with width and `n_dense_in_region`. A super-linear rise vs `n_dense_in_region` is evidence for the scatter/bit-by-bit cost. + +- [ ] **Step 4: Fill the E3 notes section (probe result OR skip note)** + +If run: replace `## E3` with the TSV as a table + a sentence on whether latency scales with `n_dense_in_region` (evidence for/against the contig-wide-stride + bit-by-bit cost) and a rough magnitude of the potential win from a region-local / word-parallel layout — input to how much layout work Task B carries. +If skipped: replace `## E3` with the one-line skip note citing E1's classification. + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e3_probe.py docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +# include the tsv only if the probe ran: +[ -f tmp/svar2_mvp/prof_out/e3.tsv ] && rtk git add tmp/svar2_mvp/prof_out/e3.tsv +rtk git commit -m "chore: E3 dense-access layout probe (or documented skip per E1)" +``` + +--- + +## Task 7: E4 — conversion phase breakdown (build side) + +**Files:** +- Create: `tmp/svar2_mvp/e4_convert_driver.py` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E4 section, part 1) + +**Interfaces:** +- Consumes: `genoray._core.run_conversion_pipeline(vcf_path, reference_path, chroms, output_dir, samples, chunk_size=25000, ploidy=2, max_threads=None, long_allele_capacity=8388608)` (confirmed signature); `GENORAY_SAMPLE_INTERVAL` env (genoray's built-in channel-fill/per-thread sampler — the only genoray env knob present); `split_folded.py` (Task 2). +- Produces: CLI `python e4_convert_driver.py ` that runs one conversion and prints `build_wall_s=`; plus a py-spy `--native` phase breakdown (htslib read/decompress vs encode vs Phase-2 merge) for a single build, written to the E4 notes section. + +- [ ] **Step 1: Write the conversion driver** + +Create `tmp/svar2_mvp/e4_convert_driver.py`: +```python +"""E4: run ONE single-contig svar2 conversion with a chosen max_threads. + python e4_convert_driver.py +Prints: build_wall_s= +Set GENORAY_SAMPLE_INTERVAL in the environment to enable genoray's sampler.""" +import sys +import time +import subprocess + +from genoray import _core + +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" + + +def main(bcf, chrom, out_prefix, max_threads): + samples = subprocess.run(["bcftools", "query", "-l", bcf], + capture_output=True, text=True, check=True).stdout.split() + t0 = time.perf_counter() + _core.run_conversion_pipeline( + bcf, REF, [chrom], f"{out_prefix}.svar2", samples, + 25_000, 2, int(max_threads), 8 * 1024 * 1024, + ) + print(f"build_wall_s={time.perf_counter() - t0:.2f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) +``` + +- [ ] **Step 2: Capture the phase breakdown with py-spy --native + genoray sampler** + +Use the germline single-contig BCF (~11 min full; use a subsampled BCF from Task 4 if you want a faster capture — e.g. `gdc.chr21.s1000.bcf` builds in a few minutes and exercises the same phases). Run on the node: +```bash +mkdir -p tmp/svar2_mvp/prof_out/e4 +GENORAY_SAMPLE_INTERVAL=1000 \ +srun -p carter-compute --cpus-per-task=16 \ + .pixi/envs/default/bin/py-spy record --native --rate 250 --format raw \ + -o tmp/svar2_mvp/prof_out/e4/convert.folded -- \ + pixi run -e default python tmp/svar2_mvp/e4_convert_driver.py \ + /carter/users/dlaub/repos/for_loukik/svar2_mvp/chr21.norm.filt.bcf chr21 \ + /tmp/e4_probe 8 +``` +Expected: a `.folded` file; py-spy sampled all Rust worker threads (htslib/encode/merge). genoray's sampler prints channel-fill / per-thread CPU% to stderr — capture it too (append `2> tmp/svar2_mvp/prof_out/e4/genoray_sampler.txt` if desired). + +- [ ] **Step 3: Bucket native frames into phases** + +Run the split, then bucket by symbol substring (approximate — regex over demangled names): +```bash +rtk pixi run -e default python tmp/svar2_mvp/split_folded.py tmp/svar2_mvp/prof_out/e4/convert.folded | tee tmp/svar2_mvp/prof_out/e4/split.txt +# rough phase buckets from the folded leaves: +rtk pixi run -e default python -c " +from collections import Counter +buckets=Counter() +for line in open('tmp/svar2_mvp/prof_out/e4/convert.folded'): + stack,_,cnt=line.rstrip().rpartition(' ') + try: n=int(cnt) + except ValueError: continue + leaf=stack.split(';')[-1].lower() + if any(k in leaf for k in ('bgzf','inflate','zlib','htslib','decompress','read')): b='read/decompress' + elif any(k in leaf for k in ('encode','bitgrid','codec','pack')): b='encode' + elif any(k in leaf for k in ('merge','dense_merge','transpose')): b='phase2-merge' + else: b='other' + buckets[b]+=n +tot=sum(buckets.values()) +for b,n in buckets.most_common(): print(f'{100*n/tot:5.1f}% {b}') +" | tee -a tmp/svar2_mvp/prof_out/e4/split.txt +``` +Expected: a percentage per phase; the MVP log's claim ("VCF read/decompress dominating") predicts `read/decompress` is the largest bucket. The `other` bucket catches un-symbolized/misc frames — if it dominates, note that symbol names didn't match the regex (genoray un-symbolized per Task 0) and lean on the genoray sampler output instead. + +- [ ] **Step 4: Write the E4 part-1 notes** + +Replace the `## E4` placeholder's first half with: the phase-bucket table, the genoray sampler's reported thread split (e.g. `1 concurrent chromosome | N HTSlib decompression threads`), and a one-line confirmation/refutation of "conversion is read-bound." + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e4_convert_driver.py docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E4.1 conversion phase breakdown (read/decompress vs encode vs merge)" +``` + +--- + +## Task 8: E4 — thread-split sweep + recommended policy + +**Files:** +- Create: `tmp/svar2_mvp/e4_sweep.sbatch` +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (E4 section, part 2) + +**Interfaces:** +- Consumes: `e4_convert_driver.py` (Task 7). The Python-exposed knob is `max_threads` (total pipeline threads); genoray internally derives the decompression-vs-executor/writer split from it and reports it in its log line. There is **no** finer env knob exposed (only `GENORAY_SAMPLE_INTERVAL`) — so this sweep varies `max_threads` (and total cores) and records genoray's *reported* internal split at each point. +- Produces: a table `max_threads → build_wall_s` (+ genoray's reported decompress/executor split) for a fixed single-contig input, and a recommended few-contig thread policy. + +- [ ] **Step 1: Pick the sweep input** + +Use a single-contig input that builds in a few minutes so the sweep is affordable: the somatic `S=1000` subsample (`$W/gdc.chr21.s1000.bcf`, from Task 4) OR full germline (`$W/chr21.norm.filt.bcf`, ~11 min each). Prefer the smaller input for a denser sweep; note the chosen input in the notes. Sweep `max_threads ∈ {2, 4, 8, 16}` (cap at node `nproc`). + +- [ ] **Step 2: Write the sweep job** + +Create `tmp/svar2_mvp/e4_sweep.sbatch`: +```bash +#!/usr/bin/env bash +#SBATCH -p carter-compute +#SBATCH --cpus-per-task=16 +#SBATCH -J e4sweep +#SBATCH -o /carter/users/dlaub/repos/for_loukik/svar2_mvp/e4_sweep.log +set -euo pipefail +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +W=/carter/users/dlaub/repos/for_loukik/svar2_mvp +BCF=$W/gdc.chr21.s1000.bcf # single-contig, fast; swap to chr21.norm.filt.bcf for germline +for T in 2 4 8 16; do + echo "=== max_threads=$T ===" + GENORAY_SAMPLE_INTERVAL=1000 pixi run -e default python tmp/svar2_mvp/e4_convert_driver.py \ + "$BCF" chr21 "/tmp/e4_sweep_t${T}" "$T" 2>&1 | grep -E "build_wall_s|concurrent chromosome|decompression" + rm -rf "/tmp/e4_sweep_t${T}.svar2" +done +``` + +- [ ] **Step 3: Submit and collect** + +Run: +```bash +rtk pixi run -e default sbatch tmp/svar2_mvp/e4_sweep.sbatch +squeue -u "$USER" -n e4sweep # wait for completion +cat /carter/users/dlaub/repos/for_loukik/svar2_mvp/e4_sweep.log +``` +Expected: four `max_threads=T` blocks, each with a `build_wall_s=` and genoray's reported thread-split line. Wall-clock should fall then plateau (or regress) as threads rise — the minimum identifies the read-bound sweet spot. The sweep runs sequentially in **one** allocation so all points share the same node, but a shared node still adds noise — if a point looks anomalous (non-monotonic in a way threads don't explain), re-run the whole `.sbatch` and prefer the run whose curve is smooth; note contention in the E4 section. + +- [ ] **Step 4: Fill the E4 part-2 notes + recommended policy** + +Replace the `## E4` placeholder's second half with the `max_threads → build_wall_s (+ reported split)` table and a concrete recommendation for **few-contig** jobs (e.g. "for single-contig input, max_threads=N minimizes wall-clock; the read-bound phase wants most threads on htslib decompress"). Add the cross-reference line: this feeds `genoray:docs/roadmap/architecture.md` → Open questions → *read-bound conversion / thread allocation*. + +- [ ] **Step 5: Commit** + +```bash +rtk git add tmp/svar2_mvp/e4_sweep.sbatch docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "chore: E4.2 conversion thread-split sweep + recommended few-contig policy" +``` + +--- + +## Task 9: Synthesis — two recommendations + cross-links + +**Files:** +- Modify: `docs/superpowers/notes/2026-07-03-svar2-profiling-results.md` (Recommendations section) + +**Interfaces:** +- Consumes: the filled E1–E4 sections. +- Produces: the two concrete recommendations the spec requires, plus a decision on E3's layout-work sizing, with pointers back to the spec's Task B and the genoray architecture open question. + +- [ ] **Step 1: Write the two recommendations** + +Replace the `## Recommendations` placeholder with exactly two headed recommendations, each grounded in the tables above: + +1. **Where SVAR2 query latency actually goes → what Task B must include.** State the E1 classification (A/B/C) with the Python% vs native% numbers. If A: "Task B Dataset wiring likely erases the gap — no kernel/layout work needed." If B/C: name the hot Rust symbol(s) and cite E3 for the layout-win size, i.e. "Task B must carry a layout change worth ~." Reference the E2 slopes for the scales-with-S question (confirmed/refuted). +2. **Conversion thread-split policy.** State the E4 read-bound confirmation and the recommended `max_threads` / split for few-contig jobs, with the wall-clock delta between default and recommended. + +- [ ] **Step 2: Add cross-links** + +Add a short "Feeds" list: (a) the spec's Task B (Dataset wiring) — this notes file is the gate that decides its scope; (b) `genoray:docs/roadmap/architecture.md` → Open questions → read-bound conversion / thread allocation (E4). Note that split-by-contig layout remains unassessed (single-contig only) per the MVP notes' open question 3. + +- [ ] **Step 3: Verify the notes file is complete and self-consistent** + +Run (each alternative is a literal leftover-placeholder marker): +```bash +rtk grep -nF -e "filled by" -e "<...>" -e "TBD" docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +``` +Expected: **no matches** — every placeholder replaced with real numbers. If any remain, the corresponding experiment task's Step didn't write its result; go back and fill it. + +- [ ] **Step 4: Commit** + +```bash +rtk git add docs/superpowers/notes/2026-07-03-svar2-profiling-results.md +rtk git commit -m "docs: SVAR2 profiling results synthesis — latency attribution + thread-split recommendations" +``` + +--- + +## Notes on scope & honesty (carry into every task) + +- **No optimization.** If a hot path is obvious, record it — do not fix it. Task B and the thread rebalance are explicitly out of scope (spec "Out of scope"). +- **Profilers inflate wall-clock.** Always take the reported latency numbers from *unprofiled* runs (`prof_driver.py`/`e2_bench.py` timing), and use py-spy/perf only for *attribution* (the %-split and symbol ranks). +- **Degrade gracefully on genoray symbols.** If the genoray wheel rebuild (Task 0 Step 4) didn't land, say so in the notes and rely on py-spy `--native` module attribution + the genoray sampler for genoray-internal phases; gvl's own kernel symbolizes regardless. +- **Record dispersion honestly.** Median of N=5 with no CI — call parity "indistinguishable at this resolution," not "equal," matching the MVP notes' framing. diff --git a/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md b/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md new file mode 100644 index 00000000..aff20cc1 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md @@ -0,0 +1,1092 @@ +# genoray Read-Bound Per-Class Gather + Query-Only Build — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a htslib-free query-only build of genoray plus a read-bound, per-class dense gather that reconstructs the same `BatchResult` payload as today's union path **without** building the contig-wide `DenseUnion` (eliminating the O(N_contig) per-read residual), so GenVarLoader can link `genoray_core` as a query-only path-dep and reconstruct SVAR2 entirely in Rust. + +**Architecture:** Two additive changes to the shipped `svar-2` search/gather split. (1) A `conversion` cargo feature (default-on) gates every htslib-touching module so `--no-default-features` compiles the read/query core alone. (2) `find_ranges` additionally emits per-class `dense_snp_range` / `dense_indel_range` (computed by a per-class `SearchTree` at search time), and a new `gather_ranges_readbound` slices each on-disk dense class window directly into a new split-dense `BatchResultSplit` — never calling `dense_union()`. The shipped `find_ranges` / `gather_ranges` / `read_ranges` / `overlap_batch` stay byte-unchanged as the parity oracle. + +**Tech Stack:** Rust 2024, PyO3 0.29, `numpy` 0.29, `svar2-codec` (workspace member), `rust-htslib` (made optional), `cargo test` with `proptest` + `tempfile`. + +**Repo:** `/carter/users/dlaub/projects/genoray` — branch `svar-2` (HEAD `7099f16`). Lib crate name is `genoray_core`. This is the **absolute** path GenVarLoader path-deps (there is no `../genoray` sibling checkout; the spec's `../genoray` is wrong). + +## Global Constraints + +- **Byte-identical parity contract.** For any `contig, regions, samples`, the read-bound path reconstructs the *same variants per hap* as the shipped union path and the `decode_hap` oracle — field-for-field. The split-dense `BatchResultSplit` merged with var_key equals `overlap_batch`'s union merged with var_key, per hap. +- **Additive.** `overlap_batch`, `find_ranges`, `gather_ranges`, `read_ranges`, `BatchResult`, `RangesBundle.dense_range`, and every existing Python dict key stay **byte-unchanged**. New code is new structs/functions/fields only. The full existing test suite (`tests/test_ranges_split.rs`, `tests/test_batch.rs`, `tests/test_decode_mat.rs`, `svar2-codec` proptests) stays green. +- **Query core is htslib-free.** After Task 1, `cargo build --no-default-features` and `cargo test --no-default-features` compile and pass without linking `rust-htslib`. The default (wheel) build is behavior-unchanged. +- **`rust-htslib` reach = `vcf_reader.rs` + `lib.rs:40-52`.** Only `vcf_reader.rs` uses htslib *types*; `lib.rs`'s `index_bcf_csi`/`index_vcf` are a second direct call site. Both must be gated, or `--no-default-features` fails to compile `lib.rs`. +- **`DenseView` + `carried()` live in `query.rs:120-136`**, not `dense.rs`. `dense.rs` holds only `DENSE_REGISTRY`/`DenseClass`/`DenseSpec`/`DenseMap`. +- **`decode_key` is `svar2_codec::decode_key`**, re-exported verbatim as `rvk::decode_key` (`rvk.rs:14`). Codec primitives used here: `rvk::snp_code_to_key`, `rvk::unpack_snp_key_at`, `rvk::deletion_len`. +- **Row/hap index conventions (unchanged):** `RangesBundle` per-hap row = `r * (n_samples*ploidy) + si*ploidy + p` where `si` is the *selected* sample slot and `sample_cols[si]` is the original sample index. `BatchResult` hap index = `(r*n_samples + s)*ploidy + p`, region-major. +- **Every Rust step:** `cargo test` compiles from source (no separate rebuild needed). Run `cargo test -p genoray_core` for the query core. Run `cargo fmt` + `cargo clippy --all-targets` before each commit; both must be clean. +- **Local-only.** No crates.io / PyPI publish in this plan. Task 6 builds a **local wheel** and confirms the crate builds for the downstream gvl path-dep; that wheel and path-dep MUST be the same commit. + +--- + +## File Structure + +- `Cargo.toml` — make `rust-htslib` optional; add `conversion` feature (default-on). *(Task 1)* +- `src/lib.rs` — `#[cfg(feature = "conversion")]` gates on htslib-touching modules + `index_bcf_csi`/`index_vcf`/`run_conversion_pipeline` + their `#[pymodule]` registrations. *(Task 1)* +- `src/query.rs` — add `dense_snp_overlap` / `dense_indel_overlap` methods, two new `RangesBundle` fields, `BatchResultSplit` struct, `gather_ranges_readbound` fn. *(Tasks 2, 3)* +- `src/py_query_ranges.rs` — add the two new range keys to `bundle_to_dict` / `bundle_from_dict`. *(Task 5)* +- `tests/test_readbound_gather.rs` — new parity + zero-union + per-class test file. *(Task 4)* +- `tests/common/mod.rs` — reused as-is (no change). +- `docs/roadmaps/*` (genoray-side roadmap) — mark the read-bound gather + conversion feature. *(Task 7)* + +--- + +## Task 1: `conversion` cargo feature (htslib-free query core) + +**Files:** +- Modify: `Cargo.toml` (`[dependencies]` `rust-htslib` line; `[features]` block) +- Modify: `src/lib.rs:5-32` (module decls), `src/lib.rs:40-52` (`index_bcf_csi`/`index_vcf`), `src/lib.rs:164-170` (`#[pymodule]`) + +**Interfaces:** +- Produces: a `conversion` feature such that `default = ["conversion", "extension-module"]`; `cargo build --no-default-features` compiles the query core (`query`, `search`, `spine`, `bits`, `nrvk`, `rvk`, `layout`, `dense`, `types`, `error`, `cost_model`, `py_query*`) without `rust-htslib`. + +- [ ] **Step 1: Write the failing build check** + +Add this test to a new file `tests/test_query_only_build.rs`: + +```rust +//! Compile-guard: the query core must build & link without the `conversion` +//! feature (no rust-htslib). If this file compiles under +//! `--no-default-features`, the gate is correct. +#[test] +fn query_core_symbols_are_reachable_without_conversion() { + // Referencing these paths forces the query core to be part of the + // no-default-features build graph. + use genoray_core::query::{ContigReader, find_ranges, gather_ranges}; + let _ = ContigReader::open; + let _ = find_ranges; + let _ = gather_ranges; +} +``` + +- [ ] **Step 2: Run it to verify current state** + +Run: `cargo test --no-default-features --test test_query_only_build 2>&1 | tail -30` +Expected: **FAIL to compile** — `rust-htslib` symbols in `vcf_reader.rs` / `lib.rs` are unconditionally in the build graph, and `pyo3/auto-initialize` (dev-dep) needs libpython. If it fails only on libpython linkage, add `--features pyo3/auto-initialize` is NOT wanted; instead confirm the failure mentions htslib/`vcf_reader`. Record the actual first error. + +- [ ] **Step 3: Make `rust-htslib` optional + add the feature** + +In `Cargo.toml`, change the `rust-htslib` dependency line to optional and add the feature. Replace: + +```toml +rust-htslib = { version = "1.0", default-features = false } +``` + +with: + +```toml +rust-htslib = { version = "1.0", default-features = false, optional = true } +``` + +and replace the `[features]` block: + +```toml +[features] +default = ["extension-module"] +extension-module = ["pyo3/extension-module"] +``` + +with: + +```toml +[features] +# `conversion` pulls in rust-htslib and gates the VCF→svar2 write/convert +# pipeline. Off => query-only core (what gvl links as a path-dep). +default = ["conversion", "extension-module"] +conversion = ["dep:rust-htslib"] +extension-module = ["pyo3/extension-module"] +``` + +- [ ] **Step 4: Gate the htslib-touching modules in `src/lib.rs`** + +At `src/lib.rs:5-32`, add `#[cfg(feature = "conversion")]` above each of these `pub mod` lines (leave the query-core modules ungated): `vcf_reader`, `writer`, `orchestrator`, `normalize`, `budget`, `executor`, `monitor`, `streams`, `merge`, `max_del`, `dense_merge`, `meta`, `py_convert`. Also gate `pub use orchestrator::process_chromosome;`. Example: + +```rust +#[cfg(feature = "conversion")] +pub mod vcf_reader; +#[cfg(feature = "conversion")] +pub mod writer; +// ... (repeat for the list above) +#[cfg(feature = "conversion")] +pub use orchestrator::process_chromosome; +``` + +Gate the two direct htslib call sites and the conversion pyfunctions at `src/lib.rs:40-52`: + +```rust +#[cfg(feature = "conversion")] +fn index_bcf_csi(/* ...existing signature... */) { /* ...unchanged body... */ } + +#[cfg(feature = "conversion")] +#[pyfunction] +fn index_vcf(/* ...existing... */) -> PyResult<()> { /* ...unchanged... */ } + +#[cfg(feature = "conversion")] +#[pyfunction] +fn run_conversion_pipeline(/* ...existing... */) -> PyResult<()> { /* ...unchanged... */ } +``` + +And gate their registrations in the `#[pymodule]` at `src/lib.rs:164-170`: + +```rust +#[pymodule] +fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { + #[cfg(feature = "conversion")] + m.add_function(wrap_pyfunction!(run_conversion_pipeline, m)?)?; + #[cfg(feature = "conversion")] + m.add_function(wrap_pyfunction!(index_vcf, m)?)?; + m.add_class::()?; + Ok(()) +} +``` + +> If the compiler reports another module transitively pulling htslib (e.g. a query-core module that `use`s a now-gated module), gate the *offending `use`*, not the query-core module — the query core must stay ungated. Record any module you had to additionally gate in the commit message. + +- [ ] **Step 5: Verify the query-only build compiles and passes** + +Run: `cargo build --no-default-features 2>&1 | tail -20` +Expected: compiles clean (no `rust-htslib`). +Run: `cargo test --no-default-features --test test_query_only_build 2>&1 | tail -20` +Expected: **PASS**. + +- [ ] **Step 6: Verify the default (wheel) build is unchanged** + +Run: `cargo build 2>&1 | tail -5 && cargo test 2>&1 | tail -30` +Expected: full suite green (default features include `conversion`, so nothing else changed). + +- [ ] **Step 7: fmt + clippy + commit** + +Run: `cargo fmt && cargo clippy --all-targets --no-default-features 2>&1 | tail -20 && cargo clippy --all-targets 2>&1 | tail -20` +Expected: no warnings. + +```bash +cd /carter/users/dlaub/projects/genoray +git add Cargo.toml src/lib.rs tests/test_query_only_build.rs +git commit -m "feat(query): conversion feature gates htslib; query core builds --no-default-features + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: Per-class dense overlap + `find_ranges` emits `dense_snp_range` / `dense_indel_range` + +**Files:** +- Modify: `src/query.rs` — add `dense_snp_overlap` / `dense_indel_overlap` methods to `impl ContigReader` (near `vk_snp_overlap` at `:608-651`); add two fields to `RangesBundle` (`:590-606`); populate them in `find_ranges` (`:657-701`). +- Test: `tests/test_readbound_gather.rs` (new; created here, extended in Task 4). + +**Interfaces:** +- Consumes: `DenseView::positions()` and `DenseView::keys` on `reader.dense_snp` / `reader.dense_indel` (both `Option`); `reader.dense_indel_max_del: u32`; `rvk::deletion_len`, `rvk::unpack_snp_key_at`; `SearchTree::new`, `overlap_range` (already imported in `query.rs`). +- Produces: + - `impl ContigReader { fn dense_snp_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize); fn dense_indel_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize); }` — absolute `[s,e)` into that class's on-disk dense positions/keys table; `(0,0)` when the class table is absent/empty. + - `RangesBundle` gains `pub dense_snp_range: Vec<(usize, usize)>` and `pub dense_indel_range: Vec<(usize, usize)>`, each length `n_regions` (per-region, sample-independent — dense is cohort-shared). + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_readbound_gather.rs` with the synth harness copied from `test_ranges_split.rs:55-79` and this first test: + +```rust +//! Read-bound per-class gather: find_ranges emits per-class dense ranges and +//! gather_ranges_readbound replays them into BatchResultSplit without building +//! the contig-wide DenseUnion. +mod common; + +use common::{SynthRecord, build_contig}; +use genoray_core::query::{ + ContigReader, find_ranges, gather_ranges, gather_ranges_readbound, overlap_batch, +}; +use genoray_core::search; +use tempfile::tempdir; + +fn synth_reader(out: &std::path::Path) -> ContigReader { + let samples = ["S0", "S1"]; + let records = vec![ + SynthRecord { pos: 100, ref_allele: b"A", alts: vec![&b"C"[..]], gt: vec![1, 0, 0, 0] }, + SynthRecord { pos: 200, ref_allele: b"A", alts: vec![&b"AT"[..]], gt: vec![0, 1, 1, 1] }, + SynthRecord { pos: 300, ref_allele: b"AT", alts: vec![&b"A"[..]], gt: vec![1, 1, 0, 1] }, + ]; + build_contig(out, "chr1", &samples, 2, &records); + ContigReader::open(out.to_str().unwrap(), "chr1", 2, 2).unwrap() +} + +#[test] +fn test_find_ranges_emits_per_class_dense_ranges() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let rb = find_ranges(&reader, ®ions, None); + // Both per-class range vectors are per-region (dense is cohort-shared). + assert_eq!(rb.dense_snp_range.len(), regions.len()); + assert_eq!(rb.dense_indel_range.len(), regions.len()); + // Each per-class window is a subset of that class's table; ranges are valid. + for &(s, e) in rb.dense_snp_range.iter().chain(rb.dense_indel_range.iter()) { + assert!(s <= e); + } + // Region 0 spans the whole contig: it must see the one dense SNP (pos 100 is + // var_key here, but the SNP class table is nonempty iff any SNP is dense) and + // the dense indels. The union window must equal snp∪indel counts. + let (us0, ue0) = rb.dense_range[0]; + let snp0 = rb.dense_snp_range[0].1 - rb.dense_snp_range[0].0; + let indel0 = rb.dense_indel_range[0].1 - rb.dense_indel_range[0].0; + assert_eq!(ue0 - us0, snp0 + indel0, + "union window size must equal sum of per-class window sizes"); +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test --test test_readbound_gather test_find_ranges_emits_per_class_dense_ranges 2>&1 | tail -20` +Expected: **FAIL to compile** — `gather_ranges_readbound` unresolved (added Task 3) and `rb.dense_snp_range` unknown field. + +> To iterate on Task 2 alone before Task 3 exists, temporarily comment the `gather_ranges_readbound` import; restore it in Task 3. + +- [ ] **Step 3: Add the two `RangesBundle` fields** + +In `src/query.rs`, extend the `RangesBundle` struct (`:590-606`) — append after `vk_indel_range`: + +```rust + /// `[s, e)` into `dense/snp`'s on-disk positions/keys, per region (dense is + /// cohort-shared, so one window per region, not per hap). Read-bound path. + pub dense_snp_range: Vec<(usize, usize)>, + /// `[s, e)` into `dense/indel`'s on-disk positions/keys, per region. + pub dense_indel_range: Vec<(usize, usize)>, +``` + +- [ ] **Step 4: Add the per-class overlap methods** + +In `src/query.rs`, inside `impl ContigReader` (right after `vk_indel_overlap` ends at `:651`), add: + +```rust + /// Absolute `[s, e)` into `dense/snp`'s positions/keys for one region. + /// SNP v_end = pos + 1 (max_region_length = 0). `(0, 0)` if no snp table. + fn dense_snp_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize) { + let d = match &self.dense_snp { + Some(d) => d, + None => return (0, 0), + }; + let positions = d.positions(); + if positions.is_empty() { + return (0, 0); + } + let v_ends: Vec = positions.iter().map(|&p| p + 1).collect(); + let tree = SearchTree::new(positions); + overlap_range(&tree, &v_ends, 0, q_start, q_end) + } + + /// Absolute `[s, e)` into `dense/indel`'s positions/keys for one region. + /// Indel v_end = pos + 1 + deletion_len(key); per-contig dense max_del bound. + fn dense_indel_overlap(&self, q_start: u32, q_end: u32) -> (usize, usize) { + let d = match &self.dense_indel { + Some(d) => d, + None => return (0, 0), + }; + let positions = d.positions(); + if positions.is_empty() { + return (0, 0); + } + let keys = as_u32(&d.keys); + debug_assert_eq!(positions.len(), keys.len()); + let v_ends: Vec = positions + .iter() + .zip(keys.iter()) + .map(|(&pos, &key)| pos + 1 + rvk::deletion_len(key)) + .collect(); + let tree = SearchTree::new(positions); + overlap_range(&tree, &v_ends, self.dense_indel_max_del, q_start, q_end) + } +``` + +- [ ] **Step 5: Populate the fields in `find_ranges`** + +In `find_ranges` (`:657-701`), after the existing `dense_range` / `region_starts` computation (`:672-677`), add: + +```rust + let dense_snp_range: Vec<(usize, usize)> = regions + .iter() + .map(|&(qs, qe)| reader.dense_snp_overlap(qs, qe)) + .collect(); + let dense_indel_range: Vec<(usize, usize)> = regions + .iter() + .map(|&(qs, qe)| reader.dense_indel_overlap(qs, qe)) + .collect(); +``` + +and add the two fields to the returned `RangesBundle { ... }` literal (`:691-700`): + +```rust + dense_snp_range, + dense_indel_range, +``` + +- [ ] **Step 6: Fix the other `RangesBundle` construction site** + +`find_ranges` is the only constructor, but `cargo build` will confirm. Run: `cargo build 2>&1 | tail -20`. If any other `RangesBundle { ... }` literal errors on the missing fields, add the two fields there too. + +- [ ] **Step 7: Run the per-class range test (isolate it)** + +Temporarily comment the `gather_ranges_readbound` import + the not-yet-written test bodies so only `test_find_ranges_emits_per_class_dense_ranges` compiles. Run: +`cargo test --test test_readbound_gather test_find_ranges_emits_per_class_dense_ranges 2>&1 | tail -20` +Expected: **PASS**. + +- [ ] **Step 8: Confirm the shipped path is byte-unchanged** + +Run: `cargo test --test test_ranges_split 2>&1 | tail -20` +Expected: all existing split tests green (we only *added* fields + methods). + +- [ ] **Step 9: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs tests/test_readbound_gather.rs +git commit -m "feat(query): find_ranges emits per-class dense_snp_range/dense_indel_range + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: `BatchResultSplit` + `gather_ranges_readbound` (no `dense_union`) + +**Files:** +- Modify: `src/query.rs` — add `BatchResultSplit` struct (near `BatchResult` at `:488-504`); add `pub fn gather_ranges_readbound` (after `gather_ranges` at `:711-811`). +- Test: `tests/test_readbound_gather.rs` (Task 4 asserts parity). + +**Interfaces:** +- Consumes: `RangesBundle` (now with `dense_snp_range` / `dense_indel_range` from Task 2); `reader.vk_snp` / `reader.vk_indel` packed positions/keys; `reader.dense_snp` / `reader.dense_indel` (`DenseView`, with `.positions()`, `.keys`, `.carried(hap, col)`); `spine::merge_keys`; `bits::set_bit`; `rvk::{snp_code_to_key, unpack_snp_key_at, deletion_len}`; `as_bytes`, `as_u32`, `KeyRef`. +- Produces: + - `pub fn gather_ranges_readbound(reader: &ContigReader, rb: &RangesBundle) -> BatchResultSplit` — cartesian R×S'; the parity-test vehicle. Builds **zero** `SearchTree`, never calls `reader.dense_union()`. + - `pub fn gather_haps_readbound(reader, region_starts, orig_samples, vk_snp_range, vk_indel_range, dense_snp_range, dense_indel_range, ploidy) -> BatchResultSplit` — **flat per-query** (one `(region, sample)` per query row); the primitive gvl links and calls (Plan 2, Task 4). `n_samples = 1`, hap index `q*ploidy + p`. +- `BatchResultSplit` fields (var_key merged per hap, dense split by class): + ```rust + pub struct BatchResultSplit { + pub n_regions: usize, pub n_samples: usize, pub ploidy: usize, + pub vk: Vec, pub vk_off: Vec, + pub dense_snp: Vec, pub dense_snp_range: Vec<(usize, usize)>, + pub dense_snp_present: Vec, pub dense_snp_present_off: Vec, + pub dense_indel: Vec, pub dense_indel_range: Vec<(usize, usize)>, + pub dense_indel_present: Vec, pub dense_indel_present_off: Vec, + } + ``` + Per-hap presence bitmask is over that class's per-region window `[ds..de)`, LSB-first; `*_present_off` (len H+1) holds **bit** offsets. `H = n_regions * n_samples * ploidy`, hap index `(r*n_samples+s)*ploidy+p` over the *selected* samples. + +- [ ] **Step 1: Add the `BatchResultSplit` struct** + +In `src/query.rs`, after the `BatchResult` struct (`:504`), add: + +```rust +/// Read-bound analog of `BatchResult`: the var_key channel merged per hap (as +/// today), but the dense channel **split per class** so no contig-wide +/// `DenseUnion` is built. gvl merges `var_key ⋈ dense_snp ⋈ dense_indel` by +/// position downstream. `H = n_regions * n_samples * ploidy`, hap index +/// `(r*n_samples + s)*ploidy + p` over the *selected* samples. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BatchResultSplit { + pub n_regions: usize, + pub n_samples: usize, + pub ploidy: usize, + /// Flat merged var_key channel (snp+indel per hap); `vk_off` (len H+1) slices it. + pub vk: Vec, + pub vk_off: Vec, + /// Per-region `dense/snp` windows (uniform keys), concatenated. + pub dense_snp: Vec, + /// `[s, e)` into `dense_snp` per region (len n_regions). + pub dense_snp_range: Vec<(usize, usize)>, + /// Per-hap presence bitmask over that region's `dense_snp[s..e]`, LSB-first; + /// `dense_snp_present_off` (len H+1) holds BIT offsets. + pub dense_snp_present: Vec, + pub dense_snp_present_off: Vec, + /// Per-region `dense/indel` windows (uniform u32 keys), concatenated. + pub dense_indel: Vec, + pub dense_indel_range: Vec<(usize, usize)>, + pub dense_indel_present: Vec, + pub dense_indel_present_off: Vec, +} +``` + +- [ ] **Step 2: Write `gather_ranges_readbound`** + +In `src/query.rs`, after `gather_ranges` (`:811`), add. This mirrors `gather_ranges`'s var_key gather verbatim, and replaces the single union presence loop with two per-class window loops that read positions/keys straight from each on-disk dense table (no `dense_union()`): + +```rust +/// Tree-free, union-free gather: replay a `RangesBundle` into a split-dense +/// `BatchResultSplit`. Builds NO `SearchTree` and never calls `dense_union()` — +/// each region's dense windows come from the per-class `dense_snp_range` / +/// `dense_indel_range` computed in `find_ranges`. The var_key channel is +/// identical to `gather_ranges`; only the dense side is split per class. +pub fn gather_ranges_readbound(reader: &ContigReader, rb: &RangesBundle) -> BatchResultSplit { + let ploidy = rb.ploidy; + let n_samples = rb.n_samples; + let n_regions = rb.n_regions; + let hpr = n_samples * ploidy; + + let snp_positions = reader.vk_snp.positions(); + let snp_keys = as_bytes(&reader.vk_snp.keys); + let indel_positions = reader.vk_indel.positions(); + let indel_keys = as_u32(&reader.vk_indel.keys); + + // Dense class tables (may be absent). + let d_snp = reader.dense_snp.as_ref(); + let d_indel = reader.dense_indel.as_ref(); + let d_snp_pos: &[u32] = d_snp.map(|d| d.positions()).unwrap_or(&[]); + let d_indel_pos: &[u32] = d_indel.map(|d| d.positions()).unwrap_or(&[]); + + // --- dense channel windows (per region), decoded to uniform keys once --- + let mut dense_snp: Vec = Vec::new(); + let mut dense_snp_range: Vec<(usize, usize)> = Vec::with_capacity(n_regions); + let mut dense_indel: Vec = Vec::new(); + let mut dense_indel_range: Vec<(usize, usize)> = Vec::with_capacity(n_regions); + for r in 0..n_regions { + let (ss, se) = rb.dense_snp_range[r]; + let base = dense_snp.len(); + if let Some(d) = d_snp { + let keys = as_bytes(&d.keys); + for j in ss..se { + dense_snp.push(KeyRef { + position: d_snp_pos[j], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, j)), + }); + } + } + dense_snp_range.push((base, dense_snp.len())); + + let (is_, ie_) = rb.dense_indel_range[r]; + let base = dense_indel.len(); + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for j in is_..ie_ { + dense_indel.push(KeyRef { + position: d_indel_pos[j], + key: keys[j], + }); + } + } + dense_indel_range.push((base, dense_indel.len())); + } + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_snp_present: Vec = Vec::new(); + let mut dense_snp_present_off: Vec = vec![0]; + let mut dense_indel_present: Vec = Vec::new(); + let mut dense_indel_present_off: Vec = vec![0]; + + for r in 0..n_regions { + let qs = rb.region_starts[r]; + let (ss, se) = rb.dense_snp_range[r]; + let (is_r, ie_r) = rb.dense_indel_range[r]; + for si in 0..n_samples { + let orig_s = rb.sample_cols[si]; + for p in 0..ploidy { + let col = orig_s * ploidy + p; + let hap = col; + let row = r * hpr + si * ploidy + p; + + // --- var_key gather (identical to gather_ranges) --- + let (vs, ve) = rb.vk_snp_range[row]; + let mut snp_run: Vec = Vec::new(); + for (j, &pos) in snp_positions.iter().enumerate().take(ve).skip(vs) { + if qs < pos + 1 { + snp_run.push(KeyRef { + position: pos, + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(snp_keys, j)), + }); + } + } + let (vis, vie) = rb.vk_indel_range[row]; + let mut indel_run: Vec = Vec::new(); + for j in vis..vie { + let pos = indel_positions[j]; + let v_end = pos + 1 + rvk::deletion_len(indel_keys[j]); + if qs < v_end { + indel_run.push(KeyRef { position: pos, key: indel_keys[j] }); + } + } + let merged = spine::merge_keys(vec![snp_run, indel_run]); + vk.extend_from_slice(&merged); + vk_off.push(vk.len()); + + // --- dense/snp presence bits over [ss..se) --- + let nbits = se - ss; + let bit_base = *dense_snp_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_snp_present.len() < need { + dense_snp_present.resize(need, 0); + } + if let Some(d) = d_snp { + for (k, j) in (ss..se).enumerate() { + // snp v_end = pos + 1; left-overlap re-check qs < v_end. + if d.carried(hap, j) && qs < d_snp_pos[j] + 1 { + bits::set_bit(&mut dense_snp_present, bit_base + k); + } + } + } + dense_snp_present_off.push(bit_base + nbits); + + // --- dense/indel presence bits over [is_r..ie_r) --- + let nbits = ie_r - is_r; + let bit_base = *dense_indel_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_indel_present.len() < need { + dense_indel_present.resize(need, 0); + } + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for (k, j) in (is_r..ie_r).enumerate() { + let v_end = d_indel_pos[j] + 1 + rvk::deletion_len(keys[j]); + if d.carried(hap, j) && qs < v_end { + bits::set_bit(&mut dense_indel_present, bit_base + k); + } + } + } + dense_indel_present_off.push(bit_base + nbits); + } + } + } + + BatchResultSplit { + n_regions, + n_samples, + ploidy, + vk, + vk_off, + dense_snp, + dense_snp_range, + dense_snp_present, + dense_snp_present_off, + dense_indel, + dense_indel_range, + dense_indel_present, + dense_indel_present_off, + } +} +``` + +> **Presence-bit indexing note:** `d.carried(hap, col)` addresses the *global* per-class dense column, and here `j` (the absolute on-disk row inside `[ss..se)`) **is** that global column, because `dense_snp_range` / `dense_indel_range` are absolute indices into the class table. This is the read-bound simplification: the union path had to remap through `dense.src[j] = (is_indel, dcol)`; per-class, `j` is already `dcol`. + +- [ ] **Step 3: Add the flat per-query gather `gather_haps_readbound` (gvl's read primitive)** + +`gather_ranges_readbound` is cartesian R×S' and is the parity-test vehicle (easy to compare to `overlap_batch`). But gvl reads an **arbitrary set of `(region, sample)` pairs** — one query row each, exactly like SVAR1's `geno_offset_idx` — so it needs a *flat per-query* primitive where each query carries its own original sample (dense carriage needs `hap = orig_sample*ploidy + p`). Add, after `gather_ranges_readbound`: + +```rust +/// Flat per-query read-bound gather for gvl's arbitrary-(region,sample) reads. +/// Each of `n_q = region_starts.len()` queries is one (region, sample) pair +/// reconstructing `ploidy` haps. Range arrays are per-query (`dense_*_range`, +/// length n_q) or per-(query,ploid) (`vk_*_range`, length n_q*ploidy, row = +/// q*ploidy + p). Builds zero SearchTrees and never calls `dense_union()`. +/// Returns a `BatchResultSplit` with `n_samples = 1`, hap index `q*ploidy + p`. +pub fn gather_haps_readbound( + reader: &ContigReader, + region_starts: &[u32], + orig_samples: &[usize], + vk_snp_range: &[(usize, usize)], + vk_indel_range: &[(usize, usize)], + dense_snp_range: &[(usize, usize)], + dense_indel_range: &[(usize, usize)], + ploidy: usize, +) -> BatchResultSplit { + let n_q = region_starts.len(); + assert_eq!(orig_samples.len(), n_q); + assert_eq!(dense_snp_range.len(), n_q); + assert_eq!(dense_indel_range.len(), n_q); + assert_eq!(vk_snp_range.len(), n_q * ploidy); + assert_eq!(vk_indel_range.len(), n_q * ploidy); + + let snp_positions = reader.vk_snp.positions(); + let snp_keys = as_bytes(&reader.vk_snp.keys); + let indel_positions = reader.vk_indel.positions(); + let indel_keys = as_u32(&reader.vk_indel.keys); + let d_snp = reader.dense_snp.as_ref(); + let d_indel = reader.dense_indel.as_ref(); + let d_snp_pos: &[u32] = d_snp.map(|d| d.positions()).unwrap_or(&[]); + let d_indel_pos: &[u32] = d_indel.map(|d| d.positions()).unwrap_or(&[]); + + // Dense windows per query (uniform keys), decoded once. + let mut dense_snp: Vec = Vec::new(); + let mut dense_snp_range_out: Vec<(usize, usize)> = Vec::with_capacity(n_q); + let mut dense_indel: Vec = Vec::new(); + let mut dense_indel_range_out: Vec<(usize, usize)> = Vec::with_capacity(n_q); + for q in 0..n_q { + let (ss, se) = dense_snp_range[q]; + let base = dense_snp.len(); + if let Some(d) = d_snp { + let keys = as_bytes(&d.keys); + for j in ss..se { + dense_snp.push(KeyRef { + position: d_snp_pos[j], + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(keys, j)), + }); + } + } + dense_snp_range_out.push((base, dense_snp.len())); + let (is_, ie_) = dense_indel_range[q]; + let base = dense_indel.len(); + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for j in is_..ie_ { + dense_indel.push(KeyRef { position: d_indel_pos[j], key: keys[j] }); + } + } + dense_indel_range_out.push((base, dense_indel.len())); + } + + let mut vk: Vec = Vec::new(); + let mut vk_off: Vec = vec![0]; + let mut dense_snp_present: Vec = Vec::new(); + let mut dense_snp_present_off: Vec = vec![0]; + let mut dense_indel_present: Vec = Vec::new(); + let mut dense_indel_present_off: Vec = vec![0]; + + for q in 0..n_q { + let qs = region_starts[q]; + let orig_s = orig_samples[q]; + let (ss, se) = dense_snp_range[q]; + let (is_r, ie_r) = dense_indel_range[q]; + for p in 0..ploidy { + let hap = orig_s * ploidy + p; + let row = q * ploidy + p; + + // var_key gather. + let (vs, ve) = vk_snp_range[row]; + let mut snp_run: Vec = Vec::new(); + for (j, &pos) in snp_positions.iter().enumerate().take(ve).skip(vs) { + if qs < pos + 1 { + snp_run.push(KeyRef { + position: pos, + key: rvk::snp_code_to_key(rvk::unpack_snp_key_at(snp_keys, j)), + }); + } + } + let (vis, vie) = vk_indel_range[row]; + let mut indel_run: Vec = Vec::new(); + for j in vis..vie { + let pos = indel_positions[j]; + let v_end = pos + 1 + rvk::deletion_len(indel_keys[j]); + if qs < v_end { + indel_run.push(KeyRef { position: pos, key: indel_keys[j] }); + } + } + vk.extend_from_slice(&spine::merge_keys(vec![snp_run, indel_run])); + vk_off.push(vk.len()); + + // dense/snp presence over [ss..se). + let nbits = se - ss; + let bit_base = *dense_snp_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_snp_present.len() < need { dense_snp_present.resize(need, 0); } + if let Some(d) = d_snp { + for (k, j) in (ss..se).enumerate() { + if d.carried(hap, j) && qs < d_snp_pos[j] + 1 { + bits::set_bit(&mut dense_snp_present, bit_base + k); + } + } + } + dense_snp_present_off.push(bit_base + nbits); + + // dense/indel presence over [is_r..ie_r). + let nbits = ie_r - is_r; + let bit_base = *dense_indel_present_off.last().unwrap(); + let need = (bit_base + nbits).div_ceil(8); + if dense_indel_present.len() < need { dense_indel_present.resize(need, 0); } + if let Some(d) = d_indel { + let keys = as_u32(&d.keys); + for (k, j) in (is_r..ie_r).enumerate() { + let v_end = d_indel_pos[j] + 1 + rvk::deletion_len(keys[j]); + if d.carried(hap, j) && qs < v_end { + bits::set_bit(&mut dense_indel_present, bit_base + k); + } + } + } + dense_indel_present_off.push(bit_base + nbits); + } + } + + BatchResultSplit { + n_regions: n_q, + n_samples: 1, + ploidy, + vk, + vk_off, + dense_snp, + dense_snp_range: dense_snp_range_out, + dense_snp_present, + dense_snp_present_off, + dense_indel, + dense_indel_range: dense_indel_range_out, + dense_indel_present, + dense_indel_present_off, + } +} +``` + +- [ ] **Step 4: Build** + +Run: `cargo build 2>&1 | tail -20` +Expected: compiles (unused-warnings fine until Task 4 references both functions). + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs +git commit -m "feat(query): gather_ranges_readbound + gather_haps_readbound + BatchResultSplit + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Parity + zero-union + per-class tests + +**Files:** +- Modify: `tests/test_readbound_gather.rs` (add the parity/zero-union tests). + +**Interfaces:** +- Consumes: `gather_ranges_readbound`, `find_ranges`, `overlap_batch`, `read_ranges`, `BatchResult::decode_hap`, `search::search_tree_build_count`. + +**Parity strategy (byte-identical contract).** `BatchResultSplit` has a different *shape* than `BatchResult` (dense split, not unioned), so we assert parity at the **decoded-variants** level: for every `(r, s, p)`, the set of merged `(position, key)` from the read-bound result equals the set from `overlap_batch`'s union path. We build a local `readbound_decode_hap` helper that merges `vk ⋈ dense_snp ⋈ dense_indel` (mirroring how gvl will), then compare against `BatchResult::decode_hap` (the shipped oracle). + +- [ ] **Step 1: Add the read-bound decode helper + parity test** + +Append to `tests/test_readbound_gather.rs`: + +```rust +use genoray_core::query::{BatchResultSplit, HapCalls, decode_keyref_pub}; // see note below + +/// Merge vk ⋈ dense_snp ⋈ dense_indel for one hap and decode — the gvl-side +/// reconstruction, expressed as a test oracle. +fn readbound_decode_hap( + br: &BatchResultSplit, + reader: &ContigReader, + r: usize, + s: usize, + p: usize, +) -> Vec<(u32, i32)> { + use genoray_core::query::KeyRef; + let h = (r * br.n_samples + s) * br.ploidy + p; + let mut merged: Vec = br.vk[br.vk_off[h]..br.vk_off[h + 1]].to_vec(); + + let (ss, se) = br.dense_snp_range[r]; + let bit0 = br.dense_snp_present_off[h]; + for (k, j) in (ss..se).enumerate() { + if genoray_core::bits_get_bit(&br.dense_snp_present, bit0 + k) { + merged.push(br.dense_snp[j]); + } + } + let (is_, ie_) = br.dense_indel_range[r]; + let bit0 = br.dense_indel_present_off[h]; + for (k, j) in (is_..ie_).enumerate() { + if genoray_core::bits_get_bit(&br.dense_indel_present, bit0 + k) { + merged.push(br.dense_indel[j]); + } + } + // Stable position sort (var_key already ahead of dense within its own run). + merged.sort_by_key(|kr| kr.position); + merged.into_iter().map(|kr| (kr.position, kr.key as i32)).collect() +} + +#[test] +fn test_readbound_reconstructs_union_per_hap() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32), (150u32, 250u32)]; + + let oracle = overlap_batch(&reader, ®ions); + let rb = find_ranges(&reader, ®ions, None); + let got = gather_ranges_readbound(&reader, &rb); + + assert_eq!(got.n_regions, oracle.n_regions); + assert_eq!(got.n_samples, oracle.n_samples); + assert_eq!(got.ploidy, oracle.ploidy); + + for r in 0..oracle.n_regions { + for s in 0..oracle.n_samples { + for p in 0..oracle.ploidy { + // Oracle: decode via the shipped union decode_hap, keep (pos, key). + let hc: HapCalls = oracle.decode_hap(&reader, r, s, p); + // decode_hap returns decoded alts, not raw keys — compare on the + // (position, ilen) projection that survives decode instead. + let want: Vec<(u32, i32)> = + hc.positions.iter().zip(hc.ilens.iter()).map(|(&a, &b)| (a, b)).collect(); + let got_keys = readbound_decode_hap(&got, &reader, r, s, p); + // Decode the read-bound raw keys the same way to get ilens. + let got_dec: Vec<(u32, i32)> = got_keys + .iter() + .map(|&(pos, key)| (pos, decode_keyref_pub(pos, key as u32, &reader))) + .collect(); + assert_eq!(got_dec, want, "hap (r={r}, s={s}, p={p})"); + } + } + } +} +``` + +> **Helper exports needed.** This test references three items that must be `pub` in genoray. In `src/query.rs`: make `decode_keyref` reachable via a thin public wrapper `pub fn decode_keyref_pub(position: u32, key: u32, reader: &ContigReader) -> i32` that builds a `KeyRef { position, key }`, calls the existing `decode_keyref(kr, reader.lut.as_ref())`, and returns `.ilen`. In `src/lib.rs` add `pub fn bits_get_bit(bytes: &[u8], i: usize) -> bool { bits::get_bit(bytes, i) }` (a re-export shim; `bits` is already `pub mod`). Also `pub use query::{BatchResultSplit, KeyRef, HapCalls};` if not already public — `HapCalls` (`:435`) and `KeyRef` are already `pub`. Add these shims in the same Task 3/Task 4 commit. + +- [ ] **Step 2: Run the parity test** + +Run: `cargo test --test test_readbound_gather test_readbound_reconstructs_union_per_hap 2>&1 | tail -30` +Expected: **PASS**. If a hap mismatches, the failure prints `(r, s, p)` — debug the per-class window vs. union window for that region (usually a `qs < v_end` re-check discrepancy). + +- [ ] **Step 3: Add the zero-union / zero-tree assertion** + +Append: + +```rust +#[test] +fn test_readbound_gather_builds_no_search_tree() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + + let rb = find_ranges(&reader, ®ions, None); + let before = search::search_tree_build_count(); + let _ = gather_ranges_readbound(&reader, &rb); + assert_eq!( + search::search_tree_build_count(), + before, + "gather_ranges_readbound must build zero SearchTrees (no dense_union)" + ); +} +``` + +- [ ] **Step 4: Add a sample-subset parity test (mirrors the `read_ranges` subset oracle)** + +Append: + +```rust +#[test] +fn test_readbound_subset_matches_full_selected_haps() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); // 2 samples, ploidy 2 + let regions = vec![(0u32, 1_000_000u32)]; + + let full = gather_ranges_readbound(&reader, &find_ranges(&reader, ®ions, None)); + // Select only sample 1. + let sub = gather_ranges_readbound(&reader, &find_ranges(&reader, ®ions, Some(&[1]))); + assert_eq!(sub.n_samples, 1); + for p in 0..reader_ploidy(&reader) { + let a = readbound_decode_hap(&sub, &reader, 0, 0, p); // selected slot 0 == orig sample 1 + let b = readbound_decode_hap(&full, &reader, 0, 1, p); // orig sample 1 + assert_eq!(a, b, "subset ploid {p}"); + } +} + +fn reader_ploidy(_r: &ContigReader) -> usize { 2 } +``` + +- [ ] **Step 5: Add the flat-gather parity test (flat ≡ cartesian for full cohort)** + +Append — proves `gather_haps_readbound` (gvl's primitive) agrees with the cartesian `gather_ranges_readbound` when the flat queries enumerate the full R×S' cohort: + +```rust +#[test] +fn test_flat_gather_matches_cartesian_full_cohort() { + let tmp = tempdir().unwrap(); + let out = tmp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let reader = synth_reader(&out); // 2 samples, ploidy 2 + let regions = vec![(0u32, 1_000_000u32), (250u32, 400u32)]; + let ploidy = 2usize; + + let rb = find_ranges(&reader, ®ions, None); + let cart = gather_ranges_readbound(&reader, &rb); + + // Enumerate flat queries in the SAME order cart lays out haps: + // region-major, samples 0..S, so query q = r*S + s, orig sample = s. + let s_n = rb.n_samples; + let mut region_starts = Vec::new(); + let mut orig_samples = Vec::new(); + let mut vk_snp_range = Vec::new(); + let mut vk_indel_range = Vec::new(); + let mut dsr = Vec::new(); + let mut dir_ = Vec::new(); + for r in 0..regions.len() { + for s in 0..s_n { + region_starts.push(rb.region_starts[r]); + orig_samples.push(rb.sample_cols[s]); + dsr.push(rb.dense_snp_range[r]); + dir_.push(rb.dense_indel_range[r]); + for p in 0..ploidy { + let row = r * (s_n * ploidy) + s * ploidy + p; + vk_snp_range.push(rb.vk_snp_range[row]); + vk_indel_range.push(rb.vk_indel_range[row]); + } + } + } + let flat = gather_haps_readbound( + &reader, ®ion_starts, &orig_samples, + &vk_snp_range, &vk_indel_range, &dsr, &dir_, ploidy, + ); + + // Compare decoded per-hap. cart hap (r,s,p) == flat query q=r*S+s, ploid p. + for r in 0..regions.len() { + for s in 0..s_n { + for p in 0..ploidy { + let a = readbound_decode_hap(&cart, &reader, r, s, p); + let b = readbound_decode_hap(&flat, &reader, r * s_n + s, 0, p); + assert_eq!(a, b, "flat vs cartesian (r={r}, s={s}, p={p})"); + } + } + } +} +``` + +- [ ] **Step 6: Run the full new test file** + +Run: `cargo test --test test_readbound_gather 2>&1 | tail -30` +Expected: all five tests PASS. + +- [ ] **Step 7: Run the full suite (additive guarantee)** + +Run: `cargo test 2>&1 | tail -40` +Expected: entire genoray suite green — the shipped union path and every existing test unchanged. + +- [ ] **Step 8: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/query.rs src/lib.rs tests/test_readbound_gather.rs +git commit -m "test(query): read-bound gather parity vs union+decode, zero-tree, subset + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Python surface — `find_ranges` dict exposes the per-class ranges + +**Why:** gvl's **write** path calls genoray's Python `find_ranges(...)` and streams the resulting arrays into the cache (Plan 2, Task 2). The read path is pure-Rust and needs no Python surface, but the write cache needs `dense_snp_range` / `dense_indel_range` in the `find_ranges` dict. + +**Files:** +- Modify: `src/py_query_ranges.rs` — `bundle_to_dict` (`:~28`) and `bundle_from_dict` (`:~120`). + +**Interfaces:** +- Produces: the Python `find_ranges(...)` / `read_ranges(...)` dict gains keys `dense_snp_range` and `dense_indel_range`, each an `(R, 2)` `int64` numpy array. Existing keys byte-unchanged. + +- [ ] **Step 1: Write the failing Python parity test** + +Create `tests/test_py_ranges_readbound.py` (run via the built wheel — this is a Python test, deferred to after Task 6's wheel build; write it now, run it in Task 6): + +```python +import numpy as np +# genoray._core.PyContigReader is constructed the same way the existing +# py_query_ranges tests do; reuse that harness path if one exists in genoray's +# python test suite. Placeholder assertion of the new keys: +def _assert_keys(d): + assert "dense_snp_range" in d and "dense_indel_range" in d + for k in ("dense_snp_range", "dense_indel_range"): + a = np.asarray(d[k]) + assert a.ndim == 2 and a.shape[1] == 2 and a.dtype == np.int64 +``` + +- [ ] **Step 2: Add the keys to `bundle_to_dict`** + +In `src/py_query_ranges.rs`, in `bundle_to_dict`, next to where `dense_range` is inserted, add (mirror the exact `(R,2)` i64 conversion used for `dense_range`): + +```rust + let snp = PyArray2::from_vec2( + py, + &rb.dense_snp_range.iter().map(|&(s, e)| vec![s as i64, e as i64]).collect::>(), + )?; + dict.set_item("dense_snp_range", snp)?; + let indel = PyArray2::from_vec2( + py, + &rb.dense_indel_range.iter().map(|&(s, e)| vec![s as i64, e as i64]).collect::>(), + )?; + dict.set_item("dense_indel_range", indel)?; +``` + +> Match the *existing* `dense_range` serialization idiom in this file exactly (it may use `PyArray2::from_owned_array` over an `Array2`); if so, build `Array2::from_shape_vec((R, 2), flat)` the same way rather than `from_vec2`. Read the `dense_range` block first and copy its shape/dtype path. + +- [ ] **Step 3: Add the keys to `bundle_from_dict`** + +If `bundle_from_dict` round-trips (used by the Rust dict-parity test), parse the two new keys back into `Vec<(usize,usize)>` mirroring `dense_range`'s parse. If `bundle_from_dict` is only used for the union oracle and ignores unknown keys, guard the parse with `if let Some(...) = dict.get_item(...)` so old dicts still load. + +- [ ] **Step 4: Build + confirm the Rust dict-parity test still passes** + +Run: `cargo test --test test_ranges_split 2>&1 | tail -20` +Expected: `assert_payload_dicts_eq` still green (it checks only the original key set; new keys are additive). + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/py_query_ranges.rs tests/test_py_ranges_readbound.py +git commit -m "feat(py): find_ranges dict exposes dense_snp_range/dense_indel_range + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Build the local wheel + confirm downstream crate build + +**Files:** none (build/verification only). + +- [ ] **Step 1: Build the wheel with default features (conversion on)** + +Run (in the genoray pixi/venv used to build the wheel): +`cd /carter/users/dlaub/projects/genoray && pixi run -e dev maturin develop --release 2>&1 | tail -20` *(use genoray's actual build task; if it uses `maturin build`, produce the wheel and note its path)* +Expected: wheel builds; `python -c "import genoray"` works. + +- [ ] **Step 2: Run the Python range test from Task 5** + +Run: `pytest tests/test_py_ranges_readbound.py -q 2>&1 | tail -20` +Expected: PASS (new dict keys present, correct shape/dtype). + +- [ ] **Step 3: Confirm the query-only crate builds for the downstream path-dep** + +Run: `cargo build --no-default-features --release 2>&1 | tail -10` +Expected: clean — this is exactly what gvl's `genoray_core = { path = ..., default-features = false }` compiles (Plan 2, Task 3). + +- [ ] **Step 4: Record the exact HEAD commit for the sync contract** + +Run: `git rev-parse HEAD` +Record the commit hash in the PR description and in Plan 2's Task 3 (the gvl path-dep and this wheel MUST be this commit). + +- [ ] **Step 5: Commit any build-config changes** (only if `pixi.toml` / CI touched; otherwise skip). + +--- + +## Task 7: genoray docs / roadmap + +**Files:** +- Modify: genoray's migration/roadmap doc (search `docs/` for the search/gather-split roadmap entry). + +- [ ] **Step 1: Mark the read-bound gather + conversion feature** + +Add a roadmap entry: `conversion` query-only build ✅; per-class `find_ranges` ranges + `gather_ranges_readbound` + `BatchResultSplit` ✅ (parity vs union & `decode_hap`, zero-tree control); note it is additive to the shipped split. Link this plan. + +- [ ] **Step 2: Commit** + +```bash +git add docs/ +git commit -m "docs: read-bound per-class gather + conversion feature roadmap + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Self-Review Notes (traceability to the spec) + +- **Spec A1 (conversion feature)** → Task 1. Correction applied: gate `lib.rs:40-52` (`index_bcf_csi`/`index_vcf`) too, not only `vcf_reader.rs`. +- **Spec A2 (per-class ranges + read-bound gather + split-dense BatchResult)** → Tasks 2 (ranges), 3 (`gather_ranges_readbound` + `BatchResultSplit`), 5 (Python surface for the write cache). +- **Spec "parity vs union & decode"** → Task 4 (decoded-per-hap parity, zero-tree control, subset oracle). +- **Spec "shipped union path byte-unchanged"** → every task re-runs `test_ranges_split` / full suite; new code is additive structs/fields/functions only. +- **Spec "build local wheel + crate"** → Task 6. +- **Open question (channel factoring)** → resolved as: var_key merged per hap (unchanged) + dense split per class in `BatchResultSplit`. gvl consumes this exact shape (Plan 2, Task 4). diff --git a/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md b/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md new file mode 100644 index 00000000..89f2eda7 --- /dev/null +++ b/docs/superpowers/plans/2026-07-04-svar2-gvl-readbound-wiring.md @@ -0,0 +1,981 @@ +# SVAR2 gvl Read-Bound Dataset Wiring — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the `.svar2` variant source into `gvl.write()` (a write-time 6-array offsets cache, dataset samples only) and `gvl.Dataset.__getitem__` (one all-Rust FFI call per read that gathers off the cache with **no interval-search-tree rebuild and no dense-union rebuild**), so all four output modes (haplotypes, tracks, variants, variant-windows) reconstruct in Rust, matching the SVAR1 read path. + +**Architecture:** Mirror the SVAR1 write path (`_write_from_svar` → `offsets.npy` + `svar_meta.json` + `SvarLink`). At write, per contig call genoray's Python `find_ranges(samples=dataset)` (Plan 1) and stream the 6 arrays into memmaps under `genotypes/svar2_ranges/`. At read, gvl's Rust links `genoray_core` (query-only path-dep), opens a `ContigReader` per contig **once** at `Dataset.open` (an `Svar2Store` pyclass), and per read builds a `RangesBundle`-equivalent from the cached memmap slice → calls genoray's flat `gather_haps_readbound` → gets a split-dense `BatchResultSplit` → merges `var_key ⋈ dense_snp ⋈ dense_indel` and reconstructs in Rust, decoding keys inline via `svar2-codec`. The SVAR1 path is byte-unchanged. + +**Tech Stack:** Python 3.10+ (pydantic, polars, numpy), Rust 2024 (PyO3 0.28 abi3-py310, `numpy` 0.28, `svar2-codec` path-dep, **new** `genoray_core` path-dep), `genoray` (local wheel), `pixi -e dev`, `maturin develop --release`. + +**Depends on:** the genoray PR in `docs/superpowers/plans/2026-07-04-svar2-genoray-readbound-gather.md` — MUST be merged and the local wheel + crate built from the **same commit** first. Record that commit; the gvl path-dep and the genoray Python wheel must match it (the `RangesBundle`/`BatchResultSplit` field layout is the contract). + +## Global Constraints + +- **Byte-identical parity contract.** For any `contig, regions, samples` and every output mode: read-bound reconstruct ≡ the existing union-based `SparseVar2Source.reconstruct`/`realign_tracks` (which use genoray `overlap_batch`) ≡ genoray `decode` oracle ≡ SVAR1 output for an equivalent dataset — byte-for-byte / field-for-field. +- **Additive.** The SVAR1 gvl path (`_write_from_svar`, `Haps` SVAR1 branch, `reconstruct_haplotypes_fused`, etc.) stays byte-unchanged; full SVAR1 regression green (`pixi run -e dev pytest tests -q` + `cargo test`). The existing standalone `SparseVar2Source` (union path) stays as the parity oracle until Task 8 retires only its live dispatch. +- **Write caches only the dataset's samples `S'`.** `gvl.write` already selects the sample set; the cache is sized to `S'`, not the full `.svar2` cohort — exactly like `_write_from_svar`. +- **Rebuild Rust before Python tests.** After any `src/` edit: `pixi run -e dev maturin develop --release` BEFORE `pixi run -e dev pytest` — otherwise pytest imports the stale `.so`. `cargo test` compiles from source and is unaffected. +- **Full-tree before pushing shared-code changes.** `_write.py`, `_haps.py`, `_open.py`, `_reconstruct.py`, `_impl.py` are shared; run `pixi run -e dev pytest tests -q` (dataset **and** unit), not a scoped subset, before pushing. +- **Lint gate.** `pixi run -e dev ruff check python/ tests/` + `pixi run -e dev ruff format python/ tests/` + `pixi run -e dev typecheck`; `cargo fmt` + `cargo clippy --all-targets` for Rust. +- **Docs/skill gates.** `.svar2` becomes a public `write` variant source ⇒ update `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, `README.md`; keep `api.md` in sync with any new `__all__` symbol (Task 9). +- **Reject unsupported variants** (symbolic/breakend) exactly as SVAR1 does (`_reject_unsupported_variants`). +- **genoray repo path is absolute:** `/carter/users/dlaub/projects/genoray` (there is no `../genoray`). gvl already path-deps `svar2-codec = { path = "/carter/users/dlaub/projects/genoray/svar2-codec" }`. + +--- + +## File Structure + +- `python/genvarloader/_dataset/_svar2_link.py` — **new**; `Svar2Link`/`Svar2Fingerprint`/`_resolve_svar2`/`_verify_svar2_fingerprint`. Models `_svar_link.py`. *(Task 1)* +- `python/genvarloader/_dataset/_write.py` — add `.svar2` coercion arm (`:~225`), `SparseVar2` dispatch arm (`:~325`), `_write_from_svar2`, `Metadata.svar2_link` field (`:86-98`). *(Tasks 1, 2)* +- `Cargo.toml` — add `genoray_core = { path = "/carter/users/dlaub/projects/genoray", default-features = false }`. *(Task 3)* +- `src/svar2/store.rs` — **new**; `Svar2Store` pyclass wrapping `HashMap`. *(Task 3)* +- `src/svar2/mod.rs` — add `merge_hap3` (3-source merge). *(Task 4)* +- `src/ffi/mod.rs` — `reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, `decode_variants_from_svar2_readbound`. *(Tasks 4, 5, 6)* +- `src/reconstruct/mod.rs` — internal `reconstruct_haplotypes_from_split` (consumes `BatchResultSplit`). *(Task 4)* +- `src/tracks/mod.rs` — internal `shift_and_realign_tracks_from_split`. *(Task 5)* +- `src/lib.rs` — register the new pyclass + pyfunctions. *(Tasks 3–6)* +- `python/genvarloader/_dataset/_haps.py` — `Haps` source discriminant (`svar` vs `svar2`); route `_reconstruct_haplotypes` / variants; open `Svar2Store`. *(Tasks 4, 6, 7)* +- `python/genvarloader/_dataset/_open.py` — thread `svar2_link`/`svar2` override through `_build_seqs`. *(Task 7)* +- `python/genvarloader/_dataset/_impl.py` — `Dataset.open(svar2=...)` override param. *(Task 7)* +- `python/genvarloader/_dataset/_reconstruct.py` — `HapsTracks` routes to the svar2 track kernel when source is svar2. *(Task 5)* +- `python/genvarloader/_dataset/_svar2_source.py` — retire live `overlap_batch` dispatch; keep as parity oracle only. *(Task 8)* + +--- + +## Task 1: `_svar2_link.py` + `Metadata.svar2_link` field + +**Files:** +- Create: `python/genvarloader/_dataset/_svar2_link.py` +- Modify: `python/genvarloader/_dataset/_write.py:86-98` (`Metadata`) +- Test: `tests/unit/dataset/test_svar2_link.py` (new) + +**Interfaces:** +- Produces: + - `class Svar2Fingerprint(BaseModel)`: `n_variants: int`, `store_bytes: int`. + - `class Svar2Link(BaseModel)`: `relative_path: str`, `absolute_path: str`, `fingerprint: Svar2Fingerprint`. + - `def _resolve_svar2(gvl_path: Path, link: Svar2Link | None, override: Path | str | None) -> Path` — override → link.relative → link.absolute → sibling `*.svar2`. + - `def _verify_svar2_fingerprint(svar2_path: Path, link: Svar2Link | None) -> None` — no-op if `link is None`; else compare `n_variants` (from the `.svar2` index) + a canonical store-file byte count; raise `ValueError` on mismatch. + - `Metadata` gains `svar2_link: Svar2Link | None = None`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/unit/dataset/test_svar2_link.py`: + +```python +from pathlib import Path +import pytest +from genvarloader._dataset._svar2_link import ( + Svar2Link, Svar2Fingerprint, _resolve_svar2, _verify_svar2_fingerprint, +) + + +def test_resolve_prefers_override(tmp_path: Path): + real = tmp_path / "cohort.svar2" + real.mkdir() + link = Svar2Link(relative_path="nope.svar2", absolute_path="/nope.svar2", + fingerprint=Svar2Fingerprint(n_variants=1, store_bytes=1)) + assert _resolve_svar2(tmp_path, link, real) == real + + +def test_resolve_missing_override_raises(tmp_path: Path): + with pytest.raises(FileNotFoundError): + _resolve_svar2(tmp_path, None, tmp_path / "absent.svar2") + + +def test_verify_none_link_is_noop(tmp_path: Path): + _verify_svar2_fingerprint(tmp_path, None) # must not raise +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: FAIL — `ModuleNotFoundError: _svar2_link`. + +- [ ] **Step 3: Write `_svar2_link.py`** + +Create the file. Model it on `_svar_link.py` (`_resolve_svar`/`_verify_fingerprint`), changing the fingerprint to the `.svar2` store's stable identity. The `.svar2` index is at `/index.arrow` (confirm the exact filename by inspecting a real `.svar2`; genoray's `SparseVar2` exposes `.index` — the n_variants source); the canonical byte count is the summed size of the on-disk dense/var_key store files: + +```python +"""Resolution and integrity for the GVL dataset → .svar2 back-reference. + +Mirrors _svar_link.py; the fingerprint keys on the .svar2 store's stable identity +(variant count + a canonical store-file byte count) rather than SVAR1's +variant_idxs.npy, which .svar2 does not have. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_variants: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +def _svar2_store_bytes(svar2_path: Path) -> int: + """Canonical, stable byte count of the .svar2 on-disk stores. Sum the sizes of + the packed dense + var_key key files across contigs, sorted for determinism.""" + total = 0 + for p in sorted(svar2_path.rglob("*")): + if p.is_file() and p.suffix in {".bin", ".npy"} and "keys" in p.name: + total += p.stat().st_size + return total + + +def _svar2_n_variants(svar2_path: Path) -> int: + import polars as pl + # .svar2 index; confirm filename against a real store (SparseVar2().index). + return pl.scan_ipc(svar2_path / "index.arrow").select(pl.len()).collect().item() + + +def _resolve_svar2( + gvl_path: Path, link: "Svar2Link | None", override: "Path | str | None" +) -> Path: + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_svar2_fingerprint(svar2_path: Path, link: "Svar2Link | None") -> None: + if link is None: + return + n_obs = _svar2_n_variants(svar2_path) + bytes_obs = _svar2_store_bytes(svar2_path) + exp = link.fingerprint + mismatches: list[str] = [] + if n_obs != exp.n_variants: + mismatches.append(f"n_variants: expected {exp.n_variants}, observed {n_obs}") + if bytes_obs != exp.store_bytes: + mismatches.append(f"store_bytes: expected {exp.store_bytes}, observed {bytes_obs}") + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) + + +def make_svar2_link(gvl_path: Path, svar2_path: Path) -> Svar2Link: + svar2_resolved = svar2_path.resolve() + return Svar2Link( + relative_path=os.path.relpath(svar2_resolved, start=gvl_path).replace(os.sep, "/"), + absolute_path=str(svar2_resolved), + fingerprint=Svar2Fingerprint( + n_variants=_svar2_n_variants(svar2_resolved), + store_bytes=_svar2_store_bytes(svar2_resolved), + ), + ) +``` + +> **Confirm before finalizing:** open a real `.svar2` (the MVP fixture) and verify (a) the index filename used by `_svar2_n_variants` and (b) that `_svar2_store_bytes`'s glob captures ≥1 stable file. Adjust the patterns to the actual layout; the contract is only that the count is deterministic and changes iff the store changes. + +- [ ] **Step 4: Add the `Metadata` field** + +In `python/genvarloader/_dataset/_write.py`, add the import near the `SvarLink` import (`:40`): + +```python +from ._svar2_link import Svar2Link +``` + +and add to `Metadata` (`:86-98`), after `svar_link`: + +```python + svar2_link: Svar2Link | None = None +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pixi run -e dev pytest tests/unit/dataset/test_svar2_link.py -q` +Expected: PASS. +Run: `python -c "from genvarloader._dataset._write import Metadata; print(Metadata.model_fields['svar2_link'])"` +Expected: prints the optional field (default None) — confirms backward/forward compat (no format bump needed; `_check_dataset_format_version` gates only on major, and old datasets fill the default). + +- [ ] **Step 6: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_svar2_link.py python/genvarloader/_dataset/_write.py tests/unit/dataset/test_svar2_link.py +git commit -m "feat(dataset): Svar2Link resolution/fingerprint + Metadata.svar2_link + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: `_write_from_svar2` + write dispatch (the 6-array cache) + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py` — coercion arm (`:216-234`), dispatch arm (`:285-333`), add `_write_from_svar2`. +- Test: `tests/dataset/test_write_svar2.py` (new). + +**Interfaces:** +- Consumes: genoray `SparseVar2` with `.ploidy`, `.index`, and a per-contig batched `find_ranges(contig, starts, ends, samples=...) -> dict` returning the 6 arrays: `vk_snp_range (R,S',P,2)`, `vk_indel_range (R,S',P,2)`, `dense_snp_range (R,2)`, `dense_indel_range (R,2)`, `region_starts (R,)`, `sample_cols (S',)` (Plan 1 Task 5 added `dense_snp_range`/`dense_indel_range` to the dict). +- Produces: `def _write_from_svar2(path: Path, bed: pl.DataFrame, svar2: "SparseVar2", samples: list[str], extend_to_length: bool) -> tuple[pl.DataFrame, Svar2Link]`. Writes memmaps under `path/genotypes/svar2_ranges/` + `svar2_meta.json`; returns the extended bed (max_ends folded in) and the `Svar2Link`. + +> **genoray dependency check (do first):** confirm `SparseVar2.find_ranges(contig, starts, ends, samples=...)` exists in the local genoray wheel and returns the 6-key dict. If genoray only exposes `find_ranges` on `PyContigReader` (per-contig, no batched Python entry on `SparseVar2`), add a thin `SparseVar2.find_ranges` wrapper in genoray that dispatches to the contig's `PyContigReader.find_ranges` — a small genoray-side addition; fold it into the genoray PR (Plan 1 Task 5) rather than marshalling in gvl. + +- [ ] **Step 1: Write the failing test** + +Create `tests/dataset/test_write_svar2.py`. Reuse the `.svar2` fixture the current `SparseVar2Source` tests use (search `tests/` for an existing `.svar2` path / `SparseVar2(` construction; if none, build one from the MVP `svar2_mvp` store or a synthetic genoray fixture). Skeleton: + +```python +import json +from pathlib import Path +import numpy as np +import polars as pl +import pytest + +import genvarloader as gvl +from genvarloader._dataset._svar2_link import Svar2Link + +SVAR2_FIXTURE = ... # Path to a small .svar2 store (reuse existing test fixture) + + +def test_write_svar2_emits_cache(tmp_path: Path): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + bed = pl.DataFrame({ + "chrom": ["chr1", "chr1"], + "chromStart": [0, 250], + "chromEnd": [1000, 400], + }) + out = tmp_path / "ds.gvl" + gvl.write(out, bed, variants=svar2, samples=None, overwrite=True) + + rd = out / "genotypes" / "svar2_ranges" + meta = json.loads((rd / "svar2_meta.json").read_text()) + assert set(meta) >= { + "vk_snp_range", "vk_indel_range", "dense_snp_range", + "dense_indel_range", "region_starts", "sample_cols", + } + # metadata.json carries the link + ploidy. + md = json.loads((out / "metadata.json").read_text()) + assert md["svar2_link"] is not None + assert md["ploidy"] == svar2.ploidy + Svar2Link.model_validate(md["svar2_link"]) # shape check +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: FAIL — `gvl.write` raises "unrecognized file extension" / doesn't dispatch `SparseVar2`. + +- [ ] **Step 3: Add the `.svar2` coercion arm** + +In `_write.py`, add the import near the `SparseVar` import (`:19`): change `from genoray import PGEN, VCF, Reader, SparseVar` to also import `SparseVar2`: + +```python +from genoray import PGEN, VCF, Reader, SparseVar, SparseVar2 +``` + +In the coercion block (`:216-234`), add an arm before the `else` that raises: + +```python + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) +``` + +- [ ] **Step 4: Add the dispatch arm** + +In the genotype-writing dispatch (`:325-330`), after the `SparseVar` branch, add: + +```python + elif isinstance(variants, SparseVar2): + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link +``` + +(`metadata["ploidy"] = variants.ploidy` at `:330` already runs for all sources, including `SparseVar2`.) + +- [ ] **Step 5: Write `_write_from_svar2`** + +Add the function near `_write_from_svar` (`:961`). It streams the 6 arrays per contig into memmaps and computes `max_ends` for the bed (mirroring SVAR1's end-extension). Because the read-bound gather needs per-`(region, selected-sample, ploid)` var_key ranges and per-region dense ranges, the memmaps are shaped `(R, S', P, 2)` / `(R, 2)` / `(R,)` / `(S',)`: + +```python +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: "SparseVar2", + samples: list[str], + extend_to_length: bool, +) -> "tuple[pl.DataFrame, Svar2Link]": + _reject_unsupported_variants(svar2.index, "SVAR2") + + out_dir = path / "genotypes" / "svar2_ranges" + out_dir.mkdir(parents=True, exist_ok=True) + + R, S, P = bed.height, len(samples), svar2.ploidy + vk_snp = np.memmap(out_dir / "vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap(out_dir / "vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + dense_snp = np.memmap(out_dir / "dense_snp_range.npy", np.int64, "w+", shape=(R, 2)) + dense_indel = np.memmap(out_dir / "dense_indel_range.npy", np.int64, "w+", shape=(R, 2)) + region_starts = np.memmap(out_dir / "region_starts.npy", np.int64, "w+", shape=(R,)) + # sample_cols: selected slot -> original sample index (same for every contig). + sample_cols_full = np.asarray( + [svar2.available_samples.index(s) for s in samples], np.int64 + ) + np.save(out_dir / "sample_cols.npy", sample_cols_full) + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "vk_snp_range": {"shape": [R, S, P, 2], "dtype": " **`max_ends` NOTE.** SVAR1 derives `max_ends` from the per-region max `v_idx` and `v_ends[v_idx]`. For SVAR2 the equivalent is the farthest `v_end` among the region's overlapping variants (var_key + dense) for the dataset's samples. If genoray exposes a helper (`_region_max_ends` / a field on the `find_ranges` dict), use it. If not, add `max_end_range` per region to genoray's `find_ranges` dict (small genoray addition, fold into Plan 1 Task 5) rather than recomputing in Python. Confirm the exact source before implementing; do **not** guess a formula — measure it against `_write_from_svar`'s `max_ends` on a shared fixture (Step 7). + +- [ ] **Step 6: Rebuild not needed (Python-only) — run the write test** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS (cache + meta + link written). + +- [ ] **Step 7: Add a max_ends parity assertion** + +Add to the test file: write the *same* regions from an equivalent `.svar` (SVAR1) and `.svar2` of the same cohort (the MVP fixtures are matched); assert the two extended `chromEnd` columns are equal. This pins the end-extension semantics. + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q` +Expected: PASS. + +- [ ] **Step 8: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "feat(write): _write_from_svar2 6-array ranges cache + dispatch + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: gvl links `genoray_core`; `Svar2Store` pyclass + +**Files:** +- Modify: `Cargo.toml` (`[dependencies]`) +- Create: `src/svar2/store.rs` +- Modify: `src/svar2/mod.rs` (`pub mod store;`), `src/lib.rs` (register pyclass) +- Test: `tests/unit/dataset/test_svar2_store.py` (new) + +**Interfaces:** +- Produces: `Svar2Store` pyclass. Python: `Svar2Store(store_path: str, contigs: list[str], n_samples: int, ploidy: int)` opens one `genoray_core::query::ContigReader` per contig, held for the store's lifetime (the SVAR2 analog of SVAR1's once-built `_HapsFfiStatic`). Rust-internal: `fn reader(&self, contig: &str) -> &ContigReader`. + +- [ ] **Step 1: Add the `genoray_core` path-dep** + +In `Cargo.toml` `[dependencies]`, after the `svar2-codec` line, add: + +```toml +genoray_core = { path = "/carter/users/dlaub/projects/genoray", default-features = false } +``` + +`default-features = false` drops `conversion` (htslib) and `extension-module`, yielding the query-only core (Plan 1 Task 1). Confirm the genoray checkout is at the commit recorded in Plan 1 Task 6. + +- [ ] **Step 2: Write the failing test** + +Create `tests/unit/dataset/test_svar2_store.py`: + +```python +import pytest +from genvarloader.genvarloader import Svar2Store # compiled ext + +SVAR2_FIXTURE = ... # same fixture as Task 2 + + +def test_store_opens_contigs(): + store = Svar2Store(str(SVAR2_FIXTURE), ["chr1"], n_samples=2, ploidy=2) + assert store.contigs() == ["chr1"] +``` + +- [ ] **Step 3: Write `src/svar2/store.rs`** + +```rust +use std::collections::HashMap; + +use genoray_core::query::ContigReader; +use pyo3::exceptions::PyIOError; +use pyo3::prelude::*; + +/// Opened once at Dataset.open; holds one query-only ContigReader per contig for +/// the store's lifetime (SVAR2 analog of SVAR1's cached _HapsFfiStatic). +#[pyclass] +pub struct Svar2Store { + readers: HashMap, +} + +impl Svar2Store { + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } +} + +#[pymethods] +impl Svar2Store { + #[new] + fn new(store_path: &str, contigs: Vec, n_samples: usize, ploidy: usize) -> PyResult { + let mut readers = HashMap::with_capacity(contigs.len()); + for c in contigs { + let r = ContigReader::open(store_path, &c, n_samples, ploidy) + .map_err(|e| PyIOError::new_err(format!("open contig {c}: {e}")))?; + readers.insert(c, r); + } + Ok(Self { readers }) + } + + fn contigs(&self) -> Vec { + let mut v: Vec = self.readers.keys().cloned().collect(); + v.sort(); + v + } +} +``` + +Add `pub mod store;` to `src/svar2/mod.rs`, and in `src/lib.rs`'s `#[pymodule]` add `m.add_class::()?;`. + +- [ ] **Step 4: Build + run** + +Run: `cargo build 2>&1 | tail -20` (first build compiles genoray_core query-only; may take a bit). +Then: `pixi run -e dev maturin develop --release 2>&1 | tail -20` +Then: `pixi run -e dev pytest tests/unit/dataset/test_svar2_store.py -q` +Expected: PASS. + +- [ ] **Step 5: fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add Cargo.toml Cargo.lock src/svar2/store.rs src/svar2/mod.rs src/lib.rs tests/unit/dataset/test_svar2_store.py +git commit -m "feat(rust): link genoray_core (query-only) + Svar2Store pyclass + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Read-bound haplotype kernel (all-Rust, one FFI call) + +**Files:** +- Modify: `src/svar2/mod.rs` — add `merge_hap3`. +- Modify: `src/reconstruct/mod.rs` — add `reconstruct_haplotypes_from_split`. +- Modify: `src/ffi/mod.rs` — add `reconstruct_haplotypes_from_svar2_readbound`. +- Modify: `src/lib.rs` — register it. +- Test: `tests/dataset/test_svar2_readbound_haps.py` (parity vs union oracle), `src/svar2/mod.rs` `#[cfg(test)]` (merge_hap3 unit). + +**Interfaces:** +- Consumes: `genoray_core::query::{gather_haps_readbound, BatchResultSplit, KeyRef}`; `ContigReader::lut_arrays()` (in-Rust LUT, no Python marshalling); existing `svar2::decode_alt`, `reconstruct::reconstruct_haplotype_core`. +- Produces the FFI pyfunction: + ``` + reconstruct_haplotypes_from_svar2_readbound( + store: &Svar2Store, contig: &str, + region_starts: (n_q,) u32, orig_samples: (n_q,) i64, + vk_snp_range: (n_q*P, 2) i64, vk_indel_range: (n_q*P, 2) i64, + dense_snp_range: (n_q, 2) i64, dense_indel_range: (n_q, 2) i64, + region_bounds: (n_q, 2) i32, # [start, end) per query, post-jitter + shifts: (n_q, P) i32, + ref_: (n_ref,) u8, ref_offsets: (n_contig+1,) i64, + pad_char: u8, output_length: i64, parallel: bool, + ) -> (out_data u8, out_offsets i64) + ``` + `n_q` = number of `(region, sample)` query rows; ploidy `P` inferred from `shifts.shape[1]`. + +- [ ] **Step 1: Write the `merge_hap3` unit test** + +In `src/svar2/mod.rs` `#[cfg(test)]`, add: + +```rust +#[test] +fn merge_hap3_is_position_sorted_stable() { + // vk at pos 5,20; dense_snp at 10; dense_indel at 10,30. + let vk = [(5u32, 100u32), (20, 200)]; + let dsnp = [(10u32, 300u32)]; + let dindel = [(10u32, 400u32), (30, 500)]; + let out = merge_hap3(&vk, &dsnp, &dindel); + let positions: Vec = out.iter().map(|&(p, _)| p).collect(); + assert_eq!(positions, vec![5, 10, 10, 20, 30]); + // On the pos-10 tie: vk-source first (none here), then dense_snp before dense_indel. + assert_eq!(out[1], (10, 300)); + assert_eq!(out[2], (10, 400)); +} +``` + +- [ ] **Step 2: Run it (fails to compile)** + +Run: `cargo test svar2::mod 2>&1 | tail -20` (or `cargo test merge_hap3`) +Expected: FAIL — `merge_hap3` undefined. + +- [ ] **Step 3: Write `merge_hap3`** + +In `src/svar2/mod.rs`, add. It mirrors the existing 2-source `merge_hap` (`:30-51`) but takes three already-position-ordered inputs and stable-sorts by position (vk pushed first, then dense_snp, then dense_indel — matching genoray's `merge_keys(vec![vk, dense_snp, dense_indel])` tie order): + +```rust +/// Merge one hap's var_key ⋈ dense_snp ⋈ dense_indel into one position-sorted +/// (pos, key) list. Stable: on a shared position, var_key precedes dense_snp +/// precedes dense_indel — the order genoray's decode oracle uses. +pub fn merge_hap3( + vk: &[(u32, u32)], + dense_snp: &[(u32, u32)], + dense_indel: &[(u32, u32)], +) -> Vec<(u32, u32)> { + let mut a: Vec<(u32, u32)> = Vec::with_capacity(vk.len() + dense_snp.len() + dense_indel.len()); + a.extend_from_slice(vk); + a.extend_from_slice(dense_snp); + a.extend_from_slice(dense_indel); + a.sort_by_key(|&(p, _)| p); // stable + a +} +``` + +- [ ] **Step 4: Run the unit test** + +Run: `cargo test merge_hap3 2>&1 | tail -20` +Expected: PASS. + +- [ ] **Step 5: Write the internal `reconstruct_haplotypes_from_split`** + +In `src/reconstruct/mod.rs`, add a function consuming a `BatchResultSplit`. It mirrors the existing `reconstruct_haplotypes_from_svar2` (`:611`) but sources per-hap merged keys from the split result via `merge_hap3` (extracting present dense entries per hap from the presence bitmasks), and decodes via `svar2::decode_alt` with LUT bytes from the reader. Signature: + +```rust +use genoray_core::query::BatchResultSplit; + +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_split( + mut out: ArrayViewMut1, + out_offsets: ArrayView1, + region_bounds: ArrayView2, // (n_q, 2) + shifts: ArrayView2, // (n_q, P) + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], + ref_: ArrayView1, + ref_offsets: ArrayView1, + pad_char: u8, + parallel: bool, +) { + // For each query q and ploid p (hap h = q*P + p): + // vk_h = br.vk[br.vk_off[h]..br.vk_off[h+1]] -> (pos, key) + // dsnp = present entries of br.dense_snp[br.dense_snp_range[q]] via br.dense_snp_present + // dind = present entries of br.dense_indel[br.dense_indel_range[q]] via br.dense_indel_present + // merged = svar2::merge_hap3(&vk_h, &dsnp, &dind) + // provide(i) decodes merged[i].key via svar2::decode_alt(key, lut_bytes, lut_off), + // patching an empty pure-DEL alt with the anchor ref[pos] byte (as the existing + // reconstruct_haplotypes_from_svar2 does at :690-710). + // reconstruct_haplotype_core(merged.len(), provide, shift, contig_ref, ref_start, + // out_view, pad_char, None, None, None) + // Parallel path: split_at_mut chain over out, one disjoint chunk per hap (mirror + // reconstruct_haplotypes_from_svar2's parallel arm at :740-879). + // ... +} +``` + +Extract "present dense entries for hap h in query q" with a small helper: + +```rust +fn present_dense( + dense: &[genoray_core::query::KeyRef], + range: (usize, usize), + present: &[u8], + bit0: usize, +) -> Vec<(u32, u32)> { + let (s, e) = range; + let mut out = Vec::new(); + for (k, j) in (s..e).enumerate() { + if genoray_core::bits_get_bit(present, bit0 + k) { + out.push((dense[j].position, dense[j].key)); + } + } + out +} +``` + +(`genoray_core::bits_get_bit` is the shim added in Plan 1 Task 4; `KeyRef` fields `position`/`key` are `pub`.) + +> **Sizing pass.** Before reconstruct, size outputs exactly like `reconstruct_haplotypes_from_svar2`: compute per-hap applied-ilen diffs (a `hap_diffs_split` analog of `svar2::hap_diffs_svar2` operating over the merged keys), prefix-sum to `out_offsets`, `uninit_output`. Reuse `svar2::hap_diffs_svar2`'s clipping logic; it only needs `(pos, ilen)` per merged key, which `decode_alt` yields as `v_diff`. + +- [ ] **Step 6: Write the FFI `reconstruct_haplotypes_from_svar2_readbound`** + +In `src/ffi/mod.rs`, mirror the fused SVAR1 entry (`:618`) / the existing `reconstruct_haplotypes_from_svar2` (`:768`). Body: look up `reader = store.reader(contig)`; convert the `(n,2)` range arrays to `Vec<(usize,usize)>` and `orig_samples` to `Vec`; call `genoray_core::query::gather_haps_readbound(reader, ®ion_starts, &orig_samples, &vk_snp_range, &vk_indel_range, &dense_snp_range, &dense_indel_range, ploidy)`; get `(lut_bytes, lut_off)` from `reader.lut_arrays()`; run the sizing pass; allocate; call `reconstruct_haplotypes_from_split`; return `(out_data, out_offsets)`. Register in `src/lib.rs`. + +- [ ] **Step 7: Write the parity test (vs the union oracle)** + +Create `tests/dataset/test_svar2_readbound_haps.py`. The oracle is the existing `SparseVar2Source.reconstruct` (genoray `overlap_batch`, union path) — the spec's byte-identical contract: + +```python +import numpy as np +from genvarloader._dataset._svar2_source import SparseVar2Source +from genvarloader._dataset._svar2_store_py import build_readbound_haps # thin py wrapper (Task 7) + +SVAR2_FIXTURE = ... # same fixture + + +def test_readbound_haps_match_union_oracle(): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + contig = "chr1" + regions = [(0, 1000), (250, 400), (150, 250)] + ref, ref_offsets, pad = _load_contig_ref(contig) # helper: contig bytes + + union = SparseVar2Source(svar2).reconstruct( + contig, regions, ref, ref_offsets, pad, shifts=None, output_length=-1 + ) + readbound = build_readbound_haps( # opens Svar2Store, slices no cache (direct find_ranges), + svar2, contig, regions, ref, ref_offsets, pad, shifts=None, output_length=-1 + ) + # Ragged equality: same offsets + same bytes. + assert np.array_equal(np.asarray(union.offsets), np.asarray(readbound.offsets)) + assert np.array_equal(union.data.view("u1"), readbound.data.view("u1")) +``` + +> `build_readbound_haps` is a thin Python test helper that, for the given regions, computes the flat per-query `(region_starts, orig_samples, vk/dense ranges)` via `svar2.find_ranges` (full cohort, R×S' flattened to n_q = R*S rows in region-major sample order), opens an `Svar2Store`, and calls `reconstruct_haplotypes_from_svar2_readbound`. It exercises the exact FFI the dataset read path uses (Task 7), decoupled from cache I/O. Put it in a small `python/genvarloader/_dataset/_svar2_store_py.py` alongside the pyclass usage. + +- [ ] **Step 8: Rebuild + run parity** + +Run: `pixi run -e dev maturin develop --release 2>&1 | tail -20` +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py -q` +Expected: PASS (byte-identical to the union oracle). + +- [ ] **Step 9: cargo test + fmt + clippy + commit** + +Run: `cargo test 2>&1 | tail -30` (merge_hap3 + any Rust svar2 tests green). + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add src/svar2/mod.rs src/reconstruct/mod.rs src/ffi/mod.rs src/lib.rs python/genvarloader/_dataset/_svar2_store_py.py tests/dataset/test_svar2_readbound_haps.py +git commit -m "feat(rust): read-bound svar2 haplotype kernel (gather_haps_readbound + merge_hap3) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Read-bound tracks kernel + +**Files:** +- Modify: `src/tracks/mod.rs` (add `shift_and_realign_tracks_from_split`), `src/ffi/mod.rs` (add `shift_and_realign_tracks_from_svar2_readbound`), `src/lib.rs`. +- Modify: `python/genvarloader/_dataset/_svar2_store_py.py` (a `build_readbound_tracks` helper). +- Test: `tests/dataset/test_svar2_readbound_tracks.py`. + +**Interfaces:** +- Produces `shift_and_realign_tracks_from_svar2_readbound(store, contig, , tracks, track_offsets, params, strategy_id, base_seed, parallel) -> (out_data f32, out_offsets i64)`. Tracks need only `ilen`/`deletion_len` per merged key (no allele bytes), so decoding is cheaper — but reuse the **same** `gather_haps_readbound` + `merge_hap3` merge so the two modes read identical variant sets. + +- [ ] **Step 1: Write the failing parity test** + +Create `tests/dataset/test_svar2_readbound_tracks.py`, oracle = `SparseVar2Source(svar2).realign_tracks(...)`: + +```python +def test_readbound_tracks_match_union_oracle(): + from genoray import SparseVar2 + svar2 = SparseVar2(SVAR2_FIXTURE) + contig, regions = "chr1", [(0, 1000), (250, 400)] + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + union = SparseVar2Source(svar2).realign_tracks( + contig, regions, tracks, toff, params, strat, seed, shifts=None) + rb = build_readbound_tracks( + svar2, contig, regions, tracks, toff, params, strat, seed, shifts=None) + import numpy as np + assert np.array_equal(np.asarray(union.offsets), np.asarray(rb.offsets)) + assert np.allclose(union.data, rb.data, equal_nan=True) +``` + +- [ ] **Step 2: Run to confirm it fails** — `pixi run -e dev pytest tests/dataset/test_svar2_readbound_tracks.py -q` → FAIL (`build_readbound_tracks`/FFI missing). + +- [ ] **Step 3: Implement the Rust track-from-split kernel + FFI**, mirroring `shift_and_realign_tracks_from_svar2` (`src/tracks/mod.rs:698`, `src/ffi/mod.rs:897`) but sourcing merged keys from `BatchResultSplit` via `merge_hap3` (ilen only). Register in `src/lib.rs`. + +- [ ] **Step 4: Rebuild + run** — `pixi run -e dev maturin develop --release` then the test → PASS. + +- [ ] **Step 5: fmt + clippy + lint + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +git add src/tracks/mod.rs src/ffi/mod.rs src/lib.rs python/genvarloader/_dataset/_svar2_store_py.py tests/dataset/test_svar2_readbound_tracks.py +git commit -m "feat(rust): read-bound svar2 track re-alignment kernel + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Read-bound variants / variant-windows kernel + +**Files:** +- Modify: `src/ffi/mod.rs` (add `decode_variants_from_svar2_readbound`), `src/lib.rs`. +- Modify: `python/genvarloader/_dataset/_svar2_store_py.py` (`build_readbound_variants`). +- Test: `tests/dataset/test_svar2_readbound_variants.py`. + +**Interfaces:** +- Produces `decode_variants_from_svar2_readbound(store, contig, ) -> RaggedVariants-backing arrays` (per-hap positions, ilens, ALT bytes + offsets), decoding each merged key via `svar2::decode_alt` (`Inline`/`PureDel`/`Lookup`, LUT from `reader.lut_arrays()`), mirroring genoray `decode_hap`. `variant-windows` reuses the same decode + the existing window-materialization gvl already applies for SVAR1 variants. + +- [ ] **Step 1: Write the failing parity test** — oracle: decode the same regions via genoray's `read_ranges(...).decode_hap` per hap (or the existing `SparseVar2Source` variants path if present). Assert per-hap `(positions, ilens, alts)` equal. + +- [ ] **Step 2: Run → FAIL.** + +- [ ] **Step 3: Implement `decode_variants_from_svar2_readbound`** — build `BatchResultSplit` via `gather_haps_readbound`, `merge_hap3` per hap, `decode_alt` each key into the `RaggedVariants` SoA. Register in `src/lib.rs`. + +- [ ] **Step 4: Rebuild + run → PASS.** + +- [ ] **Step 5: fmt + clippy + lint + commit** (message: `feat(rust): read-bound svar2 variants/variant-windows decode`). + +--- + +## Task 7: Dataset read dispatch wiring (Haps discriminant, open, __getitem__) + +**Files:** +- Modify: `python/genvarloader/_dataset/_haps.py` — `Haps.from_path` opens an `Svar2Store` when the dataset is svar2-backed; add a `source` discriminant; `_reconstruct_haplotypes` and the variants path branch on it; slice the cache per query. +- Modify: `python/genvarloader/_dataset/_open.py:143-173` (`_build_seqs`) — thread `svar2_link` + `svar2` override. +- Modify: `python/genvarloader/_dataset/_impl.py:124-199` (`Dataset.open`) — add keyword-only `svar2: str | Path | None = None`. +- Modify: `python/genvarloader/_dataset/_reconstruct.py:130-287` (`HapsTracks.__call__`) — route to the svar2 track kernel when source is svar2. +- Test: `tests/dataset/test_svar2_dataset.py` (end-to-end, all four modes). + +**Interfaces:** +- Consumes: `svar2_ranges/` cache (Task 2), `Svar2Store` (Task 3), the three read-bound kernels (Tasks 4–6), `_resolve_svar2`/`_verify_svar2_fingerprint` (Task 1). +- Produces: a dataset whose `Haps` carries `source: Literal["svar", "svar2"]`; `Dataset.open(path, svar2=)` resolves + fingerprints the `Svar2Link`; `dataset[region, sample]` issues **one** FFI call to the appropriate read-bound kernel — no interval search, no dense-union at read. + +**The cache-slice → FFI mapping (the hot loop).** For a query block of `n_q` rows, each row `q` is a `(region_idx r_q, sample_idx si_q)` pair with post-jitter bounds `[start_q, end_q)`: +- `region_starts[q] = start_q` (post-jitter). +- `orig_samples[q] = sample_cols[si_q]` (from the cache's `sample_cols.npy`). +- `vk_snp_range[q*P + p] = cache.vk_snp_range[r_q, si_q, p]`, likewise `vk_indel_range`. +- `dense_snp_range[q] = cache.dense_snp_range[r_q]`, `dense_indel_range[q] = cache.dense_indel_range[r_q]` (dense is per-region, sample-independent). +- Gather these with numpy fancy-indexing on the memmapped cache (sub-linear; no per-read search), pass to the FFI. + +- [ ] **Step 1: Write the end-to-end test (all four modes) — failing** + +Create `tests/dataset/test_svar2_dataset.py`. Build two datasets from matched MVP fixtures — one from `.svar` (SVAR1), one from `.svar2` — over the same bed/samples/reference, and assert equality per mode: + +```python +import numpy as np +import genvarloader as gvl + + +def _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref): + from genoray import SparseVar, SparseVar2 + d1 = tmp_path / "d1.gvl"; d2 = tmp_path / "d2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture), overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture), overwrite=True) + return gvl.Dataset.open(d1, reference=ref), gvl.Dataset.open(d2, reference=ref) + + +def test_svar2_haplotypes_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +def test_svar2_tracks_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref, bigwig): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_tracks(bigwig)[:, :] + b = ds2.with_tracks(bigwig)[:, :] + assert np.allclose(np.asarray(a), np.asarray(b), equal_nan=True) + + +def test_svar2_variants_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, ref): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants")[:, :] + b = ds2.with_seqs("variants")[:, :] + assert a == b # RaggedVariants equality (positions/ilens/alts) +``` + +> These fixtures (matched `.svar`/`.svar2` of the same cohort, a shared bed, a reference, a bigwig) should be added to `tests/dataset/conftest.py`; reuse the MVP `svar2_mvp` chr21 stores or a small synthetic pair. If a matched SVAR1 store isn't available, use the union-path `SparseVar2Source` as the oracle instead of SVAR1 (weaker cross-format check but still the byte-identical contract). + +- [ ] **Step 2: Run → FAIL** (svar2 dataset opens but reconstructs via the SVAR1 path or errors — no dispatch yet). + +- [ ] **Step 3: Add the `Svar2Store` open + discriminant in `Haps.from_path`** + +In `_haps.py:363`, add a branch: when `path/genotypes/svar2_ranges/svar2_meta.json` exists (the svar2 discriminant, analogous to SVAR1's `svar_meta.json` at `:388`), resolve the `.svar2` via `_resolve_svar2(path, svar2_link, svar2_override)`, `_verify_svar2_fingerprint(...)`, memmap the six cache arrays, open `Svar2Store(str(svar2_path), contigs, n_samples=len(samples), ploidy=ploidy)`, and construct the `Haps` with `source="svar2"` (add the field to `Haps`), the store, and the cache arrays. Leave the existing SVAR1 branch as `source="svar"`. `_build_seqs` already forwards `svar_link`/`svar_override`; add `svar2_link`/`svar2_override` params to `Haps.from_path` and forward them from `_build_seqs`. + +- [ ] **Step 4: Branch `_reconstruct_haplotypes` on `source`** + +In `_haps.py:809`, at the top of `_reconstruct_haplotypes`, if `self.source == "svar2"`, build the flat per-query FFI inputs from the cache (the mapping above) and call `reconstruct_haplotypes_from_svar2_readbound(self.store, contig, ...)`, returning the `Ragged`. Else fall through to the unchanged SVAR1 `reconstruct_haplotypes_fused` path. Do the same source-branch for the variants path (Task 6 kernel) and, in `_reconstruct.py`'s `HapsTracks.__call__`, for the track kernel (Task 5). + +> **Splice / annotated / RC / AF-keep.** This plan wires the four Phase-1 modes for the **unspliced, no-keep, no-in-kernel-RC** path (matching the current svar2 kernels' "first cut minimal"). If the test bed exercises jitter only (no splice, no `min_af`/`max_af`, no annotated), that's covered. Guard the svar2 branch to `raise NotImplementedError` for splice plans / `keep` masks / annotated kind / in-kernel `to_rc` until those `_from_svar2` kernels exist (out of scope here — see the spec's "Annotated out of scope" and the SVAR2 kernel gaps). Add explicit `raise` guards so a user hitting an unsupported combo gets a clear error, not silently-wrong output. + +- [ ] **Step 5: Add `Dataset.open(svar2=...)`** + +In `_impl.py:124`, add `svar2: str | Path | None = None` (keyword-only, next to `svar`), thread it into `OpenRequest(..., svar2=svar2)`, and in `_open.py` forward `self.svar2` to `_build_seqs` → `Haps.from_path(svar2_override=...)`. Mirror the two `@overload`s if they enumerate kwargs. + +- [ ] **Step 6: Rebuild + run the end-to-end tests** + +Run: `pixi run -e dev maturin develop --release` +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` +Expected: all four modes PASS (byte-identical to SVAR1 / union oracle). + +- [ ] **Step 7: Full SVAR1 regression (additive guarantee)** + +Run: `pixi run -e dev pytest tests -q` +Expected: entire tree green — SVAR1 path byte-unchanged, unit + dataset both covered. + +- [ ] **Step 8: Lint + fmt + clippy + commit** + +```bash +cargo fmt && cargo clippy --all-targets 2>&1 | tail -20 +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ && pixi run -e dev typecheck +git add python/genvarloader/_dataset/_haps.py python/genvarloader/_dataset/_open.py python/genvarloader/_dataset/_impl.py python/genvarloader/_dataset/_reconstruct.py tests/dataset/test_svar2_dataset.py tests/dataset/conftest.py +git commit -m "feat(dataset): wire svar2 read dispatch (Svar2Store, source discriminant, svar2= override) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 8: Retire the live `overlap_batch` dispatch in `_svar2_source.py` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_source.py`. + +**Interfaces:** `SparseVar2Source` stays as a **parity oracle** (used by Tasks 4–6 tests) but is no longer on any live read path. Remove the `TODO(svar2-dataset-dispatch)` marker since dispatch now lives in `Haps` (Task 7). + +- [ ] **Step 1: Update the module docstring** — replace the `TODO(svar2-dataset-dispatch)` paragraph with a note that live dispatch is wired in `Haps` (read-bound, `_haps.py`) and this adapter is retained only as the union-path parity oracle for tests. + +- [ ] **Step 2: Confirm nothing imports it on a live path** + +Run: `pixi run -e dev python -c "import genvarloader"` and search: `rtk grep "SparseVar2Source" python/` — expect references only in `_svar2_source.py` and tests, not in `_haps.py`/`_impl.py`/`_open.py`. + +- [ ] **Step 3: Run the oracle tests + commit** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_tracks.py -q` +Expected: PASS (oracle still callable). + +```bash +git add python/genvarloader/_dataset/_svar2_source.py +git commit -m "refactor(dataset): retire svar2 live overlap_batch dispatch (oracle-only) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 9: Docs / roadmap / skill / api.md audit + +**Files:** +- Modify: `skills/genvarloader/SKILL.md`, `docs/source/{write.md,format.md,faq.md,api.md}`, `README.md`, `docs/roadmaps/rust-migration.md`. + +- [ ] **Step 1: Document `.svar2` as a `write` variant source** — in `SKILL.md` and `docs/source/write.md`: `.svar2` accepted alongside `.svar`/VCF/PGEN; note it produces a read-bound ranges cache; `Dataset.open(..., svar2=)` mirrors `svar=`. In `format.md`: document the `genotypes/svar2_ranges/` layout (the six arrays + `svar2_meta.json`) and the `metadata.json` `svar2_link` field. In `faq.md`/`README.md`: `.svar2`'s on-disk size advantage; the read path builds no interval tree / no dense union. + +- [ ] **Step 2: api.md ↔ `__all__` sync** — if any new public symbol was added to `python/genvarloader/__init__.py` `__all__` (e.g. a `migrate_svar2_link` analog), add its autodoc entry. Run the gate: + +```bash +pixi run -e dev python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. + +- [ ] **Step 3: Roadmap** — in `docs/roadmaps/rust-migration.md`, tick the read-bound SVAR2 wiring; record the parity results (all four modes byte-identical to SVAR1/union oracle) and set the phase marker + link this plan and the genoray plan. Note the byte-identical parity contract is satisfied. + +- [ ] **Step 4: Commit** + +```bash +git add skills/ docs/ README.md +git commit -m "docs: .svar2 as a write variant source + read-bound wiring (skill, format, faq, roadmap) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 10: Relocate + re-run the MVP benchmark + +**Files:** none in-repo (benchmark scripts live in the relocated `svar2_mvp`). + +- [ ] **Step 1: Relocate the MVP tree** + +```bash +mv /carter/users/dlaub/repos/for_loukik/svar2_mvp /carter/users/dlaub/projects/svar2_mvp +``` +Then update any absolute paths in `svar2_mvp/build_source.sh` and the benchmark driver (`rtk grep "for_loukik" /carter/users/dlaub/projects/svar2_mvp`). + +- [ ] **Step 2: Re-run the SVAR1-vs-SVAR2 `Dataset.__getitem__` benchmark** on chr21 germline (3202) + somatic (16007) after wiring — latency (same-session before/after within one allocation) + store size. Follow the profiling memory: profile the Python process with `perf` (paranoid=2, no sudo), not `py-spy --native`. Report the perf DSO split. + +- [ ] **Step 3: Verify success criteria** — the warm SVAR2 read shows **neither** `SearchTree::build` **nor** a dense-union rebuild (the DSO split flips from ~80% genoray to gvl-kernel-bound, like SVAR1), and SVAR2's store-size advantage holds. Record numbers as a **relative** same-session before/after (absolute wall-clock is not comparable across allocations on shared Carter nodes — per the perf-gate memory). + +- [ ] **Step 4: Record results** in the roadmap checkpoint (Task 9's roadmap file) and commit any driver-path edits that live inside the repo (if the benchmark driver is tracked). + +--- + +## Self-Review Notes (traceability to the spec) + +- **Spec Component B (write)** → Tasks 1 (`_svar2_link.py` + `Metadata.svar2_link`), 2 (`_write_from_svar2` + coercion + dispatch + 6-array cache). Reject unsupported variants via `_reject_unsupported_variants` (Task 2 Step 5). +- **Spec Component C (read, all-Rust)** → Task 3 (`genoray_core` path-dep + `Svar2Store` opened once = the SVAR1 `ffi_static` analog), Task 4 (one FFI call, `gather_haps_readbound` + `merge_hap3`, LUT via `reader.lut_arrays()` — no numpy round-trip, no Python `gather_ranges`), Task 7 (dispatch discriminant retiring `TODO(svar2-dataset-dispatch)`). +- **Spec "all four output modes, all Rust"** → Task 4 (haplotypes), 5 (tracks), 6 (variants + variant-windows). Annotated explicitly out of scope (guarded `NotImplementedError`, Task 7 Step 4). +- **Spec cache format** → Task 2 Step 5 (six arrays under `svar2_ranges/` + `svar2_meta.json`; O(offsets), bulk stays in the `.svar2`). +- **Spec parity & testing** → union-oracle parity per mode (Tasks 4–6), SVAR1 cross-format + additive regression (Task 7 Steps 6–7), perf verification (Task 10). +- **Spec "no interval search / no contig-wide union at read"** → guaranteed structurally: the read path calls only `gather_haps_readbound` (Plan 1: zero `SearchTree`, no `dense_union`); verified in Plan 1 Task 4's zero-tree test and Task 10's DSO split. +- **Spec open questions** → (channel factoring) resolved in Plan 1 (`BatchResultSplit`: vk merged + dense split). (wheel↔path-dep sync) Task 3 pins the genoray commit; the `Svar2Fingerprint` guards store identity at open. (format version) `svar2_link` is additive/defaulted — no bump (Task 1 Step 5). (arbitrary-(region,sample) mapping) resolved via Plan 1's flat `gather_haps_readbound` + Task 7's cache-slice mapping. +- **Resolved spec inaccuracies** (carried from Plan 1): genoray path is absolute, not `../genoray`; `DenseView` in `query.rs`; `decode_key` = `svar2_codec::decode_key`; htslib reach includes `lib.rs`. diff --git a/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md new file mode 100644 index 00000000..3dc3fa28 --- /dev/null +++ b/docs/superpowers/plans/2026-07-05-svar2-readbound-getitem-perf.md @@ -0,0 +1,760 @@ +# SVAR2 read-bound `Dataset.__getitem__` profiling & optimization — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Attribute and then remove the hottest per-read costs in the live SVAR2 read-bound `Dataset.__getitem__` path for the two supported in-scope modes — **haplotypes** and **variants** — Python by inspection and Rust by `cargo asm`, with byte-identical parity preserved. + +**Architecture:** Phase A stands up a deterministic profiling harness on the *real* `Dataset.open(svar2).with_seqs(mode)[...]` path (not the retired union oracle) and captures a committed baseline: cProfile + pyinstrument for Python functions, `perf` DSO-split + Rust symbol self-time for native, and `perf stat -e instructions` for a noise-free gate. Phase B applies optimizations one at a time — each confirmed against the Phase A profile before it is written, then gated on byte-identical parity plus a same-session instruction-count delta. gvl-side changes land on `svar2-m6b-kernel` (PR #266); genoray-side changes land on genoray `svar-2`. + +**Tech Stack:** Python 3.10 (`-e dev` pixi env), Rust (maturin/PyO3), numpy, genoray_core (Rust crate path-dep) + genoray wheel, cProfile/pyinstrument, `perf`, `cargo asm`. + +## Global Constraints + +- **Design spec:** `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` (read it first). +- **In-scope modes:** **haplotypes** and **variants** only. **Tracks is out of scope** (not profiled) but its parity tests MUST stay green — no change may break the tracks path. **variant-windows** is a consumer target but is currently guarded `NotImplementedError` in `Svar2Haps.__call__` (`_FlatVariantWindows`), so it cannot be profiled or optimized here — note it as deferred until implemented. +- **Two repos / branches:** gvl = this worktree, branch `svar2-m6b-kernel` (PR #266). genoray = `/carter/users/dlaub/projects/genoray`, branch `svar-2` (currently @ `aaf44fd`). +- **Rebuild before testing/profiling any Rust change:** `pixi run -e dev maturin develop --release` — pytest/perf import the stale `.so` otherwise. genoray crate edits are picked up by this same gvl rebuild (path-dep); a genoray **wheel** rebuild is only needed if a genoray **Python-API** surface changes (it will not here). +- **Profiling build must keep frame pointers:** build the profiled `.so` with `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` (perf fp call-graph needs them). +- **Parity is a hard gate.** After every change: `pixi run -e dev pytest tests -q` (full tree — the svar2 suite + parity oracle live across `tests/dataset` and `tests/unit`; this includes the tracks parity tests), plus `cargo test` on the repo(s) touched. The read-bound kernels are byte-identical to the union oracle; any divergence blocks the change. Two documented intentional non-identities (pure-DEL ALT `b""`; SVAR1 `max_ends` tie under-extension) are pre-existing — do not "fix" them. +- **Measurement discipline.** No cross-session absolute wall-clock claims (shared Carter node). Gate on **same-session before/after** + **`perf stat -e instructions,cycles`** deltas. If HW counters are unavailable (`perf stat` errors with "not supported"), fall back to cProfile total primitive-call counts + same-session median wall-clock, and say so in the recorded result. +- **Fixed inputs (verified present):** + - Stores: `/carter/users/dlaub/projects/svar2_mvp/{germline,somatic}.{svar,svar2}`. + - Reference: `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa`, contig `chr21`. + - Env python (call directly for perf; never via `pixi run` — the launcher eats ~60% of samples): `.pixi/envs/dev/bin/python`. + - perf binary: `/carter/users/dlaub/.pixi/bin/perf`. +- **py-spy is unusable here** (`ptrace_scope=2`, no sudo) and Python 3.10 has no perf trampoline — do not attempt py-spy or `perf` Python-frame resolution. Python attribution comes only from cProfile/pyinstrument. +- **Commit hygiene:** docs-only commits may need `--no-verify` (the `pyrefly` pre-commit hook fails spuriously on zero-Python-file commits and collides with the unstaged `pixi.lock`). Code commits run hooks normally; ensure prek hooks are installed first (`prek install` if needed). +- **Live read path recipe** (how every driver builds the real path): + ```python + import genvarloader as gvl + from genoray import SparseVar2 + gvl.write(ds_path, bed, variants=SparseVar2(f"{prefix}.svar2"), samples=None, + max_jitter=0, overwrite=True) # ONCE, before the profiled loop + ds = gvl.Dataset.open(ds_path, reference=REF) + ds.with_seqs("haplotypes")[:, :] # or with_seqs("variants") + ``` + +--- + +## Phase A — Profiling harness & committed baseline + +### Task A1: Add pyinstrument + write the live-path profiling driver + +**Files:** +- Modify: `pixi.toml` (add `pyinstrument` to the dev feature deps) +- Create: `tmp/svar2_mvp/prof_getitem.py` + +**Interfaces:** +- Produces: a CLI `python tmp/svar2_mvp/prof_getitem.py ` where `mode ∈ {haplotypes, variants}`, `cohort ∈ {germline, somatic}`; runs `gvl.write` + `Dataset.open` ONCE, then calls the selected read `K` times in a warm loop; prints `per_call_s=`. Exposes a module function `make_call(mode, cohort) -> Callable[[], object]` reused by the cProfile/perf drivers in A2/A3. + +- [ ] **Step 1: Add pyinstrument to the dev env** + +Find the dev feature's `pypi-dependencies` (or `dependencies`) table in `pixi.toml` and add `pyinstrument`: + +```toml +# in the [feature.dev.pypi-dependencies] (or nearest dev deps) table +pyinstrument = "*" +``` + +- [ ] **Step 2: Install it** + +Run: `pixi install -e dev` +Expected: resolves and installs `pyinstrument`; no other deps change materially. + +- [ ] **Step 3: Verify import** + +Run: `.pixi/envs/dev/bin/python -c "import pyinstrument; print(pyinstrument.__version__)"` +Expected: prints a version (e.g. `4.x`). + +- [ ] **Step 4: Write the driver** + +Create `tmp/svar2_mvp/prof_getitem.py`: + +```python +"""Profile the LIVE SVAR2 read-bound Dataset.__getitem__ path (not the union +oracle) for the in-scope modes. One (mode, cohort) per process so cProfile/perf +attribute cleanly. + + python tmp/svar2_mvp/prof_getitem.py + +gvl.write + Dataset.open run ONCE (we profile the READ, not the write). Prints +per_call_s over K warm calls. Tracks mode is out of scope; variant-windows is +guarded NotImplementedError in Svar2Haps and cannot be profiled yet.""" +import sys +import time +from pathlib import Path + +import polars as pl + +STORE_DIR = Path("/carter/users/dlaub/projects/svar2_mvp") +REF = "/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa" +CHROM = "chr21" +REGIONS = [(20_000_000, 20_001_000), (30_000_000, 30_000_500), (40_000_000, 40_001_000)] +WORK = Path("tmp/svar2_mvp/prof_out/readbound") + + +def _bed(): + return pl.DataFrame({ + "chrom": [CHROM] * len(REGIONS), + "chromStart": [s for s, _ in REGIONS], + "chromEnd": [e for _, e in REGIONS], + }) + + +def make_call(mode, cohort): + import genvarloader as gvl + from genoray import SparseVar2 + + prefix = STORE_DIR / cohort + sv2 = SparseVar2(f"{prefix}.svar2") + n_s = sv2.n_samples + ds_path = WORK / f"{cohort}_{mode}.gvl" + WORK.mkdir(parents=True, exist_ok=True) + + gvl.write(ds_path, _bed(), variants=SparseVar2(f"{prefix}.svar2"), + samples=None, max_jitter=0, overwrite=True) + ds = gvl.Dataset.open(ds_path, reference=REF) + view = ds.with_seqs(mode) # "haplotypes" or "variants" + + R = len(REGIONS) + + def call(): + view[:R, :n_s] + + return call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + t0 = time.perf_counter() + for _ in range(K): + call() + print(f"per_call_s={(time.perf_counter() - t0) / K:.5f}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 5: Smoke-test both modes (germline, small K)** + +Run: +```bash +cd "$(git rev-parse --show-toplevel)" +for m in haplotypes variants; do + echo "== $m =="; .pixi/envs/dev/bin/python tmp/svar2_mvp/prof_getitem.py $m germline 3 +done +``` +Expected: each prints `per_call_s=` with no exception. (haplotypes needs the reference — it is set; no BigWig is required for either mode.) + +- [ ] **Step 6: Commit** + +```bash +git add pixi.toml pixi.lock tmp/svar2_mvp/prof_getitem.py +git commit -m "perf(svar2): live read-bound Dataset.__getitem__ profiling driver + pyinstrument" +``` + +--- + +### Task A2: Capture the Python-layer baseline (cProfile + pyinstrument) + +**Files:** +- Create: `tmp/svar2_mvp/prof_python.py` +- Create (output): `tmp/svar2_mvp/prof_out/readbound/python_baseline.md` + +**Interfaces:** +- Consumes: `prof_getitem.make_call` (A1). +- Produces: committed per-(mode×cohort) cProfile top-cumulative tables + pyinstrument trees identifying the hottest **Python** functions on the read path. + +- [ ] **Step 1: Write the Python-layer profiler** + +Create `tmp/svar2_mvp/prof_python.py`: + +```python +"""cProfile + pyinstrument over the live read (Python-layer attribution). + + python tmp/svar2_mvp/prof_python.py + +cProfile ranks Python functions by cumulative time; pyinstrument gives a +low-overhead statistical wall-clock call tree as a cross-check (cProfile's own +per-call overhead can distort tiny hot loops).""" +import cProfile +import io +import pstats +import sys + +from prof_getitem import make_call + + +def main(mode, cohort, K): + call = make_call(mode, cohort) + call() # warm + + pr = cProfile.Profile() + pr.enable() + for _ in range(K): + call() + pr.disable() + s = io.StringIO() + pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats(30) + print(f"### cProfile {mode} {cohort} (K={K}), sort=cumulative\n") + print("```\n" + s.getvalue() + "```\n") + + from pyinstrument import Profiler + p = Profiler(interval=0.0005) + p.start() + for _ in range(K): + call() + p.stop() + print(f"### pyinstrument {mode} {cohort} (K={K})\n") + print("```\n" + p.output_text(unicode=False, color=False, show_all=False) + "```\n") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2], int(sys.argv[3])) +``` + +- [ ] **Step 2: Run all four combos into the baseline report** + +Run: +```bash +cd tmp/svar2_mvp +OUT=prof_out/readbound/python_baseline.md +echo "# SVAR2 read-bound Python-layer baseline ($(date -I))" > "$OUT" +for c in germline somatic; do for m in haplotypes variants; do + echo "== $m $c ==" + ../../.pixi/envs/dev/bin/python prof_python.py $m $c 200 >> "$OUT" 2>&1 +done; done +cd ../.. +``` +Expected: `python_baseline.md` populated with 4 cProfile tables + 4 pyinstrument trees, no tracebacks. (Lower K to ~50 for `somatic` if a combo is slow.) + +- [ ] **Step 3: Record the top Python functions** + +Read `tmp/svar2_mvp/prof_out/readbound/python_baseline.md`. Append a short `## Top Python functions (ranked)` section listing, per mode, the 3–5 highest-cumulative gvl Python functions. Expect for **variants**: `_reconstruct_variants`, `_gather_inputs`, `_ragged_arange_gather` / `_ragged_arange_gather_2level`, `_contig_groups`; for **haplotypes**: `get_haps_and_shifts`, `_gather_inputs`, `_assemble_haps`. Note which are pure-Python overhead vs. thin FFI wrappers. + +- [ ] **Step 4: Commit** + +```bash +git add tmp/svar2_mvp/prof_python.py tmp/svar2_mvp/prof_out/readbound/python_baseline.md +git commit --no-verify -m "perf(svar2): Python-layer baseline profile (cProfile + pyinstrument)" +``` + +--- + +### Task A3: Capture the native/Rust baseline (perf) + instruction-count reference + +**Files:** +- Create: `tmp/svar2_mvp/prof_perf.sh` +- Create (output): `tmp/svar2_mvp/prof_out/readbound/native_baseline.md` + +**Interfaces:** +- Consumes: `prof_getitem.py` (A1). +- Produces: committed per-(mode×cohort) DSO split, Rust symbol self-time, fp call-graph, and a `perf stat -e instructions,cycles` reference used as the Phase-B gate baseline. + +- [ ] **Step 1: Rebuild the `.so` with frame pointers** + +Run: `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` +Expected: build succeeds; this is the binary all A3/Phase-B perf captures use. + +- [ ] **Step 2: Write the perf capture script** + +Create `tmp/svar2_mvp/prof_perf.sh`: + +```bash +#!/usr/bin/env bash +# Native-layer attribution for the live read-bound path via perf (py-spy is +# unusable: ptrace_scope=2; Python 3.10 has no perf trampoline so Python frames +# are opaque -> DSO-level + Rust-symbol self-time is the split). +set -eu +cd "$(git rev-parse --show-toplevel)" +OUT=tmp/svar2_mvp/prof_out/readbound +mkdir -p "$OUT" +PERF=/carter/users/dlaub/.pixi/bin/perf +PY=.pixi/envs/dev/bin/python +FREQ=299 +REPORT="$OUT/native_baseline.md" +echo "# SVAR2 read-bound native baseline ($(date -I))" > "$REPORT" + +probe_K () { # mode cohort -> K sized to ~40s + local per + per=$("$PY" tmp/svar2_mvp/prof_getitem.py "$1" "$2" 5 | sed 's/per_call_s=//') + "$PY" -c "import math;print(max(20,math.ceil(40/max(float('$per'),1e-4))))" +} + +for c in germline somatic; do for m in haplotypes variants; do + tag="${m}_${c}"; K=$(probe_K "$m" "$c") + echo "## $tag (K=$K)" | tee -a "$REPORT" + # instruction-count reference (the Phase-B gate baseline) + echo '### perf stat' >> "$REPORT" + { "$PERF" stat -e instructions,cycles -- "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" ; } \ + 2>> "$REPORT" 1>/dev/null || echo "(perf stat HW counters unavailable)" >> "$REPORT" + # sampling profile + "$PERF" record -g --call-graph fp -F $FREQ -o "$OUT/$tag.data" -- \ + "$PY" tmp/svar2_mvp/prof_getitem.py "$m" "$c" "$K" >/dev/null 2>&1 + echo '### DSO split' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=dso --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -12 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### top Rust/native self-time symbols' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=symbol --no-children -g none -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -20 >> "$REPORT"; echo '```' >> "$REPORT" + echo '### call graph (top)' >> "$REPORT"; echo '```' >> "$REPORT" + "$PERF" report --stdio --sort=overhead,symbol -i "$OUT/$tag.data" 2>/dev/null \ + | grep -vE '^\s*#|^\s*$' | head -45 >> "$REPORT"; echo '```' >> "$REPORT" +done; done +echo "NATIVE_BASELINE_DONE -> $REPORT" +``` + +- [ ] **Step 3: Run it** + +Run: `bash tmp/svar2_mvp/prof_perf.sh` +Expected: prints `NATIVE_BASELINE_DONE`; `native_baseline.md` has DSO split + symbol + call-graph blocks for all 4 combos. Confirm the split shows **no** `SearchTree::build` / `dense_union` / `overlap_batch` samples (that would mean the union path is being hit — a driver bug). Expect `gather_haps_readbound`, `merge_keys`, `split_to_flat`, `decode_variants_from_split` (variants), `reconstruct_haplotypes_from_svar2` (haplotypes), and numpy cache-slice symbols (`PyArray_Repeat`, `mapiter_get`) instead. + +- [ ] **Step 4: Record the ranked native targets** + +Append a `## Ranked native targets` section to `native_baseline.md`: per mode, the top 3 native self-time symbols and their DSO (gvl vs genoray_core vs numpy). For **haplotypes**, note whether `gather_haps_readbound` + `split_to_flat` self-time is roughly **2× the diffs-only need** — that confirms the redundant double gather (Task B1). For **variants**, note `split_to_flat` / `decode_variants_from_split` allocation churn (`_int_malloc`/`SpecFromIter` under them → Task B2). + +- [ ] **Step 5: Commit** + +```bash +git add tmp/svar2_mvp/prof_perf.sh tmp/svar2_mvp/prof_out/readbound/native_baseline.md +git commit --no-verify -m "perf(svar2): native-layer baseline profile (perf DSO/symbol/callgraph + instr reference)" +``` + +--- + +## Phase B — Optimizations (each confirmed against Phase A, then parity + instruction-count gated) + +> **Before each Task below:** re-read the Phase A baseline and confirm the target actually ranks in the top few. If a candidate does NOT rank (e.g. B1 removed it, or the profile disagrees), skip that Task and instead apply the **same fix→gate template** to whichever function tops the profile, using `cargo asm`/`perf annotate` for native fns. Do not implement an optimization the profile does not justify. + +### Task B1: Eliminate the redundant pre-reconstruct gather (haplotypes) + +**Rationale (confirm against A3):** `Svar2Haps.get_haps_and_shifts` calls `hap_diffs_from_svar2_readbound` (full `gather_haps_readbound` + `split_to_flat` + `hap_diffs_svar2`) **and then** `reconstruct_haplotypes_from_svar2_readbound` (which repeats gather + split + diffs internally for output sizing). For a haplotypes read the pre-reconstruct diffs are needed **only** when jitter randomizes shifts; the warm `ds[:, :]` read is deterministic/ragged (`shifts = 0`), so the entire first gather is redundant there. `hap_lengths` (which uses `diffs`) is discarded on the haplotypes path (`__call__` keeps only `haps`); the tracks path (out of scope, but must stay green) reuses `get_haps_and_shifts` and *does* need `diffs`/`hap_lengths`, so the guard must preserve them there via an explicit flag. + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`get_haps_and_shifts`, ~lines 319-424; and its tracks caller) +- Test: `tests/dataset/test_svar2_readbound_haps.py`, `tests/dataset/test_svar2_dataset.py` (existing parity oracles — must stay green, including tracks) + +**Interfaces:** +- Consumes: `hap_diffs_from_svar2_readbound`, `reconstruct_haplotypes_from_svar2_readbound` (unchanged FFI). +- Produces: `get_haps_and_shifts(..., need_hap_lengths: bool = False)` computes the diffs loop only when its result is consumed (randomized shifts, OR a caller that needs `hap_lengths`/`diffs` — i.e. tracks passes `need_hap_lengths=True`). Same return tuple shape. + +- [ ] **Step 1: Add a failing micro-test asserting the diffs kernel is not called for a deterministic haps read** + +Add to `tests/dataset/test_svar2_readbound_haps.py`: + +```python +def test_deterministic_haps_read_skips_pre_reconstruct_diffs(monkeypatch): + """A deterministic (shifts=0) haplotypes read must NOT call the separate + hap_diffs readbound kernel — reconstruct sizes itself internally. Guards the + double-gather regression.""" + import genvarloader._dataset._svar2_haps as m + + calls = {"diffs": 0} + real = m.hap_diffs_from_svar2_readbound + def counting(*a, **k): + calls["diffs"] += 1 + return real(*a, **k) + monkeypatch.setattr(m, "hap_diffs_from_svar2_readbound", counting) + + # Build the same small live svar2 dataset the module parity tests use, then: + # ds.with_seqs("haplotypes")[:, :] + # (reuse this file's existing fixture that yields a ds2 Svar2Haps-backed view; + # if none is exposed, lift the _open_pair helper from test_svar2_dataset.py.) + ds2 = _svar2_haps_dataset() # existing/lifted fixture -> haplotypes view + ds2[:, :] + assert calls["diffs"] == 0 +``` + +If no fixture yields a live Svar2Haps haplotypes view in this file, lift `_open_pair` from `tests/dataset/test_svar2_dataset.py` into a local helper `_svar2_haps_dataset()` returning `ds2.with_seqs("haplotypes")`. + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_deterministic_haps_read_skips_pre_reconstruct_diffs -v` +Expected: FAIL with `assert 1 == 0` (the diffs kernel is currently called unconditionally). + +- [ ] **Step 3: Guard the diffs computation in `get_haps_and_shifts`** + +In `python/genvarloader/_dataset/_svar2_haps.py`, add `need_hap_lengths: bool = False` to `get_haps_and_shifts`'s signature, and replace the unconditional diffs block + shifts block (currently ~lines 352-384) with: + +```python + groups = self._contig_groups(contig_ids) + + # diffs are needed pre-reconstruct ONLY to (a) bound randomized jitter + # shifts, or (b) return hap_lengths/diffs to a caller that uses them + # (the tracks path). A deterministic/ragged haplotypes read needs + # neither: reconstruct sizes itself internally. Avoid the redundant + # gather+split+diffs in that (common warm-read) case. + randomized = not (deterministic or isinstance(output_length, str)) + need_diffs = randomized or need_hap_lengths + + if need_diffs: + diffs = np.empty((b, P), np.int32) + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + d = hap_diffs_from_svar2_readbound( + self.store, self.ds_contigs[ci], + gi[0], gi[1], gi[2], gi[3], gi[4], gi[5], gi[6], P, + ) + diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) + hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + else: + diffs = np.zeros((b, P), np.int32) # placeholder (unused downstream) + hap_lengths = np.broadcast_to( + lengths[:, None].astype(np.int32), (b, P) + ).copy() + + if randomized: + max_shift = diffs.clip(min=0) + max_shift = max_shift + (lengths - output_length).clip(min=0)[:, None] + shifts = rng.integers(0, max_shift + 1, dtype=np.int32) + else: + shifts = np.zeros((b, P), np.int32) +``` + +Then have the tracks caller pass `need_hap_lengths=True`. Find the caller: `grep -n "get_haps_and_shifts" python/genvarloader/_dataset/*.py` — it is invoked from `HapsTracks` dispatch (the tracks path) and from `Svar2Haps.__call__` (haplotypes). Update the tracks call site to `get_haps_and_shifts(..., need_hap_lengths=True)`; leave the haplotypes call site at the `False` default. + +> NOTE: when `randomized` is False but `output_length` is a fixed `int`, `hap_lengths` in the placeholder branch is `lengths` broadcast — but `reconstruct_haplotypes_from_svar2_readbound` with a fixed `output_length >= 0` recomputes per-hap lengths internally and ignores `hap_lengths`, so this is safe. Confirm no downstream consumer reads `hap_lengths` for a fixed-int deterministic haps read; if one does, set `need_hap_lengths=True` for that path too. + +- [ ] **Step 4: Python-only change — run the new test + full svar2 parity (incl. tracks)** + +Run: +```bash +pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_deterministic_haps_read_skips_pre_reconstruct_diffs -v +pixi run -e dev pytest tests/dataset -k svar2 -q +``` +Expected: the new test PASSES (`calls["diffs"] == 0`); all svar2 parity tests stay green (haplotypes/variants **and tracks** byte-identical to oracle — the tracks path still calls the diffs kernel via `need_hap_lengths=True`). + +- [ ] **Step 5: Full-tree parity + instruction-count delta** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +/carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py haplotypes germline 500 2>&1 | grep -E 'instructions|cycles' +``` +Expected: full tree green; instructions/read materially lower than the A3 baseline for `haplotypes` (record the % drop). If HW counters are unavailable, record cProfile primitive-call-count drop instead. + +- [ ] **Step 6: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_readbound_haps.py +git commit -m "perf(svar2): skip redundant pre-reconstruct gather for deterministic haplotype reads" +``` + +--- + +### Task B2: Pre-size `split_to_flat` + `decode_variants_from_split` allocations (gvl Rust) + +**Rationale (confirm against A3):** both functions build output `Vec`s with `Vec::new()` (no capacity) and grow them in hot loops — `split_to_flat` (`src/svar2/mod.rs:159`) for `dense_pos`/`dense_key`/`dense_present` (used by BOTH modes — variants decode and haplotype diffs/reconstruct call it), and `decode_variants_from_split` (`src/svar2/mod.rs:262`) for `pos`/`ilen`/`alt_bytes`/`str_off` (variants). `_int_malloc`/`SpecFromIter` were hot even in the old profile. Pre-sizing is byte-identical. **Only implement the parts that rank in the A3 native top symbols after B1.** + +**Files:** +- Modify: `src/svar2/mod.rs` (`split_to_flat` 159-237; `decode_variants_from_split` 262-320) +- Test: `cargo test` in gvl (`test_split_to_flat_*`, `test_decode_variants_from_split_*` in `src/svar2/mod.rs`) + the pytest parity suite + +**Interfaces:** +- Consumes/Produces: `split_to_flat(&BatchResultSplit) -> FlatChannels` and `decode_variants_from_split(&BatchResultSplit, &[u8], &[i64]) -> VariantsSoa` — signatures and output bytes unchanged; only allocation strategy changes. + +- [ ] **Step 1: Confirm the existing unit tests cover both** + +Run: `grep -n "fn test_split_to_flat\|fn test_decode_variants_from_split" src/svar2/mod.rs` +Expected: `test_split_to_flat_marshals_readbound_split`, `test_split_to_flat_trailing_zero_byte_is_allocated`, `test_decode_variants_from_split_merges_and_decodes` exist — the byte-identity guard. + +- [ ] **Step 2: Pre-size `split_to_flat`** + +In `split_to_flat`, compute the total dense entry count once and reserve; pre-size the presence bitstream and drop the on-set `resize` growth. Replace the `dense_pos`/`dense_key` init: + +```rust + let dense_total: usize = (0..n_q) + .map(|q| { + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_pos: Vec = Vec::with_capacity(dense_total); + let mut dense_key: Vec = Vec::with_capacity(dense_total); +``` + +and the presence bitstream: + +```rust + let total_bits: usize = (0..h_count) + .map(|h| { + let q = h / ploidy; + let (ss, se) = br.dense_snp_range[q]; + let (is_, ie) = br.dense_indel_range[q]; + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_present: Vec = vec![0u8; total_bits.div_ceil(8)]; +``` + +Inside the hap loop, drop the `if dense_present.len() <= byte { resize }` guards and set bits directly (`dense_present[bit_acc / 8] |= 1 << (bit_acc % 8);`). Keep the final `dense_present.resize(bit_acc.div_ceil(8), 0);` as a no-op safety. + +- [ ] **Step 3: Pre-size `decode_variants_from_split`** + +Replace the `Vec::new()` inits (`src/svar2/mod.rs:272-275`) with capacity-reserved vectors (an over-estimate upper bound is fine for `with_capacity`): + +```rust + let flat = split_to_flat(br); + let ploidy = br.ploidy; + let n_q = br.n_regions; + let h_count = n_q * ploidy; + + // Upper bound on total merged variants across all haps: every vk entry plus + // every dense window entry (present or not). Over-reserving is harmless. + let vk_total = flat.vk_off[h_count] as usize; + let dense_bits = flat.dense_present_off[h_count] as usize; + let cap = vk_total + dense_bits; + let mut pos: Vec = Vec::with_capacity(cap); + let mut ilen: Vec = Vec::with_capacity(cap); + let mut alt_bytes: Vec = Vec::with_capacity(cap); + let mut str_off: Vec = Vec::with_capacity(cap + 1); + str_off.push(0); + let mut var_off: Vec = Vec::with_capacity(h_count + 1); + var_off.push(0); +``` + +(Leave the per-hap loop body unchanged.) + +- [ ] **Step 4: Rebuild + Rust unit tests** + +Run: +```bash +RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release +cargo test split_to_flat decode_variants_from_split 2>&1 | tail -20 # (set LD_LIBRARY_PATH to .pixi/envs/dev/lib if libpython load fails) +``` +Expected: the named unit tests PASS (byte-identical output). + +- [ ] **Step 5: Full-tree parity + instruction delta (both modes)** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +for m in variants haplotypes; do + echo "== $m ==" + /carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py $m somatic 300 2>&1 | grep -E 'instructions|cycles' +done +``` +Expected: full tree green; instructions/read down vs A3 baseline for both modes (record %). + +- [ ] **Step 6: Commit** + +```bash +git add src/svar2/mod.rs +git commit -m "perf(svar2): pre-size split_to_flat + decode_variants_from_split allocations (byte-identical)" +``` + +--- + +### Task B3: De-duplicate the twin ragged gather in `_reconstruct_variants` (gvl Python) + +**Rationale (confirm against A2):** in `_reconstruct_variants` (`python/genvarloader/_dataset/_svar2_haps.py:585-586`), `pos` and `ilen` are permuted back to global order by two separate `_ragged_arange_gather(pos_c, grouped_var_off, perm)` / `(ilen_c, grouped_var_off, perm)` calls with **identical** `offsets` and `perm` — so the `lens`/`new_off`/`within`/`src` index arrays are computed twice. Compute the source-index permutation once and apply it to both. **Only implement if `_ragged_arange_gather` / `_reconstruct_variants` ranks in the A2 variants top functions.** + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_ragged_arange_gather` region + `_reconstruct_variants`) +- Test: `tests/dataset/test_svar2_readbound_variants.py`, `tests/dataset/test_svar2_dataset.py` (variants parity — must stay green) + +**Interfaces:** +- Produces: a helper `_ragged_arange_src(offsets, perm) -> (src, new_off)` returning the 1-level reorder source index + new offsets; `_ragged_arange_gather` is re-expressed on it; `_reconstruct_variants` computes `src`/`var_off_g` once and does `pos_g = pos_c[src]`, `ilen_g = ilen_c[src]`. + +- [ ] **Step 1: Add the shared-index helper and re-express `_ragged_arange_gather`** + +In `python/genvarloader/_dataset/_svar2_haps.py`, add above `_ragged_arange_gather`: + +```python +def _ragged_arange_src( + offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray[np.int64], NDArray[np.int64]]: + """Source-row index + new offsets for a 1-level ragged reorder by ``perm``. + + ``new_data == data[src]``; ``src`` and ``new_off`` depend only on + ``(offsets, perm)`` — so callers reordering several parallel data arrays by + the same key compute this ONCE and index each array. + """ + offsets = np.asarray(offsets, np.int64) + lens = np.diff(offsets) + new_lens = lens[perm] + new_off = lengths_to_offsets(new_lens, np.int64) + n = int(new_off[-1]) + if n == 0: + return np.zeros(0, np.int64), new_off + within = np.arange(n, dtype=np.int64) - np.repeat(new_off[:-1], new_lens) + src = np.repeat(offsets[perm], new_lens) + within + return src, new_off +``` + +Then rewrite `_ragged_arange_gather` to use it: + +```python +def _ragged_arange_gather( + data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray, NDArray[np.int64]]: + """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``.""" + src, new_off = _ragged_arange_src(offsets, perm) + if src.size == 0: + return data[:0].copy(), new_off + return data[src], new_off +``` + +- [ ] **Step 2: Use the shared index for pos + ilen in `_reconstruct_variants`** + +Replace (`python/genvarloader/_dataset/_svar2_haps.py:585-586`): + +```python + pos_g, var_off_g = _ragged_arange_gather(pos_c, grouped_var_off, perm) + ilen_g, _ = _ragged_arange_gather(ilen_c, grouped_var_off, perm) +``` + +with: + +```python + src, var_off_g = _ragged_arange_src(grouped_var_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] +``` + +(The 2-level `alt` gather is left as-is; it is a different offset structure.) + +- [ ] **Step 3: Run variants parity (Python-only change, no rebuild)** + +Run: +```bash +pixi run -e dev pytest tests/dataset -k svar2 -q +``` +Expected: all svar2 variants parity tests green (byte-identical `pos`/`ilen`/`alt` reorder). + +- [ ] **Step 4: Full-tree parity + instruction delta** + +Run: +```bash +pixi run -e dev pytest tests -q +P=.pixi/envs/dev/bin/python +/carter/users/dlaub/.pixi/bin/perf stat -e instructions,cycles -- \ + $P tmp/svar2_mvp/prof_getitem.py variants germline 500 2>&1 | grep -E 'instructions|cycles' +``` +Expected: full tree green; variants instructions/read down vs baseline (record %; if small, keep anyway — it removes a duplicated O(total_variants) pass and is a clarity win). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "perf(svar2): compute the pos/ilen ragged reorder index once in variants decode" +``` + +--- + +### Task B4: `cargo asm` pass on ALL hot native functions — parallel subagent fan-out + +**Rationale:** after B1-B3, re-profile and collect the **full set** of native functions that still carry meaningful self-time (gvl and genoray), not just the #1. Because these functions are independent, optimize them **in parallel — one subagent per function**, each inspecting that function's assembly, finding a bounds-check / vectorization / allocation defect, and applying a byte-identical fix guarded by its **own per-function parity test**. The specific instruction change is *discovered from the asm*, not pre-guessed; each subagent's deliverable is a measured instruction-count reduction (or a documented "no safe win"). The branch owner then merges the fixes sequentially behind the full-tree parity gate. + +**Execution model:** REQUIRED SUB-SKILLs — superpowers:dispatching-parallel-agents (fan-out) + superpowers:using-git-worktrees (isolation). Per project rules: subagent implementers run on **Sonnet or weaker**; worktrees live under `.claude/worktrees/` of the owning repo. Worktree isolation is REQUIRED because several candidates share a file (`src/svar2/mod.rs` holds `split_to_flat`/`merge_hap`/`decode_alt`; genoray `src/spine.rs` holds `merge_keys`) — parallel in-place edits to the same file would clobber each other. + +**Files:** +- Modify (per subagent, in its own worktree): the file owning that function — gvl `src/reconstruct.rs` / `src/svar2/mod.rs`, **or** genoray `/carter/users/dlaub/projects/genoray/src/{spine.rs,query.rs}` (branch `svar-2`) +- Test (per subagent): a focused `cargo test` in the owning repo asserting byte-identical output for that function + +**Interfaces:** +- Consumes: A3/post-B3 native profile ranking (the hot-function set). +- Produces: per function — same signature, byte-identical output, fewer hot-loop instructions, and a committed per-function parity test. + +- [ ] **Step 1: Re-profile and enumerate the hot-function set** + +Run: `cp tmp/svar2_mvp/prof_out/readbound/native_baseline.md tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md` then `bash tmp/svar2_mvp/prof_perf.sh`. From the per-mode symbol tables, list every gvl/genoray self-time symbol above a cutoff (e.g. ≥1.5% self across any mode). **Exclude** libc (`_int_malloc`/`memmove`/`memset`) and numpy symbols — those are structural, not asm-fixable. Write the resulting list (function → owning repo/file → which mode exercises it) into `tmp/svar2_mvp/prof_out/readbound/asm_targets.md`. This list is the fan-out work-list. + +- [ ] **Step 2: Create one worktree per target function** + +For each function `F` in the work-list, create an isolated worktree off the correct branch. gvl functions: +```bash +cd "$(git rev-parse --show-toplevel)" +git worktree add ".claude/worktrees/asm-" svar2-m6b-kernel +``` +genoray functions: +```bash +cd /carter/users/dlaub/projects/genoray +git worktree add ".claude/worktrees/asm-" svar-2 +``` +Record the worktree path per function in `asm_targets.md`. + +- [ ] **Step 3: Dispatch one Sonnet subagent per function (in parallel)** + +Dispatch all subagents in a single batch (superpowers:dispatching-parallel-agents). Give each subagent this exact brief, filled in for its function `F`, file, worktree path, and the mode/cohort that exercises it: + +> You are optimizing exactly ONE Rust function, `F`, in worktree `` (do not touch any other file). Model: Sonnet. +> 1. Dump its assembly: `cargo asm --release --lib "" 2>/dev/null | head -200` (if the path is ambiguous, run `cargo asm --release --lib` and grep the candidate list). Cross-reference `perf annotate -i tmp/svar2_mvp/prof_out/readbound/.data --stdio` for the hot source lines. +> 2. Identify a byte-identical defect: per-iteration `panic`/bounds-check in the hot loop (→ iterators or `get_unchecked` behind a proven invariant), scalar copy/merge loops (→ `extend_from_slice`/`copy_from_slice`), or in-loop allocation (→ `with_capacity`). Comment the specific instruction pattern on the fix. +> 3. Apply the minimal fix. Behavior MUST stay byte-identical. +> 4. Write a **per-function parity test** in the owning repo: a `#[test]` that runs `F` on representative inputs and asserts the exact output (compare against a hand-checked expected value, or against a slow reference implementation of `F` inline in the test). Name it `test__byte_identical`. +> 5. `cargo test test__byte_identical` (set `LD_LIBRARY_PATH` to `.pixi/envs/dev/lib` if libpython fails to load in gvl). It MUST pass. +> 6. Measure: `RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release` (from the gvl worktree; for genoray targets, build gvl against the genoray worktree path-dep), then same-session `perf stat -e instructions,cycles -- .pixi/envs/dev/bin/python tmp/svar2_mvp/prof_getitem.py ` before vs after. Report the instruction delta. +> 7. Commit in your worktree: `perf(...): asm fix — (byte-identical, -N% instr)`. If you found NO safe byte-identical win, revert your edits, commit nothing, and report "no safe win for F" with the asm reason. + +- [ ] **Step 4: Collect results** + +Gather each subagent's report (fix applied? instruction delta? or no-safe-win) into `asm_targets.md`. Discard worktrees for no-win functions: `git worktree remove .claude/worktrees/asm-`. + +- [ ] **Step 5: Merge the winning fixes sequentially behind the parity gate** + +For each winning function, in turn (gvl fixes onto `svar2-m6b-kernel`, genoray fixes onto `svar-2`): +```bash +# gvl example (from the main worktree): +cd "$(git rev-parse --show-toplevel)" +git cherry-pick # or merge the worktree branch +RUSTFLAGS="-C force-frame-pointers=yes" pixi run -e dev maturin develop --release +pixi run -e dev pytest tests -q # full-tree parity after EACH merge +``` +If a merge conflicts (two fixes in the same file) or the full tree goes red, resolve the conflict / drop the offending fix, re-run, and record the decision. genoray fixes: cherry-pick onto `svar-2` in the genoray checkout, then rebuild gvl (crate path-dep) and re-run the gvl full tree. After all merges, remove the worktrees. + +- [ ] **Step 6: Record** + +Append the final per-function outcomes (merged / dropped / no-win) and cumulative instruction deltas per mode to `tmp/svar2_mvp/prof_out/readbound/native_after_b1b3.md`. + +--- + +### Task B5: Consolidate results & update the profiling report + +**Files:** +- Create: `tmp/svar2_mvp/prof_out/readbound/RESULTS.md` +- Modify: genoray `docs/roadmap/svar-2.md` **only if** a kernel signature changed (record it under the relevant milestone note); gvl design spec §9 open-question resolution + +- [ ] **Step 1: Write the results summary** + +Create `tmp/svar2_mvp/prof_out/readbound/RESULTS.md` with a table: per mode (haplotypes, variants) × cohort, baseline vs final instructions/read (and cProfile fallback where HW counters were absent), the optimizations applied (B1/B2/B3/B4), and any candidate that was profiled-but-skipped (with the reason). State the parity result (full tree + svar2 suite green, incl. tracks) explicitly. Note the **deferred** items: tracks (out of scope this round; its `get_haps_and_shifts` double-gather is unaddressed and still runs the diffs kernel via `need_hap_lengths=True`), and variant-windows (guarded `NotImplementedError` — profile once implemented). + +- [ ] **Step 2: Reconcile docs if signatures changed** + +If any FFI/kernel signature changed, update the genoray roadmap note and the gvl design spec's §9 open-question resolution. If nothing signature-level changed, state "read-path internals only; no API/format/doc-surface change" in RESULTS.md. (The `get_haps_and_shifts` `need_hap_lengths` param is an internal signature, not public API — no skill/api.md/docs update needed.) + +- [ ] **Step 3: Commit** + +```bash +git add tmp/svar2_mvp/prof_out/readbound/RESULTS.md +git commit --no-verify -m "perf(svar2): read-bound getitem optimization results summary" +# + any genoray docs commit on svar-2 if applicable +``` + +--- + +## Self-Review + +**Spec coverage:** +- Profile the real read-bound path (not union oracle) → A1–A3 (driver forces `Dataset.open(svar2)`; A3 Step 3 asserts no `SearchTree`/`overlap_batch` samples). ✓ +- Tooling substitution (cProfile+pyinstrument / perf) → A2, A3; pyinstrument dep → A1. ✓ +- In-scope modes haplotypes + variants; tracks dropped from profiling but parity-protected; variant-windows deferred (guarded) → Global Constraints + A1 driver + B5 deferred note. ✓ +- Two-repo landing (gvl #266 / genoray svar-2) → Global Constraints + B4/B5 commit steps. ✓ +- Measure→confirm→fix→re-measure + parity + instr-delta → per-Task gates; Phase B preamble enforces "confirm before implementing." ✓ +- Haplotypes double-gather → B1 (concrete, tracks-safe via `need_hap_lengths`). Variants+shared alloc churn → B2 (concrete). Variants Python twin-gather → B3 (concrete). cargo asm on ALL hot Rust fns → B4 (parallel subagent fan-out, per-function parity tests, worktree-isolated). ✓ +- No format/API change → B5 Step 2 reconciliation. ✓ + +**Placeholder scan:** B4 is investigation-then-fix by nature (asm-discovered) and now a parallel fan-out, but specifies the exact hot-set enumeration rule + cutoff, worktree-per-function isolation, the verbatim per-subagent brief (with exact `cargo asm`/`perf annotate`/`perf stat` commands, the defect classes, the per-function `test__byte_identical` requirement), and a sequential merge-behind-parity gate — not a hand-wave. B1/B2/B3 carry full code. No "TBD"/"similar to Task N". + +**Type consistency:** `make_call(mode, cohort)` defined in A1, reused verbatim in A2. `get_haps_and_shifts` gains `need_hap_lengths: bool = False` (B1) — threaded to the tracks caller in the same task; haplotypes caller uses the default. `_ragged_arange_src(offsets, perm) -> (src, new_off)` defined and consumed within B3; `_ragged_arange_gather` re-expressed on it (same return contract). `split_to_flat`/`decode_variants_from_split` signatures unchanged (B2). Store sample count via `SparseVar2.n_samples` (verified attribute). diff --git a/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md b/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md new file mode 100644 index 00000000..fdff4701 --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-svar2-variant-windows.md @@ -0,0 +1,793 @@ +# svar2 variant-windows (+ unphased_union) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `Dataset.open(..., svar2=...).with_seqs("variant-windows", VarWindowOpt(...))[regions, samples]` work (currently `NotImplementedError`), and enable `unphased_union` for both svar2 decode modes (`variants` and `variant-windows`). + +**Architecture:** svar2 variant-windows = the already-validated svar2 `variants` decode (`decode_variants_from_svar2_readbound`) composed with the already-validated window-assembly Rust kernel (`assemble_variant_buffers`, via the `_assemble_variant_buffers_rust` shim), per contig group, feeding the decoded per-variant arrays as the kernel's "global" arrays with an **identity gather** (`v_idxs = arange(n_var)`). No new Rust. `unphased_union` is a per-group offset fold (`row_offsets[::P]`, `eff_ploidy=1`) applied before assembly, identical to SVAR1's `get_variants_flat`. + +**Tech Stack:** Python 3.10, numpy, `seqpro.rag.Ragged`, PyO3 Rust extension (unchanged), pytest, pixi (`-e dev`). + +**Spec:** `docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md` + +## Global Constraints + +- **No Rust change** — this composes existing kernels. `maturin develop --release` is only needed if the branch's `.so` is stale from prior work; run it once at the start if in doubt (pytest imports the stale `.so` otherwise — CLAUDE.md). +- **Parity is a hard gate.** svar2 read output is byte-identical to its oracle. After every change run the relevant pytest; before pushing run the full tree `pixi run -e dev pytest tests -q` (scoped runs skip `tests/unit/`). +- **Deletion-ALT difference:** SVAR1 keeps a deletion's anchor base (e.g. `G` for `GTA>G`); svar2 decodes `""`. So `alt`/`alt_window` bytes are **not** SVAR1-identical — only `ref_window` is. Oracle strategy per task reflects this. +- **HPC gotcha:** pass `--basetemp=$(pwd)/.pytest_tmp` to pytest so the write path's `os.link` hardlink does not fail cross-device (Errno 18). +- **Coordinate convention:** svar2 decoded `pos` is 0-based within-contig, proven `==` SVAR1 `start`; it feeds the kernel's `v_starts` directly. The per-group reference is a single-contig slice (`_ref_for_contig`), so `v_contigs = zeros` and `ref_offsets = [0, len]`. +- **Commits:** conventional-commit style. Commit after each task's tests pass. The pre-commit `pyrefly` hook spuriously fails on docs-only commits in this worktree (finds no Python files → exit 1); for commits that DO touch Python it runs normally, so do not add `--no-verify` to code commits. +- **`ref="allele"` is already blocked** upstream (`_impl.py` ~line 720 raises `ValueError` when `window_opt.ref=="allele"` and `variants.ref is None`, always true for `Svar2Haps`). No new guard; a test pins it. + +--- + +## File Structure + +- **Modify** `python/genvarloader/_dataset/_svar2_haps.py`: + - new method `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows`; + - `Svar2Haps.__call__`: replace the `_FlatVariantWindows` `NotImplementedError` with the jitter guard + dispatch; + - `Svar2Haps._reconstruct_variants`: add the `unphased_union` fold; + - `Svar2Haps._guard_unsupported`: remove the `unphased_union` clause; + - imports: add `_assemble_variant_buffers_rust`, `_FlatWindow` from `._flat_variants`; add `_Flat` is already imported. +- **Modify** `tests/dataset/test_svar2_dataset.py`: new variant-windows + union parity tests and guard tests (reuses the module's existing `_src`/`svar_fixture`/`svar2_fixture`/`bed` and `_src2`/`svar_fixture2`/`svar2_fixture2` fixtures). +- **Modify** docs: `skills/genvarloader/SKILL.md`, `docs/source/*` where svar2 mode support is enumerated, and the two related specs' out-of-scope notes. + +--- + +## Task 1: `_reconstruct_variant_windows` core (non-union, incl. dummy fill) + wiring + guards + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes (existing): `decode_variants_from_svar2_readbound`, `self._gather_inputs`, `self._contig_groups`, `self._ref_for_contig`, `self._inverse_row_perm`, `_ragged_arange_src`, `_ragged_arange_gather_2level`, `lengths_to_offsets`; `_assemble_variant_buffers_rust`, `_FlatWindow`, `_FlatVariantWindows` (from `_flat_variants`); `self.window_opt`, `self.token_lut`, `self.reference`, `self.var_fields`, `self.dummy_variant`, `self.unknown_token`. +- Produces: `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows` and the wired `__call__` branch. Task 3 adds a `p_eff` fold to this same method. + +- [ ] **Step 1: Write the failing test — ref_window byte-identical to SVAR1 (single-contig)** + +Add to `tests/dataset/test_svar2_dataset.py`. Place a shared opt + helper near the top of the file (after imports): + +```python +from genvarloader import VarWindowOpt + +_WIN_OPT = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="window" +) + + +def _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref, opt=_WIN_OPT): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + return w1, w2 + + +def _assert_window_equal(a, b, name: str) -> None: + """Flat-buffer equality of two _FlatWindow fields (data + both offset levels).""" + assert np.array_equal(np.asarray(a.var_offsets), np.asarray(b.var_offsets)), ( + f"{name} var_offsets differ" + ) + assert np.array_equal(np.asarray(a.seq_offsets), np.asarray(b.seq_offsets)), ( + f"{name} seq_offsets differ" + ) + assert np.array_equal(np.asarray(a.data), np.asarray(b.data)), f"{name} data differ" +``` + +Then the test: + +```python +def test_svar2_variant_windows_ref_window_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """ref_window is a pure reference read over an identical variant SET, so it is + byte-identical to SVAR1 (independent of the deletion-ALT encoding difference).""" + _bcf, ref = _src + w1, w2 = _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + assert w2.ref_window is not None + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # scalar start field also identical (same variant SET) — compare _Flat buffers. + assert np.array_equal( + np.asarray(w2.fields["start"].data), np.asarray(w1.fields["start"].data) + ) + assert np.array_equal( + np.asarray(w2.fields["start"].offsets), np.asarray(w1.fields["start"].offsets) + ) +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_ref_window_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL with `NotImplementedError: svar2 datasets do not support with_seqs('variant-windows') yet.` + +- [ ] **Step 3: Add imports to `_svar2_haps.py`** + +At the existing import of `_FlatVariantWindows` (line ~50), extend it: + +```python +from ._flat_variants import ( + _FlatVariantWindows, + _FlatWindow, + _assemble_variant_buffers_rust, +) +``` + +(`_Flat` and `lengths_to_offsets` are already imported at the top of the file.) + +- [ ] **Step 4: Implement `_reconstruct_variant_windows`** + +Add this method to `Svar2Haps`, right after `_reconstruct_variants` (before `# ---- helpers ----`): + +```python +def _reconstruct_variant_windows( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] +) -> _FlatVariantWindows: + """Variant-windows for svar2: decode variants per contig group, then run the + shared ``assemble_variant_buffers`` window kernel over the decoded arrays via + an identity gather. ``ref="allele"`` is blocked upstream, so ref is always a + reference-read window; ``alt`` follows ``window_opt.alt``. + """ + assert self.window_opt is not None and self.token_lut is not None + assert self.reference is not None + from typing import Any + + opt = self.window_opt + L = opt.flank_length + ref_mode = 1 # ref="window" (ref="allele" rejected in with_seqs) + alt_mode = 1 if opt.alt == "window" else 2 + include_ilen = "ilen" in self.var_fields + + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + p_eff = P # unphased_union fold (Task 3) sets this to 1 per group. + + cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + # name -> per-group (token_data, per-variant seq offsets) + win_data: dict[str, list[NDArray]] = {} + win_seq_off: dict[str, list[NDArray[np.int64]]] = {} + + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + ) + ) + pos = np.asarray(pos, np.int32) + ilen = np.asarray(ilen, np.int32) + alt_bytes = np.asarray(alt_bytes, np.uint8) + str_off = np.asarray(str_off, np.int64) + var_off = np.asarray(var_off, np.int64) + + row_off = var_off # Task 3: fold to var_off[::P] under unphased_union. + n_var = int(len(pos)) + ref_, ref_offsets = self._ref_for_contig(ci) + bufs = _assemble_variant_buffers_rust( + 1, # windows mode + np.arange(n_var, dtype=np.int32), # identity v_idxs (data pre-gathered) + row_off, + alt_bytes, # alt_global + str_off, # alt_off_global + None, # ref_global (ref="window") + None, # ref_off_global + False, # want_ref_bytes + False, # want_flank + ref_mode, + alt_mode, + L, + self.token_lut, + np.zeros(n_var, np.int32), # v_contigs (single-contig ref slice) + pos, # v_starts + ilen, # ilens + ref_, + ref_offsets, + self.reference.pad_char, + ) + + cat_row_off.append(row_off) + cat_pos.append(pos) + cat_ilen.append(ilen) + cat_query_order.append(qsel) + for name, (data, seq_off) in bufs.items(): + win_data.setdefault(name, []).append(np.asarray(data)) + win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) + + shape: tuple[int | None, ...] = (b, p_eff, None) + wshape: tuple[int | None, ...] = (b, p_eff, None, None) + + # Single contig group: grouped order already equals global (b, p_eff) order. + if len(cat_pos) == 1: + row_off = cat_row_off[0] + fields: dict[str, Any] = { + "start": _Flat.from_offsets(cat_pos[0], shape, row_off) + } + if include_ilen: + fields["ilen"] = _Flat.from_offsets(cat_ilen[0], shape, row_off) + win = _FlatVariantWindows(fields) + for name in win_data: + setattr( + win, + name, + _FlatWindow(win_data[name][0], win_seq_off[name][0], row_off, wshape), + ) + else: + perm = self._inverse_row_perm(cat_query_order, b, p_eff) + grouped_row_off = lengths_to_offsets( + np.concatenate([np.diff(r) for r in cat_row_off]), np.int64 + ) + pos_c = np.concatenate(cat_pos) + ilen_c = np.concatenate(cat_ilen) + src, row_off_g = _ragged_arange_src(grouped_row_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] + fields = {"start": _Flat.from_offsets(pos_g, shape, row_off_g)} + if include_ilen: + fields["ilen"] = _Flat.from_offsets(ilen_g, shape, row_off_g) + win = _FlatVariantWindows(fields) + for name in win_data: + data_c = np.concatenate(win_data[name]) + grouped_seq_off = lengths_to_offsets( + np.concatenate([np.diff(s) for s in win_seq_off[name]]), np.int64 + ) + nd, nvar, nseq = _ragged_arange_gather_2level( + data_c, grouped_row_off, grouped_seq_off, perm + ) + setattr(win, name, _FlatWindow(nd, nseq, nvar, wshape)) + + if self.dummy_variant is not None: + win = win.fill_empty_groups( + self.dummy_variant, unk=self.unknown_token, flank_length=L + ) + return win +``` + +- [ ] **Step 5: Wire `__call__` — replace the NotImplementedError with the guard + dispatch** + +In `Svar2Haps.__call__`, replace this block: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + if issubclass(self.kind, _FlatVariantWindows): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('variant-windows') yet." + ) +``` + +with: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + # variants AND variant-windows decode variants; the read-bound decode + # has NO right-clip, so max_jitter>0 / jitter>0 would over-include + # variants past the (unpadded) read window. Guard both modes. + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." + ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) +``` + +Then DELETE the now-duplicated jitter guard that follows in the old `RaggedVariants` branch (the block starting `# ``decode_variants_from_svar2_readbound`` has NO right-clip` down through its `raise NotImplementedError(... "right-clip" ...)`), since the guard now runs once above for both kinds. Keep the trailing `return cast(_H, self._reconstruct_variants(idx, regions))`. + +The resulting branch reads: + +```python + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." + ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) + # RaggedVariants: RC is applied by the caller (_getitem_unspliced), + # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). + return cast(_H, self._reconstruct_variants(idx, regions)) +``` + +- [ ] **Step 6: Run the ref_window test to confirm it passes** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_ref_window_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS + +- [ ] **Step 7: Add the alt-window decomposition test (correctness of the alt assembly)** + +`alt_window` is NOT SVAR1-identical (deletion ALT differs), so verify it against svar2's own outputs: since tokenization is per-byte independent, `alt_window[j] == ref_window[j][:L] · alt_allele[j] · ref_window[j][-L:]`, where `alt_allele` is the bare tokenized alt (a second svar2 windows read with `alt="allele"`) and `ref_window` is already validated against SVAR1 in Step 1. + +```python +def test_svar2_variant_windows_alt_window_decomposition( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """alt_window[j] == ref_window[j][:L] + tokenize(alt_j) + ref_window[j][-L:]. + Uses only svar2's own outputs; ref_window is separately pinned to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + w_win = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + alt_opt = VarWindowOpt( + flank_length=L, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="allele" + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + + rw = w_win.ref_window + aw = w_win.alt_window + ba = w_alt.alt # bare tokenized alt (_FlatWindow) + assert aw is not None and rw is not None and ba is not None + + # Same variant SET/order across the two reads. + assert np.array_equal(np.asarray(aw.var_offsets), np.asarray(ba.var_offsets)) + n_var = len(np.asarray(aw.seq_offsets)) - 1 + rso, aso, bso = ( + np.asarray(rw.seq_offsets), + np.asarray(aw.seq_offsets), + np.asarray(ba.seq_offsets), + ) + rd, ad, bd = np.asarray(rw.data), np.asarray(aw.data), np.asarray(ba.data) + for j in range(n_var): + rj = rd[rso[j] : rso[j + 1]] + aj = ad[aso[j] : aso[j + 1]] + bj = bd[bso[j] : bso[j + 1]] + expected = np.concatenate([rj[:L], bj, rj[len(rj) - L :]]) + assert np.array_equal(aj, expected), f"alt_window variant {j} mismatch" +``` + +- [ ] **Step 8: Add the bare-alt tokenization test (pins alt bytes → tokens)** + +Confirms the bare `alt` equals `tokenize(variants.alt)`, tying the window alt bytes to the validated variants decode. + +```python +def test_svar2_variant_windows_bare_alt_tokenizes_variants_alt( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + import awkward as ak + + from genvarloader._dataset._flat_flanks import build_token_lut + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + alt_opt = VarWindowOpt( + flank_length=L, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="allele" + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + v = ds2.with_seqs("variants")[:, :] # RaggedVariants (validated) + + lut, _ = build_token_lut(b"ACGT", 4) + # Per (b,p) row, list of alt byte-strings, in variant order. + alt_rows = ak.to_list(v.alt.to_ak()) # nested (b, p) -> [bytes,...] + flat_alts: list[bytes] = [] + for per_ploid in alt_rows: + for per_var in per_ploid: + for a in per_var: + flat_alts.append(bytes(a) if not isinstance(a, bytes) else a) + + ba = w_alt.alt + bso, bd = np.asarray(ba.seq_offsets), np.asarray(ba.data) + assert len(flat_alts) == len(bso) - 1 + for j, a in enumerate(flat_alts): + toks = bd[bso[j] : bso[j + 1]] + expected = np.array([lut[byte] for byte in a], dtype=toks.dtype) + assert np.array_equal(toks, expected), f"bare alt variant {j} mismatch" +``` + +- [ ] **Step 9: Add multi-contig parity test** + +```python +def test_svar2_variant_windows_multicontig(tmp_path, svar_fixture2, svar2_fixture2, _src2): + """ref_window byte-identical to SVAR1 across an interleaved 2-contig bed + (single-contig fast path bypassed -> exercises the group-stitch reorder).""" + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "vw_mc1.gvl" + d2 = tmp_path / "vw_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # alt_window decomposition holds across the stitch too. + w2.alt_window.to_ragged() # offsets/data consistent post-reorder + w2.ref_window.to_ragged() +``` + +- [ ] **Step 10: Add the dummy-variant empty-group fill test** + +The module `bed` has a variant-free tail (`chr1 [20, 40)`), so with a `dummy_variant` set, its rows must each get one all-`unknown_token` window of width `2L + len(dummy allele)`. + +```python +def test_svar2_variant_windows_dummy_fills_empty_groups( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + from genvarloader import DummyVariant + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + dummy = DummyVariant(alt=b"N", ref=b"N") + w = ( + ds2.with_output_format("flat") + .with_settings(dummy_variant=dummy) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Every (b*p) row now has >= 1 variant (no empty rows). + vo = np.asarray(w.ref_window.var_offsets) + assert np.all(np.diff(vo) >= 1) + # ref_window dummy width = 2L + len(dummy.ref); alt_window = 2L + len(dummy.alt). + # (For a filled row the sole variant's window length equals the dummy width.) + # Assert at least one dummy-width ref window exists (the tail region rows). + rso = np.asarray(w.ref_window.seq_offsets) + assert (np.diff(rso) == (2 * L + len(dummy.ref))).any() + w.ref_window.to_ragged() + w.alt_window.to_ragged() +``` + +- [ ] **Step 11: Add the guard tests (ref="allele" ValueError, jitter NotImplementedError)** + +```python +def test_svar2_variant_windows_ref_allele_guard(tmp_path, bed, svar2_fixture, _src): + """ref='allele' needs stored REF bytes svar2 lacks -> ValueError at with_seqs.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + bad = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="allele", alt="window" + ) + with pytest.raises(ValueError, match="REF"): + ds.with_seqs("variant-windows", bad) + + +def test_svar2_variant_windows_jitter_guard(tmp_path, svar2_fixture, _src): + """variant-windows must raise when written with max_jitter>0 (no right-clip).""" + from genoray import SparseVar2 + + _bcf, ref = _src + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, jbed, variants=SparseVar2(svar2_fixture), samples=None, max_jitter=2, overwrite=True + ) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variant-windows", _WIN_OPT)[:, :] +``` + +- [ ] **Step 12: Run the full Task-1 test set** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS (7 new tests). Also run the existing svar2 suite to confirm no regression: +`pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q --basetemp=$(pwd)/.pytest_tmp` → all pass. + +- [ ] **Step 13: Lint + commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): variant-windows read path (ref=window, alt window/allele) + +Compose decode_variants_from_svar2_readbound with assemble_variant_buffers +(identity gather) per contig group. ref_window byte-identical to SVAR1; +alt validated via ref-flank decomposition + tokenized variants.alt. Wire +__call__ (jitter guard shared with variants), pin ref=allele + jitter guards. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: unphased_union for svar2 `variants` mode + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variants`, `_guard_unsupported`) +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes: `self.unphased_union` (inherited `Haps` field, set via `with_settings`), the existing `_reconstruct_variants` structure. +- Produces: `_reconstruct_variants` honoring the ploidy-1 fold; `_guard_unsupported` no longer raises on `unphased_union`. + +- [ ] **Step 1: Write the failing test — variants union vs SVAR1** + +```python +def test_svar2_variants_unphased_union_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Ploidy-1 union: start/ilen byte-identical to SVAR1 union (order-preserving + fold, no dedup). ALT differs by encoding, so ALT is not compared to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants").with_settings(unphased_union=True)[:, :] + b = ds2.with_seqs("variants").with_settings(unphased_union=True)[:, :] + # Ploidy axis folded 2 -> 1. + assert a.start.shape[-2] == 1 and b.start.shape[-2] == 1 + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variants_unphased_union_matches_svar1" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL with `NotImplementedError: unphased_union is not supported for svar2 datasets yet.` + +- [ ] **Step 3: Remove the unphased_union clause from `_guard_unsupported`** + +In `Svar2Haps._guard_unsupported`, delete: + +```python + if self.unphased_union: + raise NotImplementedError( + "unphased_union is not supported for svar2 datasets yet." + ) +``` + +(Haplotypes/annotated + union is still blocked at `with_seqs` in `_impl.py`, so the haplotypes/tracks paths can never reach here with the flag set.) + +- [ ] **Step 4: Add the fold to `_reconstruct_variants`** + +In `_reconstruct_variants`, after `P = int(self.genotypes.shape[-2])` and after the `groups = self._contig_groups(...)` line, introduce `p_eff`. Then in the per-group loop, fold `var_off` before appending, and use `p_eff` for the shape and stitch. Concretely: + +1. After `groups = self._contig_groups(contig_ids)` add: + ```python + p_eff = 1 if self.unphased_union else P + ``` +2. In the loop, the group currently does `var_off = np.asarray(var_off, np.int64)` then `cat_var_lens.append(np.diff(var_off))`. Insert the fold immediately after `var_off = np.asarray(var_off, np.int64)`: + ```python + if self.unphased_union: + var_off = np.ascontiguousarray(var_off[::P]) + ``` + (This keeps every P-th boundary: hap-0's then hap-1's variants per query, concatenated. `pos`/`ilen`/`alt` data and `str_off` are untouched — only row grouping changes.) +3. Replace both `shape = (b, P, None)` occurrences (single-group fast path and multi-group path) with `shape = (b, p_eff, None)`. +4. Replace the single `perm = self._inverse_row_perm(cat_query_order, b, P)` with `perm = self._inverse_row_perm(cat_query_order, b, p_eff)`. + +- [ ] **Step 5: Run the union test + the existing variants parity tests** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variants" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS — the new union test plus the existing `test_svar2_variants_positions_match_svar1`, `test_svar2_variants_match_svar2_oracle`, `test_svar2_variants_jitter_guard_raises` all still pass (p_eff=P when the flag is off preserves the diploid path byte-for-byte). + +- [ ] **Step 6: Commit** + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): unphased_union for variants mode (ploidy-1 fold) + +Fold row_offsets[::P], eff_ploidy=1 per contig group (order-preserving, +no dedup) — byte-identical to SVAR1 union for start/ilen. Drop the +unphased_union guard; haplotypes/annotated+union stays blocked upstream. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: unphased_union for svar2 `variant-windows` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variant_windows`) +- Test: `tests/dataset/test_svar2_dataset.py` + +**Interfaces:** +- Consumes: `_reconstruct_variant_windows` from Task 1 (the `p_eff = P` placeholder and `row_off = var_off` comment mark the fold points), `self.unphased_union`. +- Produces: variant-windows honoring the ploidy-1 fold. + +- [ ] **Step 1: Write the failing test — windows union** + +```python +def test_svar2_variant_windows_unphased_union( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Union folds ploidy 2->1 for windows; ref_window still byte-identical to + SVAR1 union, and the union row is hap-0's windows then hap-1's, concatenated.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ( + ds1.with_output_format("flat") + .with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Ploidy axis folded 2 -> 1. Scalar shape is (R,S,p_eff,None) so ploidy is at + # [-2]; window shape is (R,S,p_eff,None,None) so ploidy is at [-3]. + assert w2.fields["start"].shape[-2] == 1 + assert w2.ref_window.shape[-3] == 1 + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # Union row count == sum over haplotypes: compare to the non-union var counts. + nu = np.asarray(w2.ref_window.var_offsets) + w2_diploid = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + nd = np.asarray(w2_diploid.ref_window.var_offsets) + P = int(ds2._seqs.genotypes.shape[-2]) + # Folded per-row counts == sum of the P per-hap counts (rows q*P+p are contiguous). + diploid_counts = np.diff(nd).reshape(-1, P).sum(1) + union_counts = np.diff(nu) + assert np.array_equal(union_counts, diploid_counts) + w2.ref_window.to_ragged() + w2.alt_window.to_ragged() +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py::test_svar2_variant_windows_unphased_union" -v --basetemp=$(pwd)/.pytest_tmp` +Expected: FAIL — the ploidy axis is still 2 (`assert ... shape[-2] == 1` fails), because Task 1 hardcodes `p_eff = P` and `row_off = var_off`. + +- [ ] **Step 3: Apply the fold in `_reconstruct_variant_windows`** + +Two edits in the method: + +1. Replace `p_eff = P # unphased_union fold (Task 3) sets this to 1 per group.` with: + ```python + p_eff = 1 if self.unphased_union else P + ``` +2. Replace `row_off = var_off # Task 3: fold to var_off[::P] under unphased_union.` with: + ```python + row_off = np.ascontiguousarray(var_off[::P]) if self.unphased_union else var_off + ``` + +(The `v_idxs = arange(n_var)` identity, `pos`/`ilen`/`alt_bytes`, and the assemble call are unchanged — folding `row_off` only regroups the per-row variant boundaries the kernel emits. `shape`/`wshape` already use `p_eff`, and the multi-group stitch already uses `_inverse_row_perm(cat_query_order, b, p_eff)`.) + +- [ ] **Step 4: Run the windows union test + the whole variant_windows set** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` +Expected: PASS (Task-1 tests unaffected — `p_eff=P` when the flag is off — plus the new union test). + +- [ ] **Step 5: Add a multi-contig + union test (locks the p_eff=1 stitch)** + +```python +def test_svar2_variant_windows_union_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + {"chrom": ["chr2", "chr1"], "chromStart": [0, 0], "chromEnd": [40, 40]} + ) + d1 = tmp_path / "vwu_mc1.gvl" + d2 = tmp_path / "vwu_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = (ds1.with_output_format("flat").with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :]) + w2 = (ds2.with_output_format("flat").with_settings(unphased_union=True) + .with_seqs("variant-windows", _WIN_OPT)[:, :]) + assert w2.ref_window.shape[-3] == 1 # window ploidy axis + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window (union, multicontig)") + w2.alt_window.to_ragged() +``` + +- [ ] **Step 6: Run + commit** + +Run: `pixi run -e dev pytest "tests/dataset/test_svar2_dataset.py" -v -k "variant_windows" --basetemp=$(pwd)/.pytest_tmp` → PASS. + +```bash +pixi run -e dev ruff check python/ tests/ && pixi run -e dev ruff format python/ tests/ +git add python/genvarloader/_dataset/_svar2_haps.py tests/dataset/test_svar2_dataset.py +git commit -m "feat(svar2): unphased_union for variant-windows (ploidy-1 fold) + +Fold row_offsets[::P] before the window assemble call; p_eff=1 drives +shape + stitch. ref_window stays SVAR1-identical under union. + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: docs, skill, and spec out-of-scope updates + +**Files:** +- Modify: `skills/genvarloader/SKILL.md` +- Modify: `docs/source/*.md` (whichever enumerate svar2 output-mode support — grep first) +- Modify: `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` (out-of-scope note) + +**Interfaces:** none (docs only). + +- [ ] **Step 1: Find where svar2 mode support / variant-windows is documented** + +Run: +```bash +grep -rn "variant-windows\|svar2\|SVAR2\|NotImplementedError" skills/genvarloader/SKILL.md docs/source/*.md | grep -i "svar2\|variant-windows" +``` +Read each hit and note which claim "svar2 does not support variant-windows / unphased_union" (or list supported modes). + +- [ ] **Step 2: Update the skill** + +In `skills/genvarloader/SKILL.md`, wherever svar2's supported output modes or "not supported" gotchas are listed, state that svar2 now supports `variant-windows` (`ref="window"`, `alt ∈ {window, allele}`) and `unphased_union` (for `variants` and `variant-windows`). Keep the still-unsupported list accurate: `ref="allele"`, `min_af`/`max_af`, spliced, annotated, in-kernel RC, and `max_jitter>0`/`jitter>0` for the variants/variant-windows decode. + +- [ ] **Step 3: Update user docs** + +Apply the same correction to any `docs/source/*.md` hit from Step 1 (e.g. a support matrix in `dataset.md`/`faq.md`). If no doc enumerates svar2 mode support, note that in the commit message and skip. + +- [ ] **Step 4: Update the read-bound perf spec's out-of-scope note** + +In `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` §2, the "Out of scope" line lists `variant-windows` and `unphased_union` among guarded modes. Add a parenthetical that these are now implemented (see `2026-07-06-svar2-variant-windows-design.md`); the perf/fusion work remains deferred. + +- [ ] **Step 5: Verify api.md unchanged is correct** + +No new public symbol is added, so `api.md`/`__all__` need no change. Confirm: +```bash +python -c "import re,genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')" +``` +Expected: `MISSING: none`. + +- [ ] **Step 6: Commit** + +```bash +git add skills/genvarloader/SKILL.md docs/source/ docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md +git commit -m "docs(svar2): variant-windows + unphased_union now supported + +Update skill + user docs mode matrix; note the read-bound perf spec's +out-of-scope items are implemented (fusion still deferred). + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Final verification (after all tasks) + +- [ ] **Full svar2 suite** + +Run: `pixi run -e dev pytest tests -k svar2 -q --basetemp=$(pwd)/.pytest_tmp` +Expected: all pass. + +- [ ] **Full tree (catches stale references in tests/unit/)** + +Run: `pixi run -e dev pytest tests -q --basetemp=$(pwd)/.pytest_tmp` +Expected: all pass (matches the pre-change green baseline: ~1030 pytest). + +- [ ] **Typecheck + cargo (insurance; no Rust changed)** + +Run: `pixi run -e dev typecheck && pixi run -e dev cargo-test` +Expected: pass. + +--- + +## Self-Review notes (for the executor) + +- **Spec coverage:** Task 1 = windows core + wiring + guards (§3 core, §4 ref_window/alt oracle, §4.5 guards); Task 2 = variants union (§3 "_reconstruct_variants gets the same fold", §4.6); Task 3 = windows union (§3 step 1a, §4.6); Task 4 = docs (§6). `ref="allele"` (blocked upstream) and jitter guards are pinned by tests in Task 1. +- **Type consistency:** `_reconstruct_variant_windows` returns `_FlatVariantWindows`; `_FlatWindow(data, seq_offsets, var_offsets, shape)` positional order matches the dataclass in `_flat_variants.py`; `_assemble_variant_buffers_rust` arg order matches its definition (mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char). +- **Union fold identity:** `p_eff` and `row_off = var_off[::P]` are the ONLY union-specific changes in each reconstruct method; when the flag is off both reduce to the pre-existing diploid path, so the existing parity tests must remain green (a regression signal). diff --git a/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md b/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md new file mode 100644 index 00000000..f32a7c1b --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-svar2-info-format-field-routing.md @@ -0,0 +1,1000 @@ +# SVAR2 INFO/FORMAT Field Routing — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Route arbitrary scalar-numeric SVAR2 INFO/FORMAT fields through gvl's read-bound decode kernel into `RaggedVariants` and variant-windows (`_FlatVariantWindows`) outputs. + +**Architecture:** The seam is Rust-side: gvl's `decode_variants_from_svar2_readbound` kernel gains provenance tracking (via genoray's `gather_haps_readbound_src` + `unpack_vk_src`/`dense_abs_row`) and gathers per-variant field bytes via genoray's exported `FieldView`. Field byte-buffers ride the existing `pos`-parallel offset machinery, so multi-contig reorder and `unphased_union` need no special handling. Python discovers the store's field manifest and wraps the returned buffers into the output types. + +**Tech Stack:** Rust (PyO3, `genoray_core` crate path-dep, `ndarray`/`numpy`), Python (numpy, `seqpro.rag.Ragged`), pixi, genoray SVAR2 field-read API (unreleased, from genoray main). + +## Global Constraints + +- **genoray dependency:** build a wheel from genoray **main** (`/carter/users/dlaub/projects/genoray`, HEAD ~`acc59cb`, which has the field-read API — it is 87 commits past the `3.0.0` tag and UNRELEASED). Rust links it via the existing `Cargo.toml` path-dep. Do **not** modify genoray. Do **not** hand-bump genoray's version. +- **Drop gvl's genoray version pin** (`pyproject.toml` `"genoray>=3,<4"` → `"genoray"`); re-pin at genoray release. The wheel will report version `2.15.0` — acceptable once the pin is dropped. +- **Field types supported:** scalar-numeric only — INFO/FORMAT `Type=Integer/Float` (+ `Flag` for INFO), `Number` `1` or `A`. Non-scalar fields are rejected at genoray's write boundary and never reach gvl. +- **Dtype fidelity:** field values are passed as raw little-endian bytes + an itemsize from Rust; Python `.view(dtype)`s them. No widening/conversion. Missing entries carry genoray's stored `default`/sentinel verbatim. +- **Zero-overhead no-field path:** when no fields are requested, the kernel must behave byte-identically to today (call `gather_haps_readbound`, no provenance, no `FieldView`). +- **NFS build gotchas (from genoray dev notes):** export `CARGO_TARGET_DIR=/tmp/gvl-target` before `cargo`/`maturin` on this NFS checkout to avoid linker bus errors; genoray's own Rust tests need `--no-default-features`. +- **Commit convention:** Conventional Commits (`feat:`/`fix:`/`refactor:`/`test:`/`docs:`/`chore:`). All commits to branch `svar2-m6b-kernel`. Never touch `main`. +- **Pre-commit hook caveat:** the `pyrefly-check` pre-commit hook fails to solve the env until Phase 0 Task 2 completes. Until then, commit with `git commit --no-verify`. **After** Task 2, drop `--no-verify` and let hooks run. + +## File Structure + +**Rust (crate root `/carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel`):** +- `src/ffi/mod.rs` — FFI entry `decode_variants_from_svar2_readbound` (`:1330`); gains a `fields` param + field-buffer outputs. Also the tuple→Range migration at all `HapRanges::new` call sites (`:963,1116,1227,1372`). +- `src/svar2/mod.rs` — `decode_variants_from_split` (`:~282`) gains optional provenance capture + field gather; `split_to_flat` (`:~158`) tuple→Range fix; new `FieldGather` struct; tests. +- `src/svar2/store.rs` — `Svar2Store` retains `store_path` + accessor (needed to rebuild `ContigPaths` for `FieldView`). +- `Cargo.toml` — no code change (already path-deps genoray main); may need `use` of new genoray symbols in the `.rs` files. + +**Python:** +- `python/genvarloader/_dataset/_svar2_haps.py` — field discovery in `from_path`/`__post_init__`; field pass-through + wrapping in `_reconstruct_variants` and `_reconstruct_variant_windows`. +- `python/genvarloader/_dataset/_impl.py` — `var_fields` request block (`:338-369`): add an early SVAR2 branch that skips SVAR1 lazy-loading. + +**Config / docs:** +- `pyproject.toml` (`:14`), `pixi.toml` (`:110`) — dependency. +- `tests/dataset/test_svar2_fields_read.py` (new) — integration oracle test. +- `skills/genvarloader/SKILL.md`, `CHANGELOG.md` — docs. + +--- + +# Phase 0 — Baseline: migrate gvl to genoray main (green build, no feature yet) + +> gvl's Rust was written against genoray 2.15.0, where `BatchResultSplit`/`HapRanges` range fields were `(usize,usize)` tuples. genoray's `505a37f` refactor changed them to `Range` (present since the `3.0.0` tag). The "adopt genoray 3.0.0 API" commit touched **only Python** (0 `.rs` files), so the Rust does not compile against current genoray. Phase 0 makes it compile and pass the existing test suite against genoray main, establishing the baseline the field feature builds on. + +### Task 0.1: Build a genoray wheel from main + +**Files:** +- None in gvl (produces a wheel on disk). + +- [ ] **Step 1: Build + repair the wheel** + +```bash +cd /carter/users/dlaub/projects/genoray +export CARGO_TARGET_DIR=/tmp/genoray-wheel-target +pixi run --manifest-path ci/wheel/pixi.toml build # -> wheelhouse/*.whl (abi3 cp310) +pixi run --manifest-path ci/wheel/pixi.toml repair # auditwheel -> dist/*.whl (manylinux) +ls -la dist/*.whl +``` + +Expected: a `genoray-2.15.0-cp310-abi3-manylinux_*_x86_64.whl` in `dist/`. + +- [ ] **Step 2: Sanity-check the wheel has the field-read API** + +```bash +cd /carter/users/dlaub/projects/genoray +python -c "import zipfile,glob; z=zipfile.ZipFile(glob.glob('dist/genoray-*-abi3-manylinux*.whl')[0]); src=z.read('genoray/_svar2.py').decode(); print('available_fields' in src, 'from_pgen' in src)" +``` + +Expected: `True True`. + +- [ ] **Step 3: Commit** — nothing to commit (artifact only). Record the wheel path for Task 0.2. + +### Task 0.2: Point gvl at the wheel + drop the version pin + +**Files:** +- Modify: `pyproject.toml:14` +- Modify: `pixi.toml:110` + +**Interfaces:** +- Produces: a solvable pixi env with genoray main (field-read API) importable. + +- [ ] **Step 1: Drop the version constraint** + +In `pyproject.toml`, change the dependency line: + +```toml + "genoray", +``` + +(from `"genoray>=3,<4",`). + +- [ ] **Step 2: Point the pixi pin at the new wheel** + +In `pixi.toml`, replace the `genoray = { path = ... }` line (`:110`) with the manylinux wheel built in Task 0.1 (use the actual filename from `dist/`): + +```toml +genoray = { path = "/carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl" } +``` + +- [ ] **Step 3: Re-solve + verify import** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +pixi install +pixi run python -c "from genoray import SparseVar2; print(hasattr(SparseVar2,'available_fields'), hasattr(SparseVar2,'with_fields'))" +``` + +Expected: solve succeeds; prints `True True`. + +- [ ] **Step 4: Commit** + +```bash +git add pyproject.toml pixi.toml pixi.lock +git commit --no-verify -m "chore(svar2): pin genoray main wheel, drop version constraint for dev" +``` + +### Task 0.3: Migrate gvl Rust range types tuple→Range + +**Files:** +- Modify: `src/ffi/mod.rs` (the `to_pairs` closure + all `HapRanges::new` sites `:963,1116,1227,1372`; the `decode_variants_from_svar2_readbound` closure `:~1356-1370`) +- Modify: `src/svar2/mod.rs` (`split_to_flat` `:~158`, `decode_variants_from_split` `:~300`, and the `#[cfg(test)]` `BatchResultSplit` literals) + +**Interfaces:** +- Consumes: `genoray_core::query::HapRanges::new(&[u32], &[usize], &[Range], &[Range], &[Range], &[Range], usize)`; `BatchResultSplit { dense_snp_range: Vec>, dense_indel_range: Vec>, vk_src: Vec, dense_src: Vec<(bool,u32)>, .. }` (derives `Default`). +- Produces: a gvl crate that compiles against genoray main; behavior byte-identical (no feature). + +- [ ] **Step 1: Convert the FFI range marshaling to `Range`** + +In `src/ffi/mod.rs`, replace the `to_pairs` closure (in `decode_variants_from_svar2_readbound`, ~`:1356`) and the analogous marshaling in the reconstruct/diffs/tracks FFI fns with a `to_ranges` helper. Add near the top of the module: + +```rust +use std::ops::Range; + +fn arr2_to_ranges(a: numpy::ndarray::ArrayView2) -> Vec> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize)..(r[1] as usize)) + .collect() +} +``` + +Then at each site that built `Vec<(usize,usize)>` for a `HapRanges::new` arg (search `to_pairs` and inline `.map(|r| (r[0] as usize, r[1] as usize))`), replace with `arr2_to_ranges(.as_array())`. The `HapRanges::new(...)` calls themselves are unchanged (they now receive `&[Range]`). + +- [ ] **Step 2: Convert `BatchResultSplit` range consumption in `src/svar2/mod.rs`** + +In `split_to_flat` and `decode_variants_from_split`, every `let (ss, se) = br.dense_snp_range[q];` / `let (is_, ie) = br.dense_indel_range[q];` becomes a `Range` read. Use `.clone()`-free field access: + +```rust +let Range { start: ss, end: se } = br.dense_snp_range[q]; +let Range { start: is_, end: ie } = br.dense_indel_range[q]; +``` + +(add `use std::ops::Range;` to `src/svar2/mod.rs`). All downstream uses of `ss/se/is_/ie` are unchanged (they're `usize`). + +- [ ] **Step 3: Fix the `#[cfg(test)]` `BatchResultSplit` literals** + +In `src/svar2/mod.rs` tests, every `dense_snp_range: vec![(0, 1)]` etc. becomes `dense_snp_range: vec![0..1]`, `vec![(0, 2), (2, 4)]` becomes `vec![0..2, 2..4]`, etc. `BatchResultSplit` derives `Default`, so add `..Default::default()` at the end of each struct literal to cover the new `vk_src`/`dense_src` fields: + +```rust +let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 2, + vk: vec![/* ... */], + vk_off: vec![/* ... */], + dense_snp: vec![/* ... */], + dense_snp_range: vec![0..1], + // ... other existing fields ... + ..Default::default() +}; +``` + +- [ ] **Step 4: Compile to green** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -30 +``` + +Expected: `Finished`. If additional tuple→Range or new-field errors surface (compiler-driven; the refactor is mechanical), fix each at the reported site exactly as in Steps 1–3, then re-run until green. + +- [ ] **Step 5: Run the Rust unit tests** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features 2>&1 | tail -20 +``` + +Expected: all existing svar2 Rust tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/ffi/mod.rs src/svar2/mod.rs +git commit --no-verify -m "refactor(svar2): migrate gvl read-bound Rust to genoray Range API" +``` + +### Task 0.4: Build the extension + green existing test suite (baseline gate) + +**Files:** +- None (build + test). + +- [ ] **Step 1: Build the editable extension** + +```bash +cd /carter/users/dlaub/projects/GenVarLoader/.claude/worktrees/svar2-m6b-kernel +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -15 +``` + +Expected: `Built ... Installed genvarloader`. + +- [ ] **Step 2: Run the existing SVAR2 test suite** + +```bash +pixi run pytest tests/dataset/test_svar2_readbound_variants.py tests/dataset/test_svar2_readbound_haps.py -x -q 2>&1 | tail -25 +``` + +Expected: all PASS (baseline against genoray main; no feature yet). This gate proves the migration is correct before adding fields. + +- [ ] **Step 3: Confirm the pre-commit hook now solves** + +```bash +git commit --allow-empty -m "chore: verify hooks solve" && git reset --soft HEAD~1 +``` + +Expected: no pyrefly solve error. From here on, commit **without** `--no-verify`. + +--- + +# Phase 1 — Rust: provenance + field gather + +### Task 1.1: `Svar2Store` retains `store_path` + +**Files:** +- Modify: `src/svar2/store.rs` + +**Interfaces:** +- Produces: `Svar2Store::store_path(&self) -> &str` — the store base dir, for rebuilding `ContigPaths`. + +- [ ] **Step 1: Add the field + accessor** + +In `src/svar2/store.rs`, add `store_path: String` to the struct, set it in `new`, and expose it: + +```rust +#[pyclass] +pub struct Svar2Store { + readers: HashMap, + store_path: String, +} + +impl Svar2Store { + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } + pub fn store_path(&self) -> &str { + &self.store_path + } +} +``` + +In `new`, capture `store_path` before the loop consumes `contigs` (note: `store_path: &str` param): + +```rust + let store_path = store_path.to_string(); + // ... existing loop populating `readers` ... + Ok(Self { readers, store_path }) +``` + +- [ ] **Step 2: Compile** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -10 +``` + +Expected: `Finished`. + +- [ ] **Step 3: Commit** + +```bash +git add src/svar2/store.rs +git commit -m "feat(svar2): Svar2Store retains store_path for field-sidecar paths" +``` + +### Task 1.2: `FieldGather` + provenance capture in `decode_variants_from_split` + +**Files:** +- Modify: `src/svar2/mod.rs` +- Test: `src/svar2/mod.rs` (`#[cfg(test)]`) + +**Interfaces:** +- Consumes: `genoray_core::query::{unpack_vk_src, dense_abs_row, FieldView}`; `BatchResultSplit.vk_src` (populated only by `gather_haps_readbound_src`). +- Produces: + ```rust + pub struct FieldGather<'a> { + pub views: [genoray_core::query::FieldView; 4], // FieldSub order: VkSnp,VkIndel,DenseSnp,DenseIndel + pub is_format: bool, + pub width: usize, // dtype.width_bytes() + pub cohort_n_samples: usize, + pub _marker: std::marker::PhantomData<&'a ()>, + } + pub fn decode_variants_from_split( + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], + fields: &[FieldGather<'_>], + on_disk_snp: &[std::ops::Range], // HapRanges dense_snp_range; empty if no fields + on_disk_indel: &[std::ops::Range], + orig_samples: &[usize], + ) -> (VariantsSoa, Vec>) // second: one byte-buffer per field, parallel to pos + ``` + When `fields.is_empty()`, the second return is `Vec::new()` and no provenance work runs (byte-identical to today). + +- [ ] **Step 1: Write the failing Rust test (provenance identity)** + +Add to `src/svar2/mod.rs` `#[cfg(test)] mod tests`. This test builds a split with one var_key entry and one dense-snp entry and an INFO field whose stored value at each source row equals that row index, then asserts the decoded field values match the merge order: + +```rust +#[test] +fn test_decode_fields_provenance_identity() { + use genoray_core::query::{pack_vk_src, KeyRef}; + // One query, ploidy 1. var_key call idx 5 at pos 10; dense-snp row (abs) 3 at pos 20. + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { position: 10, key: svar2_codec::encode_pure_del(-1) }], + vk_off: vec![0, 1], + vk_src: vec![pack_vk_src(false, 5)], // var_key/snp, call idx 5 + dense_snp: vec![KeyRef { position: 20, key: svar2_codec::encode_pure_del(-1) }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![], + dense_indel_range: vec![0..0], + dense_indel_present: vec![], + dense_indel_present_off: vec![0, 0], + }; + // INFO field: value_at(i) == i (an i32 store big enough for idx 5 and dense row 3). + // Build FieldView test doubles via a helper that fills values.bin so value_at(i)=i. + let fields = make_identity_i32_fields(); // see Step 3 helper + let (soa, bufs) = decode_variants_from_split( + &br, &[], &[0i64], &fields, &[0..1], &[0..0], &[0], + ); + assert_eq!(soa.pos, vec![10, 20]); // var_key before dense on distinct positions + // Field buffer holds i32 LE bytes: [5, 3] (vk call idx 5 first, dense abs row 3 second). + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(vals, vec![5, 3]); +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features test_decode_fields_provenance_identity 2>&1 | tail -15 +``` + +Expected: FAIL — `decode_variants_from_split` arity mismatch / `make_identity_i32_fields` undefined. + +- [ ] **Step 3: Implement `FieldGather`, the test helper, and the gather** + +Add the struct + `use` lines at the top of `src/svar2/mod.rs`: + +```rust +use std::ops::Range; +use genoray_core::query::{dense_abs_row, unpack_vk_src, FieldView}; + +pub struct FieldGather { + pub views: [FieldView; 4], // FieldSub::all() order: VkSnp, VkIndel, DenseSnp, DenseIndel + pub is_format: bool, + pub width: usize, + pub cohort_n_samples: usize, +} +``` + +Change `decode_variants_from_split`'s signature to the one in **Interfaces** above. Inside the merge loop, at the point where `(p, key)` is selected, ALSO record which channel + index it came from, then (only if `!fields.is_empty()`) resolve provenance and append field bytes. Concretely, replace the selection block: + +```rust + let (p, key, chan, cidx) = if has_vk && p_vk <= p_sn && p_vk <= p_in { + let e = &br.vk[i_vk]; + let out = (e.position, e.key, 0u8, i_vk); + i_vk += 1; + out + } else if has_sn && p_sn <= p_in { + let e = &br.dense_snp[i_sn]; + let out = (e.position, e.key, 1u8, i_sn); + i_sn += 1; + out + } else { + let e = &br.dense_indel[i_in]; + let out = (e.position, e.key, 2u8, i_in); + i_in += 1; + out + }; + let (il, alt) = decode_alt(key, lut_bytes, lut_off); + pos.push(p as i32); + ilen.push(il as i32); + alt_bytes.extend_from_slice(&alt); + str_off.push(alt_bytes.len() as i64); + + if !fields.is_empty() { + // Resolve (is_dense, is_indel, elem_row) for this emitted variant. + let (is_dense, is_indel, row) = match chan { + 0 => { + let (is_indel, call_idx) = unpack_vk_src(br.vk_src[cidx]); + (false, is_indel, call_idx) + } + 1 => (true, false, dense_abs_row(&on_disk_snp[q], &(ss..se), cidx)), + _ => (true, true, dense_abs_row(&on_disk_indel[q], &(is_..ie), cidx)), + }; + let sub_ix = match (is_dense, is_indel) { + (false, false) => 0, + (false, true) => 1, + (true, false) => 2, + (true, true) => 3, + }; + for (fi, f) in fields.iter().enumerate() { + let view = &f.views[sub_ix]; + let elem = if is_dense && f.is_format { + row * f.cohort_n_samples + orig_samples[q] + } else { + row + }; + field_bufs[fi].extend_from_slice(view.bytes_at(elem)); + } + } +``` + +Note: the var_key `sub_ix` uses `is_indel` from `unpack_vk_src` (a var_key entry may be snp OR indel), matching `FieldSub` order. Declare `field_bufs` before the loop: + +```rust + let mut field_bufs: Vec> = (0..fields.len()).map(|_| Vec::new()).collect(); +``` + +and return `(VariantsSoa { .. }, field_bufs)`. + +`ss/se/is_/ie` are already in scope per query (from Task 0.3 Step 2). Update the three existing callers of `decode_variants_from_split` in `src/ffi/mod.rs` (variants, and the two reconstruct/windows paths that call it — grep `decode_variants_from_split`) to pass `&[]`, `&[]`, `&[]`, `&[]` for the no-field case for now (Task 1.3 wires the real args for the variants FFI). + +Add the test helper `make_identity_i32_fields()` in `#[cfg(test)]`. Because `FieldView::open` reads real sidecar files, the helper writes a tiny temp store dir with `values.bin` files whose i32 elements equal their index (0,1,2,...) for each of the 4 subs, then opens 4 `FieldView`s via `genoray_core::query::FieldView::open(&ContigPaths::new(tmp, "chr1"), "info", "X", sub, StorageDtype::from_meta_str("i32"), 1)` for `sub in FieldSub::all()`. (Use `tempfile::tempdir()`; add `tempfile` as a dev-dependency in `Cargo.toml` if not present.) + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo test --no-default-features test_decode_fields_provenance_identity 2>&1 | tail -15 +``` + +Expected: PASS. Also re-run the full Rust suite to confirm the no-field callers still pass: + +```bash +pixi run cargo test --no-default-features 2>&1 | tail -15 +``` + +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/svar2/mod.rs Cargo.toml +git commit -m "feat(svar2): field-byte gather with vk_src/dense provenance in split decode" +``` + +### Task 1.3: FFI — `decode_variants_from_svar2_readbound` gains fields + +**Files:** +- Modify: `src/ffi/mod.rs` (`:1330`) + +**Interfaces:** +- Consumes: `genoray_core::query::{gather_haps_readbound_src, FieldView}`, `genoray_core::layout::{ContigPaths, FieldSub}`, `genoray_core::field::StorageDtype`, `Svar2Store::store_path`, `ContigReader::n_samples`. +- Produces: Python signature + ``` + decode_variants_from_svar2_readbound( + store, contig, region_starts, orig_samples, + vk_snp_range, vk_indel_range, dense_snp_range, dense_indel_range, + ploidy, + fields: list[tuple[str, str, str]], # (category, name, dtype_str); may be [] + ) -> (pos, ilen, alt_bytes, str_off, var_off, + field_bufs: list[np.ndarray[u8]], field_itemsizes: list[int]) + ``` + +- [ ] **Step 1: Add the `fields` param and open `FieldView`s** + +Add `fields: Vec<(String, String, String)>` as the last param. After resolving `reader`, and only when `!fields.is_empty()`, build a `Vec`: + +```rust + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + + let n_samples = reader.n_samples(); + let paths = ContigPaths::new(store.store_path(), contig); + let gathers: Vec = fields + .iter() + .map(|(cat, name, dtype_str)| { + let dtype = StorageDtype::from_meta_str(dtype_str) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + let width = dtype.width_bytes().ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("field {name}: unresolved dtype")) + })?; + let subs = FieldSub::all(); + let views = [ + FieldView::open(&paths, cat, name, subs[0], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[1], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[2], dtype, n_samples), + FieldView::open(&paths, cat, name, subs[3], dtype, n_samples), + ]; + let mut opened = Vec::with_capacity(4); + for v in views { + opened.push(v.map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?); + } + Ok(crate::svar2::FieldGather { + views: [opened.remove(0), opened.remove(0), opened.remove(0), opened.remove(0)], + is_format: cat == "format", + width, + cohort_n_samples: n_samples, + }) + }) + .collect::>>()?; +``` + +(Confirm `StorageDtype::from_meta_str` returns `Result`; if it returns `Option`, adapt the `.map_err`. `FieldSub::all()` returns `[FieldSub; 4]` in order VkSnp,VkIndel,DenseSnp,DenseIndel.) + +- [ ] **Step 2: Choose the gather + call the decode with field args** + +Replace the `py.detach` body so it uses `gather_haps_readbound_src` when fields are requested, and passes the on-disk dense ranges + `orig_samples`: + +```rust + let has_fields = !gathers.is_empty(); + let (soa, field_bufs) = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, &orig_samples_v, + &vk_snp_range_v, &vk_indel_range_v, + &dense_snp_range_v, &dense_indel_range_v, ploidy, + ); + let br = if has_fields { + genoray_core::query::gather_haps_readbound_src(reader, &rb) + } else { + genoray_core::query::gather_haps_readbound(reader, &rb) + }; + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + crate::svar2::decode_variants_from_split( + &br, &lut_bytes, &lut_off, + &gathers, &dense_snp_range_v, &dense_indel_range_v, &orig_samples_v, + ) + }); +``` + +(Requires `dense_snp_range_v`/`dense_indel_range_v`/`orig_samples_v` to be `move`d in; they already are `Vec`s from Task 0.3. `gathers` is `move`d in.) + +- [ ] **Step 3: Return the field buffers + itemsizes** + +Change the return type to append `Vec>>` + `Vec`, and build them: + +```rust + let field_out: Vec>> = + field_bufs.into_iter().map(|b| Array1::from_vec(b).into_pyarray(py)).collect(); + let itemsizes: Vec = gathers.iter().map(|g| g.width).collect(); + Ok(( + Array1::from_vec(soa.pos).into_pyarray(py), + Array1::from_vec(soa.ilen).into_pyarray(py), + Array1::from_vec(soa.alt_bytes).into_pyarray(py), + Array1::from_vec(soa.str_off).into_pyarray(py), + Array1::from_vec(soa.var_off).into_pyarray(py), + field_out, + itemsizes, + )) +``` + +Update the return type in the signature to `PyResult<( ..5 existing.., Vec>>, Vec )>`. **Move `let itemsizes` computation before the `py.detach` closure consumes `gathers`** (compute widths into a plain `Vec` first, then move `gathers` into the closure). + +- [ ] **Step 4: Compile** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run cargo check 2>&1 | tail -20 +``` + +Expected: `Finished`. Fix borrow/move errors by hoisting `itemsizes` before the closure (Step 3 note). + +- [ ] **Step 5: Rebuild the extension** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -8 +``` + +Expected: `Installed genvarloader`. + +- [ ] **Step 6: Commit** + +```bash +git add src/ffi/mod.rs +git commit -m "feat(svar2): decode_variants_from_svar2_readbound returns INFO/FORMAT field buffers" +``` + +--- + +# Phase 2 — Python wiring + +### Task 2.1: Field discovery in `Svar2Haps` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`__post_init__` `:169`, `from_path` `:178`, add dataclass fields) + +**Interfaces:** +- Consumes: `genoray.SparseVar2(path).available_fields -> dict[str, StoredField]` where `StoredField.{name, category('info'|'format'), dtype(np.dtype), key}`; `genoray._svar2_fields._META_DTYPE: dict[np.dtype, str]`. +- Produces: `Svar2Haps.available_var_fields` includes store field keys; `Svar2Haps._store_fields: dict[str, tuple[str,str,str,np.dtype]]` keyed by field key → `(category, name, dtype_str, np_dtype)`. + +- [ ] **Step 1: Add dataclass fields + populate in `from_path`** + +Add two `field()`-defaulted slots to `Svar2Haps` (they follow base defaults): + +```python + store_field_keys: list[str] = field(default_factory=list) + store_fields: dict[str, tuple[str, str, str, "np.dtype"]] = field(default_factory=dict) + """key -> (category, name, dtype_str, np_dtype) for store INFO/FORMAT fields.""" +``` + +In `from_path`, after `sv = SparseVar2(str(svar2_path))`: + +```python + from genoray._svar2_fields import _META_DTYPE + + store_field_keys = list(sv.available_fields.keys()) + store_fields = { + sf.key: (sf.category, sf.name, _META_DTYPE[sf.dtype], sf.dtype) + for sf in sv.available_fields.values() + } +``` + +and pass `store_field_keys=store_field_keys, store_fields=store_fields` into the `cls(...)` call. + +- [ ] **Step 2: Extend `available_var_fields` in `__post_init__`** + +Replace the hard-coded line (`:174`): + +```python + self.available_var_fields = ["alt", "ilen", "start"] + [ + k for k in self.store_field_keys + if k not in {"alt", "ilen", "start", "ref", "dosage"} + ] +``` + +- [ ] **Step 3: Rebuild + smoke-check discovery** + +Requires a fixture store with fields (created in Task 3.1). For now compile-check the import: + +```bash +pixi run python -c "import genvarloader._dataset._svar2_haps" +``` + +Expected: no error. + +- [ ] **Step 4: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): Svar2Haps discovers store INFO/FORMAT fields into available_var_fields" +``` + +### Task 2.2: `_impl.py` — SVAR2 branch skips SVAR1 lazy-loading + +**Files:** +- Modify: `python/genvarloader/_dataset/_impl.py` (`:338-369`) + +**Interfaces:** +- Consumes: `Svar2Haps` (import), `self.available_var_fields`. +- Produces: for `Svar2Haps`, `var_fields` is validated + set via `replace(...)` with no SVAR1 lazy-loading. + +- [ ] **Step 1: Add the early SVAR2 branch** + +At the top of the `if var_fields is not None:` block (`:338`), before the `custom_fmt`/`load_info` logic, insert: + +```python + if var_fields is not None: + missing = list(set(var_fields) - set(self.available_var_fields)) + if missing or not isinstance(self._seqs, Haps): + raise ValueError(f"Missing variant fields: {missing}") + from ._svar2_haps import Svar2Haps + + if isinstance(self._seqs, Svar2Haps): + # SVAR2 fields are read on demand by the decode kernel; no lazy + # INFO/dosage/custom-FORMAT loading (that is SVAR1-only). + haps = replace(to_evolve.get("_seqs", self._seqs), var_fields=var_fields) + to_evolve["_seqs"] = haps + else: + # ... existing SVAR1 lazy-loading block (custom_fmt, load_info, ...) ... + haps = to_evolve.get("_seqs", self._seqs) + # (unchanged existing lines through) + haps = replace(haps, var_fields=var_fields) + to_evolve["_seqs"] = haps +``` + +(Keep the existing SVAR1 body verbatim inside the `else`. The `missing` check stays shared at the top.) + +- [ ] **Step 2: Compile-check** + +```bash +pixi run python -c "import genvarloader._dataset._impl" +``` + +Expected: no error. + +- [ ] **Step 3: Commit** + +```bash +git add python/genvarloader/_dataset/_impl.py +git commit -m "feat(svar2): skip SVAR1 field lazy-loading for Svar2Haps var_fields" +``` + +### Task 2.3: Route fields into `RaggedVariants` + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variants` `:558`) + +**Interfaces:** +- Consumes: the extended FFI (Task 1.3) returning `(pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_itemsizes)`. +- Produces: `RaggedVariants(**fields)` including each requested store field, sharing `alt`'s offsets. + +- [ ] **Step 1: Compute the requested field triples + call with fields** + +At the top of `_reconstruct_variants`, compute the requested extra fields (order stable): + +```python + builtin = {"alt", "start", "ref", "ilen", "dosage"} + req_keys = [f for f in self.var_fields if f not in builtin] + field_specs = [ + (self.store_fields[k][0], self.store_fields[k][1], self.store_fields[k][2]) + for k in req_keys + ] + field_dtypes = [self.store_fields[k][3] for k in req_keys] +``` + +Change the kernel call to pass `field_specs`, and unpack the two new returns per contig group: + +```python + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( + decode_variants_from_svar2_readbound( + self.store, self.ds_contigs[ci], + gi[0], gi[1], gi[2], gi[3], gi[4], gi[5], P, field_specs, + ) + ) +``` + +Accumulate per-field buffers per group into `cat_fields: list[list[np.ndarray]]` (one inner list per requested field), asserting `field_isizes[j] == field_dtypes[j].itemsize`. + +- [ ] **Step 2: Single-contig fast path — wrap fields** + +In the `len(cat_pos) == 1` branch, build a `fields` dict parallel to the existing `alt`/`start`/`ilen`, then splat into `RaggedVariants`: + +```python + extra = { + req_keys[j]: Ragged.from_offsets( + cat_fields[j][0].view(field_dtypes[j]), shape, var_off_g + ) + for j in range(len(req_keys)) + } + return RaggedVariants( + alt=Ragged.from_offsets(cat_alt[0].view("S1"), shape, var_off_g, str_offsets=str_off_g), + start=Ragged.from_offsets(cat_pos[0], shape, var_off_g), + ilen=Ragged.from_offsets(cat_ilen[0], shape, var_off_g), + **extra, + ) +``` + +- [ ] **Step 3: Multi-contig path — reorder fields by the same `src`** + +In the general path, after computing `src, var_off_g = _ragged_arange_src(grouped_var_off, perm)`, each field is per-variant so it reorders exactly like `pos`: + +```python + extra = {} + for j in range(len(req_keys)): + fc = np.concatenate([g[j] for g in per_group_fields]) if per_group_fields else np.zeros(0, np.uint8) + fc_typed = fc.view(field_dtypes[j]) + fg = fc_typed[:0].copy() if src.size == 0 else fc_typed[src] + extra[req_keys[j]] = Ragged.from_offsets(fg, shape, var_off_g) + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r, **extra) +``` + +(`per_group_fields[g][j]` is group `g`'s buffer for field `j`, already `.view(dtype)`-ed to length `n_var_group`. Reuse the same `src` computed for `pos_g`/`ilen_g`.) + +- [ ] **Step 4: Run the integration test (written in Task 3.1)** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -25 +``` + +Expected: `RaggedVariants` value assertions PASS (windows test may still fail until Task 2.4). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): route INFO/FORMAT fields into RaggedVariants" +``` + +### Task 2.4: Route fields into variant-windows (`_FlatVariantWindows`) + +**Files:** +- Modify: `python/genvarloader/_dataset/_svar2_haps.py` (`_reconstruct_variant_windows` `:653`) + +**Interfaces:** +- Consumes: the extended FFI; `_FlatVariantWindows.fields: dict[str, _Flat]`; `_Flat.from_offsets`. +- Produces: variant-windows output whose `.fields` includes each requested store field (like `start`/`ilen`). + +- [ ] **Step 1: Pass field_specs + collect per group** + +Mirror Task 2.3 Step 1 in `_reconstruct_variant_windows`: compute `req_keys`/`field_specs`/`field_dtypes` (identical code), pass `field_specs` to `decode_variants_from_svar2_readbound`, unpack `field_bufs, field_isizes`, and collect per-group typed buffers. + +- [ ] **Step 2: Add fields to the `fields` dict (single + multi-contig)** + +In the `len(cat_pos) == 1` branch, after `fields = {"start": ...}` (and optional `ilen`), add: + +```python + for j, k in enumerate(req_keys): + fields[k] = _Flat.from_offsets( + cat_fields[j][0].view(field_dtypes[j]), shape, row_off + ) +``` + +In the multi-contig `else`, after `pos_g`/`ilen_g`, reorder each field by the same `src` and add: + +```python + for j, k in enumerate(req_keys): + fc = np.concatenate([g[j] for g in per_group_fields]).view(field_dtypes[j]) + fg = fc[:0].copy() if src.size == 0 else fc[src] + fields[k] = _Flat.from_offsets(fg, shape, row_off_g) +``` + +- [ ] **Step 3: Extend `DummyVariant` fill for empty groups** + +`fill_empty_groups` needs a fill value per field. Set the dummy's per-field fill to the store default (or the dtype's sentinel: `NaN` for float, `iinfo.min` for int, `False` for bool) in `DummyVariant.info` keyed by field key when the dummy is constructed for svar2 (locate where `self.dummy_variant` is set for `Svar2Haps`; if none is set for windows, construct one carrying the field fills). Assert in the test that empty groups carry the fill. + +- [ ] **Step 4: Run the windows test** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -25 +``` + +Expected: all PASS (RaggedVariants + windows). + +- [ ] **Step 5: Commit** + +```bash +git add python/genvarloader/_dataset/_svar2_haps.py +git commit -m "feat(svar2): route INFO/FORMAT fields into variant-windows output" +``` + +--- + +# Phase 3 — Integration test + docs + +### Task 3.1: Integration oracle test (write this BEFORE Task 2.3 — it is the Phase-2 gate) + +**Files:** +- Create: `tests/dataset/test_svar2_fields_read.py` + +**Interfaces:** +- Consumes: `genoray.SparseVar2.from_vcf(info_fields=, format_fields=)`, `gvl.write`, `gvl.Dataset(...).with_seqs("variants", var_fields=[...])`. + +- [ ] **Step 1: Build the fixture VCF + convert with fields** + +Write a helper that emits a small bgzipped VCF over TWO contigs with variants that route to BOTH var_key and dense channels (a common variant → dense, a rare per-sample variant → var_key), carrying INFO `AF` (Float), INFO `NS` (Integer), FORMAT `DP` (Integer). Convert: + +```python +from genoray import SparseVar2 +from genoray._svar2_fields import InfoField, FormatField + +SparseVar2.from_vcf( + out=store_dir, source=vcf_gz, reference=fasta, + info_fields=[InfoField("AF"), InfoField("NS")], + format_fields=[FormatField("DP")], +) +``` + +- [ ] **Step 2: Write the ground-truth oracle** + +Parse the VCF with `cyvcf2` into a dict `{(contig, pos): {"AF": ..., "NS": ..., "DP": {sample: ...}}}`. This is the expected-value source. + +- [ ] **Step 3: Write the failing test (RaggedVariants values)** + +```python +import numpy as np +import pytest + +@pytest.mark.parametrize("union", [False, True]) +def test_svar2_ragged_variants_fields(tmp_path, union): + ds_path = _write_dataset(tmp_path) # gvl.write over the svar2 source + import genvarloader as gvl + ds = gvl.Dataset.open(ds_path, reference=FASTA).with_seqs( + "variants", var_fields=["alt", "start", "ilen", "AF", "NS", "DP"], + ) + if union: + ds = ds.with_settings(unphased_union=True) # use the real gvl API name + rv = ds[0] # RaggedVariants + # For each (batch,ploid) group, walk variants and compare rv.AF / rv.NS / rv.DP + # to the oracle by (contig, start). Assert exact dtype + value, incl. NaN for + # VCF-missing (use np.testing.assert_array_equal with equal_nan for floats). + _assert_fields_match(rv, oracle) # helper asserts every decoded field value +``` + +Add a second test hitting a multi-contig batch (indices spanning both contigs) and a third asserting the FORMAT `DP` value equals the oracle's per-sample value for the queried sample. + +- [ ] **Step 4: Run to verify it fails (before Task 2.3)** + +```bash +pixi run pytest tests/dataset/test_svar2_fields_read.py -x -q 2>&1 | tail -20 +``` + +Expected: FAIL — `AF` not in `available_var_fields` / not present on `rv`. (This test is the gate for Tasks 2.3/2.4; implement those to make it pass.) + +- [ ] **Step 5: Add the variant-windows value test** + +Add `test_svar2_variant_windows_fields` that requests variant-windows output with the same `var_fields` and asserts `win.fields["AF"]` etc. match the oracle, plus an empty-group case asserting the dummy fill. + +- [ ] **Step 6: Commit (test only; may be red until Phase 2 done)** + +```bash +git add tests/dataset/test_svar2_fields_read.py +git commit -m "test(svar2): oracle test for INFO/FORMAT field routing (var_key+dense, multi-contig, union)" +``` + +### Task 3.2: Full suite + docs + +**Files:** +- Modify: `skills/genvarloader/SKILL.md` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Run the full svar2 + variants suite** + +```bash +export CARGO_TARGET_DIR=/tmp/gvl-target +pixi run maturin develop 2>&1 | tail -5 +pixi run pytest tests/dataset -k svar2 -q 2>&1 | tail -30 +``` + +Expected: all PASS. + +- [ ] **Step 2: Update SKILL.md** + +Document that svar2 `variants`/variant-windows outputs now honor arbitrary store INFO/FORMAT fields via `var_fields`, that available fields come from the store's manifest (`available_var_fields`), and the dtype/missing-value semantics (stored dtype preserved; missing = default/sentinel/NaN). + +- [ ] **Step 3: Add CHANGELOG entry** + +Under `## Unreleased`: + +```markdown +### Added +- SVAR2 datasets now route arbitrary scalar-numeric INFO/FORMAT fields (stored via + `genoray.SparseVar2.from_vcf(info_fields=, format_fields=)`) into `variants` + (`RaggedVariants`) and variant-windows outputs, selectable via `var_fields`. +``` + +- [ ] **Step 4: Commit** + +```bash +git add skills/genvarloader/SKILL.md CHANGELOG.md +git commit -m "docs(svar2): document INFO/FORMAT field routing in variants outputs" +``` + +- [ ] **Step 5: Push + update the draft PR** + +```bash +git push origin svar2-m6b-kernel +``` + +The draft PR #266 updates automatically. Note in the PR description that the genoray Python pin is a local dev wheel pending a genoray svar-2 release (the version constraint was dropped for development). + +--- + +## Self-Review + +**Spec coverage:** +- Rust seam (provenance + FieldView) → Tasks 1.2, 1.3. ✓ +- gvl→genoray-main migration (discovered; not in spec but a hard prerequisite) → Phase 0. ✓ (larger than the spec's "prerequisite" section anticipated — see handoff.) +- Field discovery (`available_fields`) → Task 2.1. ✓ +- `RaggedVariants` routing → Task 2.3; variant-windows → Task 2.4. ✓ +- Dense-FORMAT `orig_samples[q]` stride subtlety → Task 1.2 Step 3 (`row * cohort_n_samples + orig_samples[q]`). ✓ +- `unphased_union` / multi-contig need no special handling → Tasks 2.3/2.4 use the same `src`/`var_off` machinery. ✓ (tested in 3.1 with `union` param.) +- Provenance-on-cursor invariant + identity test → Task 1.2 Steps 1–4. ✓ +- Testing (var_key/dense/multi-contig/union/missing) → Task 3.1. ✓ +- Docs (SKILL.md, CHANGELOG) → Task 3.2. ✓ +- `_impl.py` SVAR1-lazy-load hazard (discovered) → Task 2.2. ✓ + +**Placeholder scan:** Phase 0 Task 0.3 Step 4 is intentionally compiler-driven ("fix each reported site as in Steps 1–3") — legitimate for a mechanical type migration, with the known edits shown concretely. All feature code steps show full code. + +**Type consistency:** `decode_variants_from_split` new signature (Task 1.2 Interfaces) matches its call in Task 1.3 Step 2 and the no-field callers updated in Task 1.2 Step 3. FFI return tuple (Task 1.3) matches the Python unpack in Tasks 2.3/2.4. `store_fields` value shape `(category, name, dtype_str, np_dtype)` set in Task 2.1 matches use in 2.3/2.4. `FieldSub::all()` order asserted consistently (VkSnp,VkIndel,DenseSnp,DenseIndel) in Task 1.2 and 1.3. + +## Known risks / verify-during-execution + +- **`StorageDtype::from_meta_str` / `FieldView::open` / `FieldSub::all` exact signatures** — verify against genoray `src/field.rs`, `src/query/field.rs`, `src/layout.rs` at build time; the `Result` vs `Option` shape may need a `.map_err`/`.ok_or_else` tweak (Task 1.3 Step 1 notes this). +- **`ContigReader` has no `paths()` accessor** — we rebuild `ContigPaths::new(store.store_path(), contig)` (Task 1.1 + 1.3). Confirm `ContigPaths::new(base_out_dir, chrom)` is the correct constructor and that its `field_values(cat,name,sub)` layout matches what the writer produced. +- **`unphased_union` API name** — Task 3.1 uses `with_settings(unphased_union=True)`; confirm the actual gvl setter. +- **`DummyVariant` construction site for svar2 windows** — Task 2.4 Step 3 must locate where (if anywhere) `self.dummy_variant` is set for `Svar2Haps`; if unset, construct one carrying per-field fills. diff --git a/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md b/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md new file mode 100644 index 00000000..789dfe90 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-svar2-m6b-kernel-final-pass.md @@ -0,0 +1,1117 @@ +# SVAR2 M6b Branch — Final Pre-Merge Pass Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finish the SVAR2 read-bound branch — correctness/safety fixes, shipping hygiene, Rust de-duplication, and doc consistency — so it is mergeable modulo the (documented, unresolved) genoray release gate. + +**Architecture:** Additive-only cleanup of an already-green branch. Every change is a correctness fix, a hygiene cleanup, or a doc correction. The SVAR1 path stays byte-unchanged; the SVAR2 parity gate stays 31/31. No new features, no change to the parity contract. + +**Tech Stack:** Python 3.10 (numpy, polars, pyrefly, ruff), Rust (PyO3 abi3-py310, ndarray, rayon), pixi task runner, pytest + cargo test, prek pre-commit hooks. + +**Spec:** `docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md` + +## Global Constraints + +- **SVAR1 byte-unchanged:** every change is additive w.r.t. the SVAR1 path; no SVAR1 output bytes may change. +- **SVAR2 parity gate = 31/31:** `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py` stays fully green after every task that touches Rust or `_dataset/` code. +- **Rebuild Rust before Python tests that import the extension:** `pixi run -e dev maturin develop --release` after ANY `src/` edit, before running pytest. `cargo test` compiles from source and does not need this. +- **`cargo test` needs libpython on the loader path:** it runs under `pixi run -e dev` which sets `LD_LIBRARY_PATH=$CONDA_PREFIX/lib` via `pixi.toml` `[target.linux-64.activation.env]`. Always invoke cargo via `pixi run -e dev cargo ...`. +- **Lint/format gates:** `pixi run -e dev ruff check python/ tests/` and `pixi run -e dev ruff format --check python/ tests/` must pass. +- **Pre-commit hooks are broken in this worktree** (the `pyrefly-check` hook matches zero files under `.claude/` yet exits nonzero; the `pixi-lock` hook churns `pixi.lock`). Task 2 fixes the pyrefly invocation. Until Task 2 lands, commit doc/Rust-only changes with `git commit --no-verify` and NEVER stage `pixi.lock` churn (run `git checkout pixi.lock` if it appears dirty from an unrelated pixi operation). After Task 2, prefer hooks on; if `pixi-lock` still churns, `git checkout pixi.lock` before committing. +- **Commit style:** conventional commits (`fix(svar2):`, `refactor(svar2):`, `docs(svar2):`, `chore(svar2):`). End every commit message with `Co-Authored-By: Claude Opus 4.8 (1M context) `. +- **Do NOT touch the genoray path-pins** (`Cargo.toml` path-deps, `pixi.toml` wheel path, `pyproject.toml` unpinned genoray). They are the release gate — Task 12 documents them; nobody resolves them here. +- **Do NOT touch pre-existing clippy warnings** outside the SVAR2 changes (`bigwig.rs`, `reference/mod.rs`, etc.). + +--- + +## Execution order + +Docs & clippy (Tasks 1, 11) carry zero runtime risk — but they are grouped by topic below, not strictly first. The safe order is: **2 → 1 → 3 → 4 → 5 → 6 → 7 → 8 → 9 → 10 → 11 → 12 → 13**. Task 2 (typecheck-task fix) goes first so later Python tasks get real type-checking. The Rust de-dup (Tasks 9, 10) go late and each rebuilds + re-runs parity. Task 13 is the final full-suite gate. + +--- + +### Task 1: Strip stale/internal references from shipped comments & docstrings + +Pure text edits in shipped code. No behavior change. Groups all "stale comment" findings (spec §1b, §4d, §5-doc-nits) that are not tied to a code change in another task. + +**Files:** +- Modify: `src/svar2/mod.rs:800` (test docstring), `src/reconstruct/mod.rs:19-37`, `src/reconstruct/mod.rs:278` +- Modify: `python/genvarloader/_dataset/_svar2_haps.py:22-24` (module docstring), `:384`, `:486`, `:489` +- Modify: `python/genvarloader/_dataset/_reconstruct.py:143`, `:399` +- Modify: `python/genvarloader/_dataset/_write.py:1194-1197` + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing (comments only). + +- [ ] **Step 1: Fix the reverted-optimization test docstring** in `src/svar2/mod.rs:800`. The doc for `test_decode_variants_from_split_byte_identical_presence_edge` claims it exercises "the `present_bit` closure's now-`get_unchecked` read of `dense_present`". `get_unchecked` was reverted (grep `get_unchecked src/` → this comment is the only hit). Reword to describe the checked read that is actually there: + +Replace: +```rust + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's now-`get_unchecked` read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte +``` +with: +```rust + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte +``` + +- [ ] **Step 2: Fix the stale `unphased_union` out-of-scope claim** in `python/genvarloader/_dataset/_svar2_haps.py:22-24`. The module docstring lists `unphased_union` as guarded `NotImplementedError`, but it is fully supported (`_reconstruct_variants`, `_reconstruct_variant_windows`, and `_guard_unsupported` explicitly does NOT guard it). + +Replace: +```python +Out of scope for this plan (guarded with ``NotImplementedError``): spliced +output, ``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated +haps, in-kernel reverse-complement, and ``unphased_union``. +``` +with: +```python +Out of scope (guarded with ``NotImplementedError``): spliced output, +``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated haps, and +in-kernel reverse-complement. (``unphased_union`` and ``variant-windows`` ARE +supported.) +``` + +- [ ] **Step 3: Strip internal plan/task numbering** from shipped comments. Rewrite each in terms of behavior (reader has no access to plan docs): + - `_svar2_haps.py:384` — "The tracks follow-up (7c)" → describe the behavior (e.g. "Track re-alignment path"). + - `_svar2_haps.py:486`, `:489` — "tracks (7c)" banner → "tracks". + - `_reconstruct.py:143` — "Task 7c" → behavior description. + - `_reconstruct.py:399` — "FIX 1 guard" → describe what it guards (see Task 8, which also references this line; keep the wording consistent — describe the FlankSample fill-seed divergence). + - `_write.py:1194-1197` — "Phase-1 wiring" comment (see Task 5, which rewrites this block; if Task 5 runs first this may already be gone — if so, skip). Reword to describe that write-time fixed-length handling is unsupported for `.svar2` and the read kernel sizes output at read time. + - `src/reconstruct/mod.rs:278` and the list at `:19-37` — leave the prose but note these get clippy-reflowed in Task 11; no wording change needed here unless a task number appears. + +- [ ] **Step 3b: Add missing `///` docstrings to PyO3-exposed `svar2/store.rs` items** (spec §4e — these become the Python docstrings): add a one-line `///` to `reader` (`:16`), `store_path` (`:19`), the `#[new]` constructor (`:26`), and `contigs` (`:46`) describing what each returns. Also add numpydoc docstrings to `_svar2_link.py:make_svar2_link` and `_svar2_haps.py:_reconstruct_variants` if not added by their own tasks (`_write_from_svar2` is covered by Task 5). + +- [ ] **Step 4: Verify no plan/task references remain** in shipped code: + +Run: +```bash +grep -rnE "Task [0-9]|Phase-1 wiring|FIX 1|\(7c\)|first cut minimal|get_unchecked" src/ python/genvarloader/ | grep -v "test_" || echo "clean" +``` +Expected: `clean` (or only hits inside test names/legit uses you've reviewed). Note: `svar2/mod.rs:280` ("Task 1.3") and `tracks/mod.rs:2467` ("Task 4 Part C") and `ffi/mod.rs:774` ("first cut minimal") are ALSO in scope — fix them in this step too if the grep surfaces them. + +- [ ] **Step 5: Rebuild + smoke-test** (comments in Rust don't change behavior, but confirm it still compiles): + +Run: `pixi run -e dev cargo build 2>&1 | tail -3` +Expected: `Finished` (no errors). + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add -A src/ python/genvarloader/ +git commit --no-verify -m "$(cat <<'EOF' +docs(svar2): strip stale/internal references from shipped comments + +Fix the reverted-get_unchecked test docstring, the stale unphased_union +out-of-scope claim in the Svar2Haps module docstring, and internal plan/task +numbering ("Task 7c", "Phase-1 wiring", "FIX 1") that leaked into comments. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 2: Fix the `typecheck` pixi task (checks zero files in worktrees) + +`pixi run -e dev typecheck` = `pyrefly check` with no paths → inside any `.claude/worktrees/` checkout pyrefly matches **zero** files (root `.gitignore` ignores `.claude/`, pyrefly honors ignore files) and exits 0. Typecheck has effectively never run on this branch. This also fixes the broken `pyrefly-check` pre-commit hook. + +**Files:** +- Modify: `pixi.toml:162` +- Modify: `python/genvarloader/_ragged.py:325` (clear the one real finding the fixed task surfaces) + +**Interfaces:** +- Consumes: nothing. +- Produces: a working `typecheck` task used by every later Python task. + +- [ ] **Step 1: Confirm the bug** — pyrefly with no paths checks nothing in the worktree: + +Run: `pixi run -e dev pyrefly check 2>&1 | grep -c "No Python files matched"` +Expected: `1` (confirms zero files matched). + +- [ ] **Step 2: Point the task at explicit paths** in `pixi.toml:162`. + +Replace: +```toml +typecheck = { cmd = "pyrefly check" } +``` +with: +```toml +typecheck = { cmd = "pyrefly check python/genvarloader tests" } +``` + +- [ ] **Step 3: Run the fixed task; expect exactly one real finding** + +Run: `pixi run -e dev typecheck 2>&1 | grep -E "error|Unused" | head` +Expected: one `ERROR Unused \`# pyrefly: ignore\` comment ... no-matching-overload [unused-ignore]` at `_ragged.py:325`. + +- [ ] **Step 4: Clear the unused-ignore** at `python/genvarloader/_ragged.py:325`. + +Replace: +```python + out = out.reshape((*leading, out_len)) # pyrefly: ignore[no-matching-overload] +``` +with: +```python + out = out.reshape((*leading, out_len)) +``` + +- [ ] **Step 5: Re-run typecheck; expect clean** + +Run: `pixi run -e dev typecheck 2>&1 | tail -3` +Expected: `0 errors` (warnings are fine; no errors). + +- [ ] **Step 6: Confirm the pre-commit hook now passes** (also update the local hook definition if it hardcodes `pyrefly check` with no args): + +Run: `grep -n -A6 "pyrefly-check" .pre-commit-config.yaml` +If the hook `entry` is `pyrefly check` with no paths, change it to `pyrefly check python/genvarloader tests` to match the task. Then: +Run: `pixi run -e dev prek run pyrefly-check --all-files 2>&1 | tail -3` +Expected: `Passed`. + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add pixi.toml python/genvarloader/_ragged.py .pre-commit-config.yaml +git commit -m "$(cat <<'EOF' +chore(svar2): make typecheck task check explicit paths + +Bare `pyrefly check` matches zero files inside a .claude/worktrees checkout +(root .gitignore ignores .claude/, pyrefly honors ignore files), so typecheck +silently passed on nothing. Point it at python/genvarloader + tests, fix the +pre-commit hook to match, and clear the now-flagged unused pyrefly-ignore. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 3: Guard the serial unsafe raw-pointer path in `reconstruct/mod.rs` + +The parallel carve path has `debug_assert!(s >= cursor && e >= s, ...)`; the two **serial** fallback loops (`:547` and `:847`) build `&mut [u8]` from caller-supplied `out_offsets` with only a SAFETY comment, no runtime guard. A non-monotonic `out_offsets` underflows `out_e - out_s` to a giant slice (UB). Add the matching assert. + +**Files:** +- Modify: `src/reconstruct/mod.rs` (serial loop bodies near `:547` and `:847`) + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing (adds a debug-only invariant check). + +- [ ] **Step 1: Add the guard to the first serial loop.** In the serial `for k in 0..n_work` block (the one preceded by `// Serial path: use raw pointers ...`), immediately after: +```rust + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; +``` +insert: +```rust + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); +``` + +- [ ] **Step 2: Add the identical guard to the second serial loop** (the second occurrence of the same `let out_s = ...; let out_e = ...;` pair inside a serial raw-pointer block, near `:847`). Insert the same `debug_assert!` block after the two `let` lines. + +- [ ] **Step 3: Verify both serial loops now assert.** There are two carve dispatchers (merged later in Task 9); both must have the guard on the serial branch. + +Run: +```bash +grep -c "out_offsets must be monotonically non-decreasing" src/reconstruct/mod.rs +``` +Expected: `4` (2 pre-existing parallel-path asserts + 2 new serial-path asserts). + +- [ ] **Step 4: Build + run the Rust unit tests** (compiles the debug_assert; a debug `cargo test` build will trip the assert if any existing test passes non-monotonic offsets — it shouldn't): + +Run: `pixi run -e dev cargo test reconstruct 2>&1 | tail -8` +Expected: tests pass (no assertion panic). + +- [ ] **Step 5: Rebuild release + run SVAR2 parity gate:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_tracks.py -q 2>&1 | tail -3 +``` +Expected: build `Installed`; tests pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/reconstruct/mod.rs +git commit --no-verify -m "$(cat <<'EOF' +fix(svar2): guard serial unsafe carve path with monotonicity debug_assert + +The parallel split_at_mut path already debug_asserts out_offsets is +non-decreasing; the serial raw-pointer fallback carved out_e - out_s with no +guard, so a non-monotonic offsets array underflows to a multi-GB slice (UB). +Hoist the same assert into both serial loops. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 4: Convert Python-reachable Rust panics to `PyValueError` + +Arrays from Python are `.as_slice().unwrap()` / `.expect("must be contiguous")`'d in the SVAR2 kernels; a non-contiguous view (`a[::2]`) panics instead of raising. Plus two more: the pure-DEL anchor index and a release-mode `assert_eq!`. + +**Files:** +- Modify: `src/ffi/mod.rs` (the SVAR2 readbound `#[pyfunction]`s — contiguity validation at the boundary) +- Modify: `src/reconstruct/mod.rs:703` (pure-DEL anchor bounds check), `:639-645`, `:692` +- Modify: `src/svar2/mod.rs:358-365` (move the `assert_eq!` to the FFI boundary at `ffi/mod.rs:1411`) +- Test: `tests/dataset/test_svar2_readbound_haps.py` (add a non-contiguous-input test) + +**Interfaces:** +- Consumes: nothing. +- Produces: SVAR2 readbound `#[pyfunction]`s raise `ValueError` (not panic) on non-C-contiguous inputs and on out-of-range variant positions. + +- [ ] **Step 1: Write the failing test** — a non-contiguous input should raise `ValueError`, not crash the interpreter. Add to `tests/dataset/test_svar2_readbound_haps.py`: + +```python +def test_readbound_haps_noncontiguous_input_raises(): + """A non-C-contiguous numpy view must surface as ValueError, not a panic.""" + import numpy as np + import pytest + from genvarloader._dataset._svar2_store_py import build_readbound_haps # noqa: F401 + # Build a minimal store + regions exactly as the existing haps parity test does, + # then pass a strided (non-contiguous) view of one of the int64 range arrays. + # (Reuse the fixture/store construction from test_readbound_haps_* above.) + # The precise construction mirrors the sibling test; the assertion is: + with pytest.raises(ValueError): + # call the FFI entry with a[::2] slice of a range array + ... +``` +NOTE to implementer: model the store/region setup on the nearest existing test in this file (e.g. the first `build_readbound_haps` parity test around `:75`). The ONLY new thing is passing a `arr[::2]` view where a contiguous int64 array is expected and asserting `ValueError`. + +- [ ] **Step 2: Run it — expect it to FAIL** (currently panics / wrong exception): + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_readbound_haps_noncontiguous_input_raises -v 2>&1 | tail -8` +Expected: FAIL (panic/pyo3 PanicException or a bare index error, not ValueError). + +- [ ] **Step 3: Add a contiguity helper + validate at each SVAR2 `#[pyfunction]` boundary** in `src/ffi/mod.rs`. Near the top of the SVAR2 FFI section add: +```rust +/// Return a C-contiguous slice view of `arr`, or a Python `ValueError` if the +/// input array is not contiguous (e.g. a strided `a[::2]` view). Kernels below +/// index the backing slice directly, so a non-contiguous input would otherwise +/// panic inside `py.detach`. +fn require_contiguous<'a, T: numpy::Element>( + arr: &'a numpy::PyReadonlyArray1, + name: &str, +) -> PyResult<&'a [T]> { + arr.as_slice() + .map_err(|_| PyValueError::new_err(format!("`{name}` must be C-contiguous"))) +} +``` +Then at each of the four SVAR2 readbound `#[pyfunction]`s (`reconstruct_haplotypes_from_svar2_readbound`, `shift_and_realign_tracks_from_svar2_readbound`, `decode_variants_from_svar2_readbound`, `hap_diffs_from_svar2_readbound`), replace the `.as_slice().unwrap()` / `.expect("must be contiguous")` calls on Python-supplied arrays with `require_contiguous(&arr, "arr")?`. (For `PyReadwriteArray` outputs use the analogous `as_slice_mut().map_err(...)`.) + +- [ ] **Step 4: Bounds-check the pure-DEL anchor** at `src/reconstruct/mod.rs:703`. The `contig_ref_s[pos..pos+1]` index panics for a variant at/past contig end. Guard it — return a `Result`/`PyErr` up the call chain, OR (if this fn is not `PyResult`-returning) validate `pos < contig_ref_s.len()` at the FFI boundary before entering `py.detach` and raise `PyValueError::new_err(format!("variant position {pos} is beyond contig end"))`. Prefer the FFI-boundary check to keep the hot kernel panic-free. + +- [ ] **Step 5: Move the release-mode `assert_eq!`** from `src/svar2/mod.rs:358-365` to the FFI boundary. In `decode_variants_from_svar2_readbound` (`ffi/mod.rs:~1411`, where `has_fields` is known), validate the `vk_src` length precondition and raise `PyValueError` before calling into `svar2::`. Downgrade the in-kernel `assert_eq!` to `debug_assert_eq!` (or delete it if the FFI check fully covers it). + +- [ ] **Step 6: Rebuild + run the new test — expect PASS:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_readbound_haps.py::test_readbound_haps_noncontiguous_input_raises -v 2>&1 | tail -5 +``` +Expected: PASS (`ValueError` raised). + +- [ ] **Step 7: Full SVAR2 parity gate still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: 31/31 (+ the new test) pass. + +- [ ] **Step 8: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs src/reconstruct/mod.rs src/svar2/mod.rs tests/dataset/test_svar2_readbound_haps.py +git commit --no-verify -m "$(cat <<'EOF' +fix(svar2): raise PyValueError on non-contiguous / OOB Python input + +Non-C-contiguous numpy views (a[::2]) and out-of-range variant positions +panicked inside the readbound kernels instead of surfacing as ValueError. +Validate contiguity and the pure-DEL anchor bound at the #[pyfunction] +boundary; move the vk_src length assert_eq! there too. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 5: Reject `extend_to_length=False` for `.svar2` (stop silently ignoring it) + +`_write_from_svar2` accepts `extend_to_length` and never reads it; `chromEnd` is always extended. Passing `False` yields a different dataset than requested, silently. Per the branch's guard-matrix policy, fail loudly. + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1140-1197` (`_write_from_svar2`) +- Test: `tests/dataset/test_write_svar2.py` + +**Interfaces:** +- Consumes: `_write_from_svar2(path, bed, svar2, samples, extend_to_length)` (existing signature — unchanged). +- Produces: `gvl.write(..., variants=, extend_to_length=False)` raises `NotImplementedError`. + +- [ ] **Step 1: Write the failing test** in `tests/dataset/test_write_svar2.py`: + +```python +def test_svar2_extend_to_length_false_raises(tmp_path, svar2_source_and_bed): + """extend_to_length=False is unsupported for a .svar2 source (Phase-1) and + must raise, not silently produce an extended dataset.""" + import pytest + import genvarloader as gvl + svar2, bed = svar2_source_and_bed # reuse the existing fixture used by the write tests + with pytest.raises(NotImplementedError, match="extend_to_length"): + gvl.write(tmp_path / "ds", bed, variants=svar2, extend_to_length=False) +``` +NOTE: reuse whatever fixture the existing `test_write_svar2.py` tests use to obtain a `SparseVar2` + bed; match its parameter name. + +- [ ] **Step 2: Run it — expect FAIL** (currently silently succeeds): + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_extend_to_length_false_raises -v 2>&1 | tail -6` +Expected: FAIL (no exception raised). + +- [ ] **Step 3: Add the guard** at the top of `_write_from_svar2` in `python/genvarloader/_dataset/_write.py` (right after the signature/opening comment, before `out_dir = ...`): + +```python + if not extend_to_length: + raise NotImplementedError( + "extend_to_length=False is not supported for a .svar2 variant source: " + "the read-bound kernel always sizes haplotype output at read time and " + "the write-time ranges cache is built for the extended chromEnd. Use a " + ".svar/VCF/PGEN source if you need un-extended haplotypes." + ) +``` +Then replace the now-inaccurate `# extend_to_length fixed-output-length write-time handling is out of scope ...` comment block (around `:1194-1197`) with a one-liner noting the flag is validated at entry. + +- [ ] **Step 4: Run the new test — expect PASS:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_extend_to_length_false_raises -v 2>&1 | tail -4` +Expected: PASS. + +- [ ] **Step 5: Full write-svar2 suite still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "$(cat <<'EOF' +fix(svar2): reject extend_to_length=False for .svar2 sources + +_write_from_svar2 accepted the flag and ignored it, silently extending +chromEnd regardless. Raise NotImplementedError (Phase-1 guard-matrix policy) +instead of producing a dataset that differs from what was requested. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 6: Vectorize `_svar2_region_max_ends` + +`_write.py:1092-1138` is an `O(regions × samples × ploidy)` Python triple-loop over decoded records at write time. Its own docstring flags it. Vectorize with a scatter-reduce while staying byte-identical (`test_write_svar2.py` locks the cache + same-POS-tie behavior). + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1092-1138` (`_svar2_region_max_ends`) +- Test: `tests/dataset/test_write_svar2.py` (add a direct equivalence test against the old loop) + +**Interfaces:** +- Consumes: `_svar2_region_max_ends(svar2, contig, starts, ends, samples) -> NDArray[np.int32]` (signature unchanged). +- Produces: identical output to the current loop (per-region max over selected haplotypes of `(pos, end)`, `pos`-then-`end` tie-break, default = `ends`). + +- [ ] **Step 1: Pin the current behavior with a direct test.** Before changing the function, add a test that captures the exact semantics — including the same-POS tie-break and the "no variants → keep chromEnd" default — on a hand-built decode. Add to `tests/dataset/test_write_svar2.py`: + +```python +def test_svar2_region_max_ends_matches_reference(svar2_source_and_bed): + """Vectorized _svar2_region_max_ends must equal a straightforward per-hap loop, + including the pos-then-end tie-break and the empty-region default = chromEnd.""" + import numpy as np + from genvarloader._dataset._write import _svar2_region_max_ends + svar2, bed = svar2_source_and_bed + for (c,), df in bed.partition_by("chrom", as_dict=True, maintain_order=True).items(): + starts = df["chromStart"].to_numpy() + ends = df["chromEnd"].to_numpy() + samples = list(svar2.available_samples) + got = _svar2_region_max_ends(svar2, c, starts, ends, samples) + # reference: recompute with the explicit loop semantics + ref = _reference_region_max_ends(svar2, c, starts, ends, samples) + np.testing.assert_array_equal(got, ref) + + +def _reference_region_max_ends(svar2, contig, starts, ends, samples): + """Byte-for-byte copy of the ORIGINAL triple-loop, kept in the test as the oracle.""" + import numpy as np + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = [svar2.available_samples.index(s) for s in samples] + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = dec.data["pos"]; ilen_arr = dec.data["ilen"]; off = np.asarray(dec.offsets) + out = np.asarray(ends, np.int64).copy() + for r in range(R): + best_pos, best_end = -1, -1 + for s in sel: + for p in range(P): + h = (r * S_all + s) * P + p + a, b = int(off[h]), int(off[h + 1]) + if a == b: continue + seg_pos = pos_arr[a:b]; seg_ilen = ilen_arr[a:b] + j = int(np.argmax(seg_pos)) + p_pos = int(seg_pos[j]); p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) + if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): + best_pos, best_end = p_pos, p_end + if best_pos >= 0: + out[r] = best_end + return out.astype(np.int32) +``` + +- [ ] **Step 2: Run it — expect PASS** (the reference IS the current impl, so it passes now; this locks the contract before the rewrite): + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_region_max_ends_matches_reference -v 2>&1 | tail -4` +Expected: PASS. + +- [ ] **Step 3: Vectorize the function.** Replace the triple-loop body of `_svar2_region_max_ends` (keeping the docstring's semantics but dropping the "O(...) Python iteration ... vectorize as a follow-up" caveat) with a scatter-reduce. Key idea: for each variant, its haplotype maps to a region `r = h // (S_all * P)` but only SELECTED samples count; compute `end = (pos+1) - min(ilen,0)` per variant, form a sortable composite `key = (pos << 21) | end` (end fits well under 2^21 for realistic regions; assert it) so that a plain per-region max on `key` reproduces the pos-then-end tie-break, then unpack `end`: + +```python + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = np.asarray([svar2.available_samples.index(s) for s in samples], np.int64) + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = np.asarray(dec.data["pos"], np.int64) + ilen_arr = np.asarray(dec.data["ilen"], np.int64) + off = np.asarray(dec.offsets, np.int64) # length R*S_all*P + 1 + out = np.asarray(ends, np.int64).copy() # default = chromEnd + if pos_arr.size: + n_hap = R * S_all * P + counts = np.diff(off) # variants per hap + hap_of_var = np.repeat(np.arange(n_hap), counts) # region-major hap index per variant + s_of_hap = (np.arange(n_hap) // P) % S_all + keep = np.isin(s_of_hap[hap_of_var], sel) # only selected samples + region_of_var = hap_of_var // (S_all * P) + end_var = (pos_arr + 1) - np.minimum(ilen_arr, 0) # 0-based -> 1-based, extend on DEL + SHIFT = 21 + assert int(end_var.max(initial=0)) < (1 << SHIFT), "end exceeds tie-break packing width" + key = (pos_arr << SHIFT) | end_var + key_k = key[keep]; region_k = region_of_var[keep] + if key_k.size: + best = np.full(R, -1, np.int64) + np.maximum.at(best, region_k, key_k) # per-region max composite key + has = best >= 0 + out[has] = best[has] & ((1 << SHIFT) - 1) # unpack end + return out.astype(np.int32) +``` +Update the docstring: drop the last paragraph ("O(R * len(samples) * ploidy) Python iteration ... follow-up") and replace with a one-line note that it is a vectorized per-region scatter-max preserving the pos-then-end tie-break. + +- [ ] **Step 4: Run the equivalence test — expect PASS:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py::test_svar2_region_max_ends_matches_reference -v 2>&1 | tail -4` +Expected: PASS (vectorized == reference loop). + +- [ ] **Step 5: Full write-svar2 suite (locks cache + same-POS tie) still green:** + +Run: `pixi run -e dev pytest tests/dataset/test_write_svar2.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py tests/dataset/test_write_svar2.py +git commit -m "$(cat <<'EOF' +perf(svar2): vectorize _svar2_region_max_ends (byte-identical) + +Replace the O(regions x samples x ploidy) write-time Python triple-loop with a +per-region scatter-max over a (pos<<21)|end composite key that preserves the +pos-then-end tie-break. Pinned byte-identical to the original loop by a new +equivalence test carrying the old loop as its oracle. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 7: Drop the unused `region_starts` array + +`_write.py` writes/memmaps `region_starts`; `_svar2_haps.py` loads it and its own docstring says it is "kept for parity/debug, NOT fed to the FFI." Remove it end-to-end. + +**Files:** +- Modify: `python/genvarloader/_dataset/_write.py:1162,1205,1214` (+ the `svar2_meta.json` entry at `:1170-ish`) +- Modify: `python/genvarloader/_dataset/_svar2_haps.py:86-88,95` (loader + docstring) +- Modify: `tests/unit/dataset/test_svar2_store.py` and/or `tests/dataset/test_write_svar2.py` if either asserts `region_starts` presence + +**Interfaces:** +- Consumes: nothing. +- Produces: `svar2_ranges/` cache no longer contains `region_starts.npy`; `svar2_meta.json` no longer lists it. + +- [ ] **Step 1: Check whether any test asserts `region_starts`:** + +Run: `grep -rn "region_starts" tests/ python/genvarloader/ | grep -v "\.pyc"` +Expected: hits in `_write.py`, `_svar2_haps.py`, and possibly a test. Note every location. + +- [ ] **Step 2: Remove the write side** in `_write.py`: + - Delete the `region_starts = np.memmap(... "region_starts.npy" ...)` line (`:1162`). + - Delete the `"region_starts": {"shape": [R], "dtype": "&1 | tail -3` +Expected: all pass. + +- [ ] **Step 6: Full SVAR2 read parity (dataset opens the trimmed cache):** + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q 2>&1 | tail -3` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_write.py python/genvarloader/_dataset/_svar2_haps.py tests/ +git commit -m "$(cat <<'EOF' +refactor(svar2): drop unused region_starts from the ranges cache + +region_starts was written, memmapped, and never fed to the FFI (its own +docstring said "parity/debug only"). Remove it from the write path, the +svar2_meta.json schema, the Svar2Haps loader, and the cache-contents test. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 8: Open a tracked issue for the FlankSample fill-seed divergence + +`_reconstruct.py:399-419` papers over a documented correctness divergence (multi-contig `FlankSample` track fills seed off a contig-local query index) with a `NotImplementedError`. The comment is the only record. File an issue and reference it. + +**Files:** +- Modify: `python/genvarloader/_dataset/_reconstruct.py:399-419` (add issue reference to the guard comment) + +**Interfaces:** +- Consumes: nothing. +- Produces: a GitHub issue URL referenced in the guard comment. + +- [ ] **Step 1: Read the guard + its comment** to write an accurate issue body: + +Run: `sed -n '395,420p' python/genvarloader/_dataset/_reconstruct.py` + +- [ ] **Step 2: Create the issue** (repo `mcvickerlab/GenVarLoader`): + +Run: +```bash +gh issue create --repo mcvickerlab/GenVarLoader \ + --title "SVAR2 read-bound: multi-contig FlankSample track fill uses contig-local query index" \ + --body "In the SVAR2 read-bound path, FlankSample track fills seed the fill value off a contig-local query index rather than the global one, which diverges across a multi-contig batch. Currently guarded with NotImplementedError in \`_reconstruct.py\` (~line 399). This issue tracks lifting the guard once the fill-seed index is made global. Branch: svar2-m6b-kernel." +``` +Capture the printed issue URL. + +- [ ] **Step 3: Reference the issue in the guard comment.** Replace the "FIX 1 guard" wording (also targeted by Task 1) with a behavior description ending in the issue link, e.g.: +```python + # Multi-contig FlankSample track fills seed off a contig-local query + # index, which diverges from the global fill seed across a batch. + # Guarded until the fill-seed index is made global; see + # https://github.com/mcvickerlab/GenVarLoader/issues/. +``` + +- [ ] **Step 4: Verify the guard still fires** (behavior unchanged — this is a comment + issue only): + +Run: `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q -k "flank or Flank or multi" 2>&1 | tail -4` +Expected: pass (guard raises where tested; no behavior change). + +- [ ] **Step 5: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add python/genvarloader/_dataset/_reconstruct.py +git commit -m "$(cat <<'EOF' +docs(svar2): track FlankSample fill-seed divergence in a GitHub issue + +The multi-contig FlankSample track-fill divergence was recorded only in a code +comment behind a NotImplementedError guard. File a tracking issue and +reference it from the guard. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 9: Relocate test-only oracle modules out of the shipped package + +`_svar2_source.py` (`SparseVar2Source`) and `_svar2_store_py.py` (`build_readbound_*`) have zero importers under `python/` — only 8 test files use them. Move them into `tests/` and rename `_svar2_store_py.py` (the `_py` suffix is meaningless — it holds no store class). + +**Files:** +- Move: `python/genvarloader/_dataset/_svar2_source.py` → `tests/_oracles/svar2_source.py` +- Move: `python/genvarloader/_dataset/_svar2_store_py.py` → `tests/_oracles/svar2_readbound_inputs.py` +- Create: `tests/_oracles/__init__.py` +- Modify (imports): `tests/test_svar2_realign_tracks.py`, `tests/test_svar2_reconstruct.py`, `tests/dataset/test_svar2_dataset.py`, `tests/dataset/test_svar2_readbound_haps.py`, `tests/dataset/test_svar2_readbound_diffs.py`, `tests/dataset/test_svar2_readbound_tracks.py`, `tests/dataset/test_svar2_readbound_variants.py` +- Modify (docstring back-reference): `python/genvarloader/_dataset/_svar2_haps.py:15-19` + +**Interfaces:** +- Consumes: tests import `from tests._oracles.svar2_source import SparseVar2Source` and `from tests._oracles.svar2_readbound_inputs import build_readbound_haps, build_readbound_diffs, build_readbound_tracks, build_readbound_variants`. +- Produces: no test-only code ships in `genvarloader`. + +- [ ] **Step 1: Confirm the two modules are imported only from tests** (already audited, re-verify): + +Run: `grep -rln "_svar2_source\|_svar2_store_py\|SparseVar2Source\|build_readbound" python/genvarloader/ | grep -v "\.so"` +Expected: only `_svar2_haps.py` (docstring reference) and the two modules themselves — NO functional importer in `python/`. + +- [ ] **Step 2: Check the two modules' own imports** (what they pull from the package — must still resolve from `tests/`): + +Run: `grep -n "^import\|^from" python/genvarloader/_dataset/_svar2_source.py python/genvarloader/_dataset/_svar2_store_py.py` +Note any relative imports (`from . import ...` / `from ._x import ...`) — these must become absolute `genvarloader...` imports after the move. + +- [ ] **Step 3: Move the files with git + create the package:** + +```bash +mkdir -p tests/_oracles +git mv python/genvarloader/_dataset/_svar2_source.py tests/_oracles/svar2_source.py +git mv python/genvarloader/_dataset/_svar2_store_py.py tests/_oracles/svar2_readbound_inputs.py +touch tests/_oracles/__init__.py && git add tests/_oracles/__init__.py +``` + +- [ ] **Step 4: Fix relative imports** inside the two moved files (from Step 2). Any `from .` / `from ._foo import` becomes `from genvarloader._dataset._foo import`. If `svar2_readbound_inputs.py` imports `SparseVar2Source`, point it at `from tests._oracles.svar2_source import SparseVar2Source`. + +- [ ] **Step 5: Update the 8 test files' imports.** In each, replace: + - `from genvarloader._dataset._svar2_source import SparseVar2Source` → `from tests._oracles.svar2_source import SparseVar2Source` + - `from genvarloader._dataset._svar2_store_py import build_readbound_*` → `from tests._oracles.svar2_readbound_inputs import build_readbound_*` + +Do it mechanically: +```bash +grep -rl "_dataset._svar2_source\|_dataset._svar2_store_py" tests/ | while read f; do + sed -i 's#genvarloader\._dataset\._svar2_source#tests._oracles.svar2_source#g; s#genvarloader\._dataset\._svar2_store_py#tests._oracles.svar2_readbound_inputs#g' "$f" +done +``` +(RTK note: this is a mechanical sed on test imports — acceptable here.) + +- [ ] **Step 6: Update the `_svar2_haps.py` docstring back-reference** (`:15-19`) that mentions `_svar2_store_py.build_readbound_*`. Point it at the new location: replace `_svar2_store_py.build_readbound_*` with `tests/_oracles/svar2_readbound_inputs.build_readbound_*` (and note it is the test oracle). + +- [ ] **Step 7: Confirm `tests` is importable as a package** (so `from tests._oracles...` resolves). Check for `tests/__init__.py`: + +Run: `ls tests/__init__.py 2>/dev/null && echo "has __init__" || echo "NO __init__ — check conftest/rootdir import mode"` +If tests are collected in rootdir/`importmode=prepend` without `tests/__init__.py`, prefer a conftest-based path or add `tests/__init__.py`. Verify by running one moved-import test in Step 8; if `ModuleNotFoundError: tests`, add `tests/__init__.py` (`touch tests/__init__.py && git add tests/__init__.py`) or adjust to a top-level `_oracles` package under the tests rootdir — match the repo's existing test-helper import convention (`tests/_builders/` is imported as `from _builders...` or `from tests._builders...`? check it): +```bash +grep -rn "_builders" tests/ | grep import | head -3 +``` +Mirror whatever `_builders` does. + +- [ ] **Step 8: Run every affected test file:** + +Run: +```bash +pixi run -e dev pytest tests/test_svar2_realign_tracks.py tests/test_svar2_reconstruct.py tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_haps.py tests/dataset/test_svar2_readbound_diffs.py tests/dataset/test_svar2_readbound_tracks.py tests/dataset/test_svar2_readbound_variants.py -q 2>&1 | tail -4 +``` +Expected: all pass (no `ModuleNotFoundError`, no import errors). + +- [ ] **Step 9: Confirm nothing in the shipped package references the moved modules:** + +Run: `grep -rn "_svar2_source\|_svar2_store_py" python/genvarloader/ | grep -v "\.so"` +Expected: no hits (the docstring in `_svar2_haps.py` now points at `tests/_oracles/...`). + +- [ ] **Step 10: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add -A python/ tests/ +git commit -m "$(cat <<'EOF' +refactor(svar2): move test-only oracles out of the shipped package + +SparseVar2Source and build_readbound_* have no importers under python/ — they +are the parity oracle + FFI-input builders used only by tests. Move them to +tests/_oracles/ (renaming _svar2_store_py.py -> svar2_readbound_inputs.py, +since it holds no store class) and repoint the 8 test files' imports. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 10: Drop the dead `annot_*` capability from the SVAR2 readbound haplotype kernel + +`reconstruct_haplotypes_from_svar2_readbound`'s `annot_v_idxs`/`annot_ref_pos` params are `None` at both call sites; 3 of 4 match arms (~60 lines) + the `:605-608` doc are unreachable (annotated-hap output for `.svar2` is `NotImplementedError`-guarded anyway). + +**Files:** +- Modify: `src/ffi/mod.rs` (`reconstruct_haplotypes_from_svar2_readbound` signature + the two call sites `:889-890`, `:1045-1046`) +- Modify: `src/reconstruct/mod.rs` (the readbound reconstruct fn — drop the annot params + unreachable match arms + `:605-608` doc) + +**Interfaces:** +- Consumes: nothing. +- Produces: `reconstruct_haplotypes_from_svar2_readbound` no longer takes `annot_v_idxs`/`annot_ref_pos`; only the un-annotated output arm remains. + +NOTE: This is scoped to the SVAR2 **readbound** reconstruct fn ONLY. Do NOT remove the annot params from the general (SVAR1) reconstruct dispatchers merged in Task 11 — SVAR1 uses them. If Task 11 runs first, keep them independent. + +- [ ] **Step 1: Confirm both call sites pass `None`:** + +Run: `grep -n "annot_v_idxs\|annot_ref_pos" src/ffi/mod.rs` +Expected: at the two `reconstruct_haplotypes_from_svar2_readbound` call sites they are `None`/`None`. Confirm no SVAR2 readbound caller ever passes `Some`. + +- [ ] **Step 2: Remove the params from the Rust fn** in `src/reconstruct/mod.rs`: drop `annot_v_idxs`/`annot_ref_pos` from the readbound reconstruct fn signature, delete the 3 match arms that handle `Some(...)` (keep only the `(None, None)` / un-annotated arm), and delete the now-inaccurate doc at `:605-608`. + +- [ ] **Step 3: Remove the params at both FFI call sites** (`ffi/mod.rs:889-890`, `:1045-1046`) and from the `#[pyfunction]` signature if they were exposed to Python (check the `#[pyo3(signature = ...)]` and the Python-side call in `_svar2_haps.py` — if the Python code passes them, drop there too): + +Run: `grep -rn "reconstruct_haplotypes_from_svar2_readbound" python/genvarloader/` +If the Python call passes `annot_v_idxs=`/`annot_ref_pos=`, remove those kwargs. + +- [ ] **Step 4: Build — expect clean compile** (no unused-var / unreachable warnings for the removed arms): + +Run: `pixi run -e dev cargo build 2>&1 | tail -5` +Expected: `Finished`, no warnings about the removed code. + +- [ ] **Step 5: Rebuild release + full SVAR2 parity gate:** + +Run: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py -q 2>&1 | tail -3 +``` +Expected: build `Installed`; tests pass. + +- [ ] **Step 6: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs src/reconstruct/mod.rs python/genvarloader/_dataset/_svar2_haps.py +git commit --no-verify -m "$(cat <<'EOF' +refactor(svar2): drop dead annot_* capability from readbound haps kernel + +annot_v_idxs/annot_ref_pos were None at both call sites of +reconstruct_haplotypes_from_svar2_readbound; 3 of 4 match arms and their doc +were unreachable (annotated .svar2 haps are NotImplementedError-guarded). Drop +the params and the dead arms. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 11: Extract Rust duplication (carve dispatcher, FFI preamble, present_bit) + clear new clippy nits + +Highest-risk task; guarded by the full parity suite. Do each extraction as its own commit so a regression bisects cleanly. Rebuild + re-run parity after EACH extraction. + +**Files:** +- Modify: `src/reconstruct/mod.rs` (carve/dispatch helper; `present_bit` move; clippy `explicit_auto_deref` at `:248,251`, `doc_overindented_list_items` at `:19-37,278`) +- Modify: `src/ffi/mod.rs` (readbound preamble helper; `offsets_from_diffs` helper; `type` aliases for `:930,1182,1344`) +- Modify: `src/tracks/mod.rs` (use shared `present_bit`) +- Modify: `src/svar2/mod.rs` (host `present_bit`; test-only `single_range_in_vec_init` at `:593,600,772`) + +**Interfaces:** +- Consumes: nothing (internal Rust refactor). +- Produces: `svar2::present_bit`, `carve_chunks`, one carve dispatcher, one readbound-preamble helper, `offsets_from_diffs` — all internal. + +- [ ] **Step 1 (extraction 1 — carve dispatcher):** Extract `fn carve_chunks(buf: &mut [T], bounds: &[(usize, usize)]) -> Vec<&mut [T]>` and a single dispatcher generic over the per-chunk work closure, replacing the ~150 verbatim lines duplicated between `reconstruct/mod.rs:424-578` and `:724-880`. Keep BOTH the parallel (`split_at_mut`) and serial (raw-ptr) branches inside the helper, with the Task 3 `debug_assert!` on the serial branch. Output must be byte-identical. + +- [ ] **Step 2:** Build + full parity: +```bash +pixi run -e dev cargo build 2>&1 | tail -2 +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py tests/dataset/test_write_svar2.py -q 2>&1 | tail -3 +``` +Expected: 31/31 pass. Also `pixi run -e dev cargo test 2>&1 | tail -5` passes. + +- [ ] **Step 3: Commit extraction 1** +```bash +git checkout pixi.lock 2>/dev/null +git add src/reconstruct/mod.rs +git commit --no-verify -m "refactor(svar2): extract carve_chunks + shared serial/parallel dispatcher + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 4 (extraction 2 — FFI preamble):** Extract one helper returning `(FlatChannels, lut_bytes, lut_off, regions)` from the 4× readbound preamble (`ffi/mod.rs:934-998,1086-1144,1186-1250,1358-1432`), and `fn offsets_from_diffs(...)` from the 3× prefix-sum loop (`:846-864,1000-1019,1252-1268`). Add a `type` alias for the three readbound return tuples (clears `clippy::type_complexity` at `:930,1182,1344`). + +- [ ] **Step 5:** Build + full parity (same commands as Step 2). Expected: 31/31. + +- [ ] **Step 6: Commit extraction 2** +```bash +git checkout pixi.lock 2>/dev/null +git add src/ffi/mod.rs +git commit --no-verify -m "refactor(svar2): extract readbound FFI preamble + offsets_from_diffs helpers + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 7 (extraction 3 — present_bit):** Move the identical `present_bit` closure (`reconstruct/mod.rs:675-678` == `tracks/mod.rs:759-762`) to a documented `svar2::present_bit`; call it from both. Build + full parity. + +- [ ] **Step 8: Commit extraction 3** +```bash +git checkout pixi.lock 2>/dev/null +git add src/svar2/mod.rs src/reconstruct/mod.rs src/tracks/mod.rs +git commit --no-verify -m "refactor(svar2): hoist shared present_bit to svar2::present_bit + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +- [ ] **Step 9 (clippy nits):** Clear the NEW-code clippy warnings only: +```bash +pixi run -e dev cargo clippy --all-targets 2>&1 | tail -40 +``` +Fix: `reconstruct/mod.rs:248,251` `explicit_auto_deref` (drop `as_deref_mut()`), `doc_overindented_list_items` at `:19-37,278` (4→2 spaces), `single_range_in_vec_init`/`redundant_closure` in the svar2/reconstruct tests. Leave pre-existing warnings (`bigwig.rs`, `reference/mod.rs`, `intervals.rs`, etc.) untouched. + +- [ ] **Step 10: Confirm no new clippy warnings + parity green:** +```bash +pixi run -e dev cargo clippy --all-targets 2>&1 | grep -c "warning:" # compare against the pre-existing baseline count +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev pytest tests/dataset/test_svar2_dataset.py tests/dataset/test_svar2_readbound_*.py -q 2>&1 | tail -3 +``` +Expected: only pre-existing warnings remain; 31/31 pass. + +- [ ] **Step 11: Commit clippy nits** +```bash +git checkout pixi.lock 2>/dev/null +git add src/ +git commit --no-verify -m "$(cat <<'EOF' +style(svar2): clear new-code clippy nits (auto-deref, doc indent, type alias) + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 12: Documentation consistency + RELEASE-GATE checklist + +All doc corrections (spec §4a–c) plus the release-gate documentation (spec "Out of scope"). + +**Files:** +- Modify: `docs/roadmaps/rust-migration.md` (Phase 6a: guard matrix, gate footnote, notes log, field-routing task line, PR link, RELEASE-GATE subsection) +- Modify: `skills/genvarloader/SKILL.md:66,91,128,170,193-195,437,442` +- Modify: `docs/source/index.md:51`, `docs/source/faq.md`, `docs/source/write.md`, `docs/source/dataset.md`, `docs/source/format.md` + +**Interfaces:** +- Consumes: nothing. +- Produces: docs consistent with shipped behavior; a release-gate checklist in the roadmap. + +- [ ] **Step 1: Roadmap `docs/roadmaps/rust-migration.md` Phase 6a:** + - In the guard-matrix bullet (~`:812-816`), remove `unphased_union` and `"variant-windows"` (both ship now); move them to a supported note. + - Delete the gate footnote (~`:822-826`) claiming variant-windows parity is untested — it is covered by `test_svar2_readbound_variants.py` + `test_svar2_fields_read.py`. + - Amend the 2026-07-05 notes-log entry (~`:890`) OR add a 2026-07-13 entry reflecting shipped scope (unphased_union + variant-windows + INFO/FORMAT field routing done). + - Add a ticked task line: `- [x] var_fields → .svar2 store INFO/FORMAT field routing (plan 2026-07-12-svar2-info-format-field-routing.md).` + - Fill the `_PR: TBD (branch svar2-m6b-kernel)_` link once the PR exists (Task 13 opens it — come back and fill the number, or leave 🚧 until then). + +- [ ] **Step 2: Add the RELEASE-GATE subsection** under Phase 6a in the roadmap: +```markdown +#### ⛔ Release gate (do NOT merge until genoray is released) + +This branch is dev-wired to a local genoray checkout and cannot build off this +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API lives on genoray main (unreleased). Flip ALL of these at +genoray release, then re-run the full py3xx matrix: + +- `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → published crates.io versions. +- `pixi.toml` [feature.py310.pypi-dependencies]: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"`. +- Verify the version-floor bumps already made are intended: numpy 0.29, pyo3 0.29, seqpro 0.21.1. +``` + +- [ ] **Step 3: `skills/genvarloader/SKILL.md` corrections:** + - `:193-195`: `var_fields` on `.svar2` accepts only `alt|ilen|start` + store INFO/FORMAT fields — NOT `ref`/`dosage` (they raise). State it. + - `:66,170,437`: "`min_af`/`max_af` requires SVAR-backed genotypes" → "`.svar` only (not `.svar2`, which raises `NotImplementedError`)". + - `:128,442`: note `extend_to_length` is unsupported for a `.svar2` source (raises — matches Task 5). + - `:91`: qualify "byte-identical … all four output modes" with "except pure-deletion ALT bytes (see below)". + +- [ ] **Step 4: Prose docs `docs/source/`:** + - `index.md:51`: "Currently supports VCF, PGEN, and BigWig" → mirror README's `.svar`/`.svar2` wording. + - `format.md`: add the `.svar2` guard-matrix list (unsupported combos) that `faq.md:81` and `write.md:98` promise "for the full list"; OR repoint those two to `faq.md`. Pin the `(unreleased)` changelog row (`:145`) to the target version. + - `write.md` §"Variants from a genoray sparse store": add a 2-line build snippet (how to create `.svar`/`.svar2` via genoray), since `faq.md:76` promises it. + - `dataset.md`: add a short "Variant fields (`var_fields`)" section — `.svar2` store INFO/FORMAT fields on `variants`/`variant-windows` (`rv["AF"]`, `win.fields["AF"]`), with the "only alt/ilen/start + store fields" caveat. + +- [ ] **Step 5: Verify api.md ↔ __all__ still clean** (no public symbol added by this branch): + +Run: `pixi run -e dev python -c "import genvarloader as g; api=open('docs/source/api.md').read(); print('MISSING:', [n for n in g.__all__ if n not in api] or 'none')"` +Expected: `MISSING: none`. + +- [ ] **Step 6: Docs build sanity** (optional but preferred): + +Run: `pixi run -e docs doc 2>&1 | tail -5` +Expected: build succeeds (or the same warnings as `main` — compare; no NEW errors). + +- [ ] **Step 7: Commit** + +```bash +git checkout pixi.lock 2>/dev/null +git add docs/ skills/ +git commit --no-verify -m "$(cat <<'EOF' +docs(svar2): sync roadmap/skill/prose docs + add release-gate checklist + +Roadmap: move unphased_union/variant-windows to supported, drop the stale +gate footnote, add the field-routing task line, add a ⛔ release-gate section +for the genoray path-pins. SKILL: correct var_fields (.svar2 = alt/ilen/start ++ store fields, no ref/dosage), min_af/max_af = .svar only, extend_to_length +unsupported for .svar2. Prose: index.md source list, format.md guard matrix + +changelog pin, write.md build snippet, dataset.md var_fields section. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +### Task 13: `tmp/svar2_mvp/` relocation, final full-suite gate, push, draft PR + +Relocate the scratch tree into `tests/benchmarks/`, run the complete verification gate, and open the draft PR with the release-gate warning. + +**Files:** +- Move: benchmark/profiling drivers from `tmp/svar2_mvp/` → `tests/benchmarks/` and `tests/benchmarks/profiling/` +- Delete: `.sbatch` files, `env_baseline.txt`, `prof_out/*.md` (from git) +- Modify: `.gitignore` (drop the `tmp/svar2_mvp/prof_out/` line; add `tmp/`) + +**Interfaces:** +- Consumes: existing `tests/benchmarks/conftest.py` fixtures (`data_dir`, `kg_dir`, etc.). +- Produces: no machine-specific scratch tracked in git; useful drivers live under `tests/benchmarks/` with parameterized paths. + +- [ ] **Step 1: Inventory the tracked tmp files:** + +Run: `git ls-files tmp/svar2_mvp/; echo "--- untracked ---"; git status --short tmp/svar2_mvp/` + +- [ ] **Step 2: Move the reusable drivers** (`benchmark.py`, `bench_gvl_svar1_vs_svar2.py`, `build_stores.py`, `validate.py`, `split_folded.py`, `prof_*.py`, `perf_kernel_driver.py`) into `tests/benchmarks/`, and the perf shells (`prof_perf.sh`, `e1_profile.sh`, `prof_python.py`) into `tests/benchmarks/profiling/`: +```bash +git mv tmp/svar2_mvp/benchmark.py tests/benchmarks/bench_svar2_getitem.py +# ...repeat per file, choosing descriptive names; use `git add` for currently-untracked ones after moving +``` +For untracked files (`bench_gvl_svar1_vs_svar2.py`, `perf_kernel_driver.py`, `prof_cprofile.py`), `mv` then `git add` at the new path. + +- [ ] **Step 3: Parameterize hardcoded paths.** Replace `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` and similar absolutes with the `tests/benchmarks/conftest.py` fixtures where the file becomes a pytest module, or a module-level constant + `argparse` default for standalone scripts. Match the pattern in the existing `tests/benchmarks/profiling/profile_*.py`. + +- [ ] **Step 4: Drop machine-specific scratch from git:** +```bash +git rm tmp/svar2_mvp/e2_build.sbatch tmp/svar2_mvp/e2_subsample.sbatch tmp/svar2_mvp/genoray_debug_build.sbatch tmp/svar2_mvp/e1_bucket_dso.py tmp/svar2_mvp/env_baseline.txt +git rm -r tmp/svar2_mvp/prof_out/ +git rm tmp/svar2_mvp/prof_driver.py tmp/svar2_mvp/prof_getitem.py 2>/dev/null || true +``` +(Keep only what's genuinely reusable; the perf conclusions already live in `docs/superpowers/notes/` and the roadmap.) + +- [ ] **Step 5: Fix `.gitignore`** — remove the self-contradicting `tmp/svar2_mvp/prof_out/` line, add a blanket `tmp/`: +```bash +grep -v "tmp/svar2_mvp/prof_out/" .gitignore > .gitignore.new && mv .gitignore.new .gitignore +printf 'tmp/\n' >> .gitignore +``` +Then confirm `tmp/` is gone from tracking: `git status --short tmp/` should show nothing staged except the removals/moves. + +- [ ] **Step 6: Sanity-run one relocated benchmark** (if it became a pytest module) or import-check it: +```bash +pixi run -e dev python -c "import ast; ast.parse(open('tests/benchmarks/bench_svar2_getitem.py').read()); print('parses')" +``` +Expected: `parses` (full run needs real data; a parse + import check is enough for the gate). + +- [ ] **Step 7: FINAL FULL-SUITE GATE.** Rebuild and run everything: +```bash +pixi run -e dev maturin develop --release 2>&1 | tail -1 +pixi run -e dev cargo test 2>&1 | tail -6 +pixi run -e dev pytest tests -q 2>&1 | tail -6 +pixi run -e dev ruff check python/ tests/ 2>&1 | tail -2 +pixi run -e dev ruff format --check python/ tests/ 2>&1 | tail -2 +pixi run -e dev typecheck 2>&1 | tail -3 +pixi run -e dev cargo clippy --all-targets 2>&1 | grep -c "warning:" +``` +Expected: cargo tests pass; full pytest tree green (SVAR1 unchanged, SVAR2 31/31, all new tests pass); ruff clean; ruff format clean; typecheck 0 errors; clippy warning count == pre-existing baseline. + +- [ ] **Step 8: Commit the relocation** +```bash +git checkout pixi.lock 2>/dev/null +git add -A +git commit -m "$(cat <<'EOF' +chore(svar2): relocate tmp/svar2_mvp scratch into tests/benchmarks + +Move the reusable SVAR2 benchmark/profiling drivers under tests/benchmarks/ +with parameterized paths (via the existing conftest fixtures), drop +machine-specific sbatch/env scratch and prof_out reports from git (perf +conclusions already live in docs/superpowers/notes/ + the roadmap), and fix +the self-contradicting .gitignore (blanket-ignore tmp/). + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +- [ ] **Step 9: Push the branch:** +```bash +git push -u origin svar2-m6b-kernel 2>&1 | tail -3 +``` + +- [ ] **Step 10: Open the draft PR with the release-gate warning:** +```bash +gh pr create --draft --repo mcvickerlab/GenVarLoader --base master --head svar2-m6b-kernel \ + --title "SVAR2 read-bound dataset support (M6b)" \ + --body "$(cat <<'EOF' +## ⛔ DO NOT MERGE until genoray is released + +This branch is dev-wired to a local genoray checkout and builds only on the dev +machine. PyPI genoray tops out at 2.15.0; the INFO/FORMAT field-read + +read-bound gather API is on genoray main (unreleased). Before merge, flip ALL: + +- [ ] `Cargo.toml`: `svar2-codec` / `genoray_core` path-deps → crates.io versions +- [ ] `pixi.toml`: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="` +- [ ] `pyproject.toml`: `"genoray"` (unpinned) → `"genoray>=,"` +- [ ] Re-run the full py3xx matrix on the released wheel + +## Summary + +Adds SVAR2 (`.svar2` sparse variant format) as a `gvl.write` source and a live, +read-bound `Dataset` backend: all-Rust FFI kernels (haplotypes, tracks, +variants, variant-windows), INFO/FORMAT field routing into variant outputs via +`var_fields`, and `unphased_union`. See `docs/roadmaps/rust-migration.md` +Phase 6a and the specs/plans under `docs/superpowers/`. + +Final pre-merge pass (this session): serial-unsafe-path guard, Python-reachable +panics → PyValueError, `extend_to_length` guard, vectorized write-time +max_ends, test-only oracle relocation, dead FFI capability removal, Rust +de-duplication, typecheck-task fix, and doc consistency. + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" 2>&1 | tail -3 +``` + +- [ ] **Step 11: Backfill the PR link** into `docs/roadmaps/rust-migration.md` Phase 6a (`_PR: TBD_` → the PR URL from Step 10), commit `--no-verify`, and push. + +- [ ] **Step 12: Report** the PR URL and the final gate results. + +--- + +## Self-Review + +**Spec coverage:** +- §Release gate → Task 12 (roadmap section) + Task 13 (PR warning). ✅ +- §1a serial unsafe guard → Task 3. §1b get_unchecked doc → Task 1. §1c panics→PyErr → Task 4. §1d extend_to_length → Task 5. ✅ +- §2a oracle relocation → Task 9. §2b dead FFI capability → Task 10. §2c max_ends vectorize → Task 6. §2d region_starts → Task 7. §2e typecheck task → Task 2. §2f tmp/ relocation → Task 13. ✅ +- §3a carve helper, §3b FFI preamble, §3c present_bit → Task 11. ✅ +- §4a roadmap, §4b SKILL, §4c prose, §4d strip internal refs → Tasks 12 + 1. §4e missing docstrings → covered opportunistically in Tasks 1/9 (see note below). ✅ +- §5 clippy → Task 11 Step 9. ✅ +- §8 FlankSample issue → Task 8. ✅ + +**Note on §4e (missing docstrings):** the spec lists docstrings to add (`make_svar2_link`, `_reconstruct_variants`, `_write_from_svar2`, `svar2/store.rs` items). These are folded into the tasks that already touch those files: `_write_from_svar2` in Task 5, the `svar2/store.rs` PyO3 items are low-risk — **add a short Step to Task 10** or fold into Task 1's comment pass. To avoid a gap, the implementer should add the four docstrings as part of whichever task first opens each file; if none does (e.g. `store.rs`), append them to Task 1. Explicitly: add `///` to `svar2/store.rs` `reader`/`store_path`/`#[new]`/`contigs` in Task 1 Step 3. + +**Placeholder scan:** no TBD/TODO left except the deliberate `_PR: TBD` which Task 13 Step 11 fills. The Task 4 Step 1 test body has a modeled-on-sibling `...` — flagged explicitly with instructions, acceptable since the exact fixture is repo-specific and the assertion (`pytest.raises(ValueError)` on a strided view) is concrete. + +**Type consistency:** helper names consistent across tasks — `require_contiguous` (Task 4), `carve_chunks`/`offsets_from_diffs`/`svar2::present_bit` (Task 11), `build_readbound_*`/`SparseVar2Source` at new paths (Task 9). `_svar2_region_max_ends` signature unchanged (Task 6). `reconstruct_haplotypes_from_svar2_readbound` loses `annot_*` in Task 10 — no later task references those params. diff --git a/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md b/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md new file mode 100644 index 00000000..63b07b9e --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-dataset-wiring-design.md @@ -0,0 +1,198 @@ +# SVAR2 Dataset Wiring — Design Spec + +> **Purpose:** wire the SVAR2 format into the gvl `Dataset` the way SVAR1 already is — +> by caching genoray's interval-search result at `gvl.write` time and replaying it at read +> time — so SVAR2 haplotype/track reads stop paying a per-query search-tree rebuild. This is +> the deferred **Task B** (`TODO(svar2-dataset-dispatch)` in `_svar2_source.py`), now +> justified and shaped by the E1 profiling result below. + +**Date:** 2026-07-03 · **Depends on:** the M6b SVAR2 kernels (`reconstruct_haplotypes_from_svar2`, +`shift_and_realign_tracks_from_svar2`, already built + parity-validated vs the genoray `decode` +oracle), the `SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`), and the +E1 profiling result (`docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md`). **Two +deliverables:** a genoray PR (search/gather split) then a gvl PR (write + read wiring). + +## Problem & evidence (E1) + +The benchmarked SVAR2 path is the **raw `SparseVar2Source` adapter**, which calls genoray +`SparseVar2.overlap_batch` **live on every query**. E1 profiled it with `perf --call-graph fp` +(py-spy is unusable on Carter compute nodes — `ptrace_scope=2`), attributing time by DSO on the +3-region × all-samples chr21 workload: + +| path | native genoray `_core.so` | gvl kernel | numpy conv | python interp | +| --- | --- | --- | --- | --- | +| **svar2 germline** (3202) | **88.3%** (68.6% `SearchTree::build` + 14.3% `overlap_batch`) | 1.1% | 0.1% | 3.9% | +| **svar2 somatic** (16007) | **78.5%** (60.3% `SearchTree::build`) | 0.7% | 0.2% | 4.8% | +| svar1 germline | 0% | 56.9% (`get_diffs_sparse` + `reconstruct…_from_sparse`) | 20.4% | 7.3% | +| svar1 somatic | 0% | 10.2% (rest: genotype IO / ZSTD decompress + page faults) | 24.2% | 12.1% | + +**Conclusion:** SVAR2 adapter latency is **neither Python-adapter overhead (≤5%) nor the gvl +reconstruct kernel (~1%) nor numpy conversions (~0.1%)** — it is genoray **rebuilding interval +search trees on every `overlap_batch` call**. In genoray's source: `DenseUnion::overlap` builds a +fresh `SearchTree::new(&self.positions)` **per region** (`src/query.rs:288`, comment "one per region +in a batch") over the contig-wide dense union, and `vk_slice → spine::gather_keys` builds one **per +hap** (`src/spine.rs:48`). SVAR1 does not pay this: `gvl.write` calls genoray's **search-only** +`_find_starts_ends` once and caches a compact `(2, R, S, P)` offsets memmap +(`_write_from_svar` in `python/genvarloader/_dataset/_write.py:961`); reads then **slice the shared +`.svar` store directly** by those offsets, with no search. **This spec gives SVAR2 the same +write-time cache** — the fix E1 points to. (The earlier "dense-gather layout" hypothesis, E3, is +**moot**: the gvl kernel where that gather lives is ~1% of time; the cost is the tree build.) + +## Approach — mirror SVAR1 + +Split genoray's fused `overlap_batch` into a **search-only** step and a **tree-free gather** step, +mirroring SparseVar's `_find_starts_ends` / `read_ranges` pair. `gvl.write` runs the search once and +caches the compact result into `.gvl`; `Dataset` reads replay the cache through the gather step and +the existing SVAR2 kernels. The bulk variant data stays in the compressed `.svar2`, referenced by a +`Svar2Link` — so the cache is O(offsets) and **SVAR2's on-disk size advantage (1.46–5.67× smaller +than `.svar`, established in the MVP benchmark) is preserved**, not materialized away. + +Unlike SVAR1 (whose per-hap variants are a contiguous slice, so reads slice the shared store with no +genoray call), SVAR2's dense-presence bits are **computed** from the class tables (the `carried` +tests), not sliceable. So SVAR2's read path makes one genoray gather call per read — but with **no +interval search**, which is the entire cost E1 identified. + +## Component A — genoray API (dependency; separate genoray PR) + +Refactor `overlap_batch` into three public methods on `SparseVar2`, mirroring SparseVar's naming and +all accepting a `samples=` subset (like every SparseVar range method). Signatures follow SparseVar's +`_find_starts_ends` / `read_ranges` exactly where analogous: + +- **`find_ranges(contig, starts, ends, samples=None, out=None) → ranges`** — *search-only*, the + analog of `SparseVar._find_starts_ends`. Builds each search tree **≤ once per contig** and returns + the compact bundle needed to gather later: per-region `dense_range` `(R, 2)` and per-hap var_key + column offsets (no gathered `KeyRef`s, no presence bits). `samples=` restricts which samples' + offsets are computed; `out=` writes into a preallocated memmap so `gvl.write` streams the cache + straight to disk (exactly as `_find_starts_ends(..., out=out)` does for SVAR1). + +- **`gather_ranges(contig, ranges, samples=None) → payload`** — the *gather* step. Consumes a + `find_ranges` bundle (the **cached ranges**, not `starts/ends`) and therefore performs **no + interval search**: it slices var_key from the shared `.svar2` store and computes dense-presence + bits tree-free, returning the full reconstruct payload dict (`vk_pos`, `vk_key`, `vk_off`, + `dense_pos`, `dense_key`, `dense_range`, `dense_present`, `dense_present_off`, `lut_bytes`, + `lut_off`) — the exact input shape the SVAR2 kernels already accept. `samples=` restricts which + samples are materialized (cache all at write, read any subset). + +- **`read_ranges(contig, starts, ends, samples=None) → payload`** — the public **fused** API, exactly + like `SparseVar.read_ranges`. A thin wrapper: + `gather_ranges(contig, find_ranges(contig, starts, ends, samples), samples)`. It replaces + `overlap_batch`'s role for live/uncached queries and serves as the parity oracle. (`overlap_batch` + may be kept as a deprecated alias or removed — genoray maintainer's call.) + +**Contract (byte-identical parity):** for any `contig, starts, ends, samples`, +`reconstruct(read_ranges(...))` ≡ `reconstruct(gather_ranges(find_ranges(...)))` ≡ +`reconstruct(overlap_batch(...))` ≡ genoray `decode` oracle, byte-for-byte. gvl treats the +`find_ranges` bundle as an **opaque** array bundle — it persists and replays it, with no gvl coupling +to genoray's internal layout (the same way the adapter treats the `overlap_batch` dict today). + +## Component B — gvl write (`_write.py`, new `_write_from_svar2`, new `_svar2_link.py`) + +Mirror the SVAR1 write path: + +- **Detect `.svar2`.** In `write()` variant-source dispatch (`_write.py:~225`, alongside the existing + `.svar` branch), a directory with suffix `.svar2` → `SparseVar2(dir)`. Add a `SparseVar2` branch to + the genotype-writing dispatch (`_write.py:~325`, next to `isinstance(variants, SparseVar)`). + +- **`_write_from_svar2(path, bed, svar2, samples, extend_to_length)`** mirrors `_write_from_svar` + (`_write.py:961`): allocate a memmap in `/genotypes/`, and per contig call + `svar2.find_ranges(c, df["chromStart"], df["chromEnd"], samples=samples, out=out)` to stream the + compact ranges cache to disk. Write `/genotypes/svar2_meta.json` (bundle shapes/dtypes) and + a `Svar2Link` into `metadata.json`. Set `metadata["ploidy"] = svar2.ploidy`. + +- **`_svar2_link.py`** mirrors `_svar_link.py`: a `Svar2Link` pydantic model + (`relative_path`, `absolute_path`, `fingerprint`) + `_resolve_svar2(gvl_path, link, override)` and + `_verify_fingerprint(...)`. Fingerprint the `.svar2` on its stable identity (e.g. `n_variants` from + its index + a byte count of a canonical store file), analogous to `SvarFingerprint`. + +- **Reject unsupported variants** (symbolic/breakend ALTs) as SVAR1 does; upstream normalization + (genoray roadmap M13, `-V other,bnd`) already filters these in the MVP build scripts. + +## Component C — gvl read (`_open.py`, `_haps.py`, `_reconstruct.py`, `_tracks.py`; refactor `_svar2_source.py`) + +- **`Dataset.open`** resolves + fingerprints the `Svar2Link` (mirroring `_resolve_svar` / + `_verify_fingerprint`, with a `svar2=` override on `open` paralleling `svar=`), and holds a + `SparseVar2` handle plus the memmapped cached-ranges bundle. + +- **Haplotypes.** The `Haps` reconstruction path routes SVAR2 datasets to: load the cached ranges for + the requested `(region, sample)` block → `svar2.gather_ranges(contig, ranges_block, samples=block)` + → `reconstruct_haplotypes_from_svar2(...)` → `Ragged` haps. This retires + `TODO(svar2-dataset-dispatch)`. A dataset carries a source discriminant (svar2_link present) that + selects the SVAR2 reconstructor over the SVAR1 one. + +- **Tracks.** `with_tracks` / the track re-aligner routes to `shift_and_realign_tracks_from_svar2` + via the **same** cached ranges + `gather_ranges` — the cache is written once and serves both + haplotype and track reconstruction. + +- **Refactor `_svar2_source.py`.** Replace the adapter's live `overlap_batch` call with the + cached-ranges + `gather_ranges` path (or fold its kernel-marshalling into `Haps`). The + region→(R·S) expansion and `ascontiguousarray` marshalling already there stay valid; only the data + source changes from live query to cache+gather. + +## Cache format (`.gvl/genotypes/`) + +- The compact `find_ranges` bundle, region/sample-sharded to match the dataset layout (per-region + `dense_range` + per-hap var_key offsets), stored as memmaps written in place via `out=`. + `svar2_meta.json` records shapes/dtypes (mirrors SVAR1's `svar_meta.json`). `Svar2Link` → + shared `.svar2`. +- **Size:** O(offsets) — same order as SVAR1's `offsets.npy`, **not** the ~1.8 MB/3-region + `overlap_batch` payload. The bulk (var_key positions/keys, dense class tables, LUT) stays in the + compressed `.svar2`. + +## Data flow + +- **WRITE:** `bed + .svar2` → per contig `find_ranges(..., out=memmap)` (search once) → compact ranges + cache in `.gvl/genotypes/` + `Svar2Link` + `svar2_meta.json`. +- **READ:** `dataset[regions, samples]` → load cached ranges block → `gather_ranges` (tree-free) → + `reconstruct_haplotypes_from_svar2` / `shift_and_realign_tracks_from_svar2` → haps / tracks. **No + interval search at read.** + +## Parity & testing (the contract) + +- **Byte-identical:** cached-path reconstruct ≡ live `read_ranges`/`overlap_batch` reconstruct ≡ + `decode` oracle, on the M6b matrix (SNP / INS / DEL × samples × ploids) + real chr21 germline & + somatic stores. Track re-alignment matched the same way. +- **Additive guarantee:** the SVAR1 path is byte-unchanged — full SVAR1 regression suite green + (`pixi run -e dev pytest tests -q`; and `cargo test` for the kernels). Follows the rust-migration + byte-identical parity contract and the numba-oracle-bug policy (if the cached path and a numba + oracle disagree, check whether numba is the buggy one before "fixing" the new path). +- **Perf verification (same-session, shared-node caveat):** a warm SVAR2 `Dataset` read no longer + shows `SearchTree::build` — the perf DSO split flips from ~80% genoray to gvl-kernel-bound, like + SVAR1. Report as a relative before/after within one allocation (absolute wall-clock is not + comparable across allocations on the shared Carter nodes). +- **Docs/roadmaps:** update the genoray roadmap (search/gather split + read-bound conversion open + question) and `docs/roadmaps/rust-migration.md`; update user-facing docs for `.svar2` as a `write` + variant source — `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, + `README.md` (per the repo's docs-audit + skill-maintenance gates), and keep `api.md` in sync with + any new `__all__` symbols. + +## Out of scope + +- SVAR2 **variants** and **annotated** output modes (this spec covers haplotypes + tracks only; the + same cache extends to them later). +- The remaining profiling experiments **E2** (same-cohort sample-count sweep) and **E4** (conversion + build-thread allocation) — independent of this fix; parked, resumable from the profiling plan. + **E3** (dense-access layout probe) is **dropped** — E1 established the gvl dense-gather is ~1% of + time, not the hot path. +- Any change to the on-disk `.svar2` format itself (that is genoray's; this spec only adds the + search/gather split and the gvl-side cache). + +## Deliverables & sequencing + +1. **genoray PR** — `find_ranges` / `gather_ranges` / `read_ranges` split with `samples=`, parity + tests vs the current `overlap_batch`, release a wheel. (Crate/wheel release gate per the existing + SVAR2 dev-wiring notes.) +2. **gvl PR** — `_write_from_svar2` + `_svar2_link.py` (write) and the `_open.py` / `_haps.py` / + `_reconstruct.py` / `_tracks.py` dispatch + `_svar2_source.py` refactor (read), targeting the + genoray API from (1); byte-identical parity + docs/roadmap updates. + +## Open questions / risks + +- **`find_ranges` bundle contents:** the exact compact fields (are per-hap var_key offsets one range + per hap, or separate SNP + indel channel ranges?) are a genoray-internal detail settled in the + genoray PR; the gvl side stays agnostic (opaque persist/replay), so this does not block the gvl + design. +- **`gather_ranges` presence-bit cost at read:** the per-hap `carried` tests over `dense[ds..de]` + remain at read (they are cheap, no tree) — confirm in the perf verification they don't become the + new hot path (E1 already shows this work is a small fraction of the gvl-side once the tree is gone). +- **Format version:** adding `svar2_link` to `metadata.json` is additive; confirm + `_check_dataset_format_version` tolerates it and that a bump (if any) is handled by `_migrate.py`. diff --git a/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md new file mode 100644 index 00000000..8eaff462 --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-gvl-mvp-validate-benchmark-design.md @@ -0,0 +1,203 @@ +# SVAR2 gvl MVP — validate & benchmark first, wire second + +**Date:** 2026-07-03 · **Epic:** SVAR 2.0 · **Repo:** GenVarLoader +**Worktree:** `.claude/worktrees/svar2-m6b-kernel` (branch `svar2-m6b-kernel`) +**Follows:** `tmp/handoffs/2026-07-03-svar2-m6-plan3-mvp-benchmark-handoff.md` + +## Summary + +Plan 3 delivered the SVAR2 two-source reconstruction kernels and a validated +`SparseVar2Source` adapter (`python/genvarloader/_dataset/_svar2_source.py`). This +work finishes the MVP by **proving the SVAR2 value proposition on real chr21 data +before committing to invasive Dataset integration**. The session runs: + +- **A** — clean up merged genoray worktrees (mechanical) +- **C** — end-to-end test for the `realign_tracks` adapter path +- **D** — validate both backends on real chr21 germline + somatic data +- **E** — benchmark SVAR1-backed vs SVAR2-backed gvl (latency + store size) +- **B** — Dataset dispatch wiring — **design sketch only**, finalized in a later + brainstorm once E's numbers are in + +### Guiding decision + +Task D (validation) and Task E (benchmark) **do not depend on Task B** (full Dataset +dispatch): both run against `SparseVar2Source` directly. Task B is invasive to the +central reconstructor and its cost/shape depend on facts E will surface (notably the +all-samples-per-batch decode cost). So we **prove value first (A→C→D→E), then design +and wire B** with the benchmark in hand. This is measure-first and YAGNI: we do not +build invasive core integration before the payoff is confirmed. + +## Background: why SVAR2 doesn't fit the SVAR1 reconstructor + +The SVAR1 path (`Haps`, `_haps.py`) materializes per-region genotype offsets plus a +variant table on disk and memmaps them; `Haps` holds `genotypes` (a Ragged of +`v_idxs`), a `_Variants` table, and cached `ffi_static` arrays. + +The SVAR2 path is a **live-query** model: `SparseVar2Source` calls genoray +`SparseVar2.overlap_batch(contig, regions)` at request time and decodes +`var_key ⋈ dense` inline through the Plan-3 kernels — there is no materialized +genotype table. The two models share none of that state, which is why B is a new +reconstructor rather than a flag on `Haps` (see Task B). + +Three frictions make B genuinely invasive (deferred, but recorded here so the +validation/benchmark methodology accounts for them): + +1. **Query-granularity mismatch.** `overlap_batch` is *per-contig, all-samples, + all-ploidy* — it has no sample-subset argument (confirmed in genoray + `python/genoray/_svar2_batch.py`). gvl's `Reconstructor.__call__` receives an + arbitrary `idx` of `(region, sample)` pairs, possibly across multiple contigs. +2. **No write/detection path.** `_write.py` only handles `SparseVar`/VCF/PGEN; nothing + writes an SVAR2-backed gvl dataset, and `_build_seqs` (`_open.py`) has no SVAR2 + branch. +3. **Reconstructor shape.** A branch inside `Haps` would leave `genotypes`/`_Variants`/ + `ffi_static` dead for SVAR2. + +## Environment (from the handoff — do not re-derive) + +- gvl worktree build: `pixi run -e default maturin develop` after Rust edits; the + compiled module is `genvarloader.genvarloader` (abi3). +- Rust tests: `pixi run -e default cargo test --no-default-features [FILTER]`. +- Commits: prek hooks intentionally not installed here — use `git commit --no-verify`. +- genoray is the pre-built **2.15.0 wheel** in gvl's `default` env (not editable). + Rebuild via `cd /carter/users/dlaub/projects/genoray && pixi run -e py310 maturin + build --release` then re-point `pixi.toml` + `pixi install -e default`. + +## Task A — Worktree cleanup (mechanical) + +Confirm `git -C status` is clean, then remove the three merged/stale genoray +worktrees: + +``` +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6b +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6c +git -C /carter/users/dlaub/projects/genoray worktree remove .claude/worktrees/svar-2-m6-core +``` + +No design surface. Abort a removal if that worktree reports uncommitted work. + +## Task C — `realign_tracks` e2e test + +`SparseVar2Source.realign_tracks` (the region→R·S track expansion) has no end-to-end +test; only the Rust driver unit test `svar2_track_realign_del` exists. + +**Oracle: gvl's SVAR1 _pure-Python_ `shift_and_realign_track_sparse` +(`_dataset/_tracks.py:708`), fed the variants materialized from genoray +`SparseVar2.decode`.** This is the strongest low-cost oracle and resolves the handoff's +(a)/(b) tension: +- It reuses *trusted* SVAR1 realign logic — including the DEL-anchor branch already + present at `_tracks.py:755` — so we don't hand-roll a second track-shift + implementation (the weakness of a fresh pure-Python consensus). +- It is a **pure-Python** implementation, genuinely independent of the SVAR2 **Rust** + kernel under test (`shift_and_realign_tracks_from_svar2`, a closure refactor of the + *Rust* SVAR1 kernel — a different implementation from this Python fallback). +- Setup is trivial: build a synthetic single-hap genotype layout from decode records — + `geno_v_idxs = arange(n_h)`, `geno_offsets = [0, n_h]`, `v_starts = pos_h`, + `ilens = ilen_h` — mirroring how `test_svar2_reconstruct.py` feeds decode records to + its oracle. + +**Construction:** +- Reuse the store builder + `decode_batch` extraction from + `tests/test_svar2_reconstruct.py`. +- Use **DEL-only variants** so insertion-fill is bypassed and the strategy choice is + irrelevant (any valid `strategy_id`/`params` works — track realign is ilen-only for + DELs). +- Define a per-region reference track (f32); realign through the SVAR2 adapter path + (`SparseVar2Source.realign_tracks`) and, per hap, through the pure-Python oracle fed + that hap's decoded `(pos, ilen)` with `shift=0` and `out` sized to the hap length. +- **Assert per-`(r,s,p)`** float equality between the two (`np.testing.assert_allclose`). + +Place alongside `tests/test_svar2_reconstruct.py`. + +## Task D — Real-data validation (exploratory, not pytest) + +Prove the whole thing on real chr21 in `/carter/users/dlaub/repos/for_loukik/`. + +**Inputs** +- Germline (1000G): `chr21.bcf` (177 MB, 3202 samples) — **no `.csi`**, must index. +- Somatic (GDC): `gdc.chr21.bcf` (1.1 GB, 16007 samples), `.csi` present. +- **Reference:** `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` (GDC GRCh38; + `.fai` and a gvl-preprocessed `.gvlfa` already present). + +**Steps** +1. Verify contig naming (`chr21` vs `21`) between the FASTA and each BCF; if the + germline BCF uses `21`, `bcftools annotate --rename-chrs` (or query with the BCF's + own naming). +2. `bcftools index --csi chr21.bcf` (germline). +3. Normalize+atomize both BCFs to biallelic: `bcftools norm -m -any --atomize -Ob`, + reusing the `genoray_pipeline.py` / `run_conversion_pipeline` recipe (samples via + `bcftools query -l`, chroms via the canonical `##contig` regex, + `run_conversion_pipeline(..., chunk_size=25_000, ploidy=2, ...)`). +4. Build **both** stores per source: `.svar` (SVAR1, `SparseVar.from_vcf`) and `.svar2` + (SVAR2, `run_conversion_pipeline`) — genoray 2.15.0 wheel. +5. **Validate** gvl returns haplotypes + variants through both backends: + - SVAR2 haplotypes: `SparseVar2Source(sv2).reconstruct("chr21", regions, + ref_bytes, ref_offsets, pad_char)`. + - SVAR2 variants: genoray `SparseVar2.decode`. + - SVAR1: a gvl `Dataset` over the `.svar` store (haplotype + variants modes). + - Spot-check a few regions agree in spirit (both from the same source variants). + +**Scope:** small — a handful of regions × a few samples for the correctness spot-check. +Correctness is already proven by the Rust + adapter tests; D proves the **real-data +plumbing**: that genoray converts real multi-thousand-sample BCFs and that the adapter +works on a non-synthetic store. + +**Compute:** germline may run interactively; the somatic conversion (16k samples, +1.1 GB) is heavy — run via `sbatch -p carter-compute`. + +**Success:** both stores build from the real BCFs, and gvl returns non-empty, sane +haplotypes + variants through both backends, with spot-checks agreeing. + +## Task E — Benchmark SVAR1-backed vs SVAR2-backed gvl + +Once D produces both stores, measure and tabulate. + +**Fairness rule (the crux):** query the **same workload** on both backends — **all +samples for a fixed region set**. This matches genoray's per-contig/all-samples query +granularity and is a realistic population-scale workload; a `(region, sample)`-subset +workload would unfairly tax the SVAR2 adapter, which always decodes all samples. + +**Method:** warm caches, N repeats, report **median**. Same regions/sample set for both +backends. + +**Measurements** (germline **and** somatic): +- **Hap latency** — SVAR1 gvl `Dataset` vs SVAR2 `SparseVar2Source`. +- **Variant latency** — SVAR1 gvl variants mode vs genoray `SparseVar2.decode`. +- **Store size** — `du` of the `.svar` (SVAR1) vs `.svar2` (SVAR2) directory. + +**Output:** a 2×3 table — germline/somatic × {hap latency, variant latency, store +size}. + +**Recorded caveat:** this compares adapter-vs-Dataset, not Dataset-vs-Dataset (B is not +wired). SVAR2 latency therefore excludes gvl's batching/collation overhead. Somatic +(dense somatic mutations) is where SVAR2's two-channel `var_key ⋈ dense` layout should +win on store size. + +## Task B — Dataset dispatch (design sketch; finalize after E) + +Deferred by the session's sequencing decision. Direction for the later brainstorm: + +- A **new `SparseVar2Reconstructor`** implementing the `Reconstructor` protocol + (`_protocol.py`) — *not* a branch inside `Haps` (SVAR2 shares none of `Haps`'s + `genotypes`/`_Variants`/`ffi_static` state). +- `__call__` adapts gvl's batch model to genoray's: group `idx`→`(r,s)` **by contig**, + decode all-samples-per-region via `SparseVar2Source`, **select** the requested + `(r,s,p)` rows, thread jitter `shifts` and contig `ref` bytes from `Reference` + (`_reference.py`), and return the identical `_Flat`/`Ragged` output contract. +- A new **write branch** (record an svar2 link + a `backend:"svar2"` flag in + `metadata.json`, no genotype materialization) plus **open-path** detection in + `_build_seqs` (`_open.py`). +- **Open question E will inform:** the all-samples-per-batch decode cost — whether to + push sample-subsetting down into genoray (`overlap_batch`) or accept + decode-all-then-select for the MVP. +- **Additive guarantee:** the SVAR 1.0 path must stay byte-identical; existing SVAR1 + haplotype/track tests must remain green. + +## Carried-over gotchas (from the handoff) + +- Subagents default to the **main repo, not the worktree** — every dispatch must `cd` + the gvl worktree and guard with `git rev-parse --show-toplevel`. The gvl worktree is + a different repo from genoray. +- `cargo test` needs `--no-default-features`; `maturin develop` after Rust edits. +- genoray `SparseVar2.decode`/`overlap_batch` return **empty ALT for pure DELs** — + reconstruction injects the anchor `ref[pos]`. Any SVAR2 consumer must honor this. +- genoray positions are **0-based**; hap order is region-major `h=(r·S+s)·P+p`. diff --git a/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md b/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md new file mode 100644 index 00000000..eb051f75 --- /dev/null +++ b/docs/superpowers/specs/2026-07-03-svar2-profiling-followup.md @@ -0,0 +1,153 @@ +# SVAR2 Profiling Follow-up — Spec + +> **Purpose:** turn the SVAR2 MVP benchmark's *unprofiled* latency observations into +> **concrete, attributed numbers** so we can decide (a) how much of the SVAR2 query-latency +> gap is Python-adapter overhead vs Rust hot paths vs memory layout, and (b) how to +> rebalance conversion threads for read-bound (few-contig) builds. **Measurement only — no +> optimization work is in scope here.** + +**Date:** 2026-07-03 · **Depends on:** the MVP stores + scripts from +`2026-07-03-svar2-gvl-mvp-validate-benchmark` (plan) and its notes +[`../notes/2026-07-03-svar2-mvp-benchmark.md`](../notes/2026-07-03-svar2-mvp-benchmark.md), +[`../notes/2026-07-03-svar2-mvp-validation.md`](../notes/2026-07-03-svar2-mvp-validation.md). + +## Why (background the next session needs) + +The MVP benchmark measured SVAR1 (gvl `Dataset` over `.svar`) vs SVAR2 (`SparseVar2Source` +adapter over `.svar2`), all-samples × 3 chr21 regions, warm, median N=5: + +| cohort | SVAR1 hap (s) | SVAR2 hap (s) | store SVAR1→SVAR2 | +| --- | --- | --- | --- | +| germline (3202 smp) | 0.0555 | 0.2834 | 5.67× smaller | +| somatic (16007 smp) | 0.3798 | 0.3797 | 1.46× smaller | + +**Established (unconfounded):** SVAR2's on-disk store is 1.46–5.67× smaller. + +**NOT established — the reasons this spec exists:** +1. The latency comparison is **not apples-to-apples**: the SVAR2 path is the *raw adapter* + (new Python orchestration — per-call genoray `overlap_batch`, numpy conversions, the + two-source Rust kernel), **not** the gvl `Dataset`. Prior gvl profiling (Dataset latency + is Rust-bound, Python-orchestration negligible) does **not** transfer to this adapter. +2. The "SVAR2 scales better with cohort size" reading is **confounded** — germline vs + somatic differ in AF/density/source, not just sample count. +3. The dense presence matrix is **hap-major** (`genoray:src/query.rs:131-134`, bit + `hap * n_dense_variants + col`; refutes a `(V,S,P)` guess) — per-hap contiguous — **but** + `n_dense_variants` is **contig-wide**, so a narrow-region × all-samples read scatters one + cache line per hap, and presence is read **bit-by-bit** (`get_bit`), not word-parallel. + Plausible but unmeasured. +4. Conversion is **read-bound**: single-contig builds show `1 concurrent chromosome | 3 + HTSlib decompression threads (7/8 active)`, VCF read/decompress dominating (chr21: + germline ~11 min, somatic ~2 h). Thread rebalance is untested. + +## Environment & tooling (read before profiling) + +- **Env:** `pixi run -e default` (only env installed). **Python 3.10.20.** +- **CRITICAL perf/Python caveat:** on **Python < 3.12**, `perf` **cannot** symbolize Python + stack frames — the `-X perf` / `PYTHONPERFSUPPORT` trampoline that emits `perf`-readable + Python symbols only exists in **3.12+** + (see ). This env is 3.10, so `perf` + alone shows resolved **Rust/native** symbols but **opaque** Python frames + (`_PyEval_EvalFrameDefault` …). Therefore: + - **py-spy** (`.pixi/envs/default/bin/py-spy`, confirmed present) — sampling profiler that + resolves **Python** frames on any version and, with `--native`, also unwinds into the + **Rust** extension. Use it for the **Python-vs-native split** (the "total Rust %") and + Python-side hotspots. + - **perf** (`/carter/users/dlaub/.pixi/bin/perf`, confirmed present) — use for + **fine-grained Rust symbol-level** profiling once py-spy says Rust dominates. +- **Rust symbolization for perf/py-spy --native:** rebuild the genoray + gvl extensions + with frame pointers + line-table debuginfo. In the relevant `Cargo.toml` profile (or via + env): `RUSTFLAGS="-C force-frame-pointers=yes"` and `debug = "line-tables-only"` (or + `debug = true`); then `pixi run -e default maturin develop` for gvl, and rebuild the + genoray wheel (its build is `pixi run -e py310 maturin build --release` in the genoray + repo — note genoray is a **frozen wheel** in gvl's env, so a debug-enabled rebuild + wheel + swap is needed to symbolize genoray's `query.rs`/`svar2-codec` frames). Prefer + `perf record -g --call-graph dwarf` if frame pointers are unavailable. +- **Noise control:** run on a dedicated compute node (`sbatch -p carter-compute + --exclusive` or an interactive alloc), warm caches, median of N≥5, pin threads, record + CPU governor / turbo state. Keep the MVP workload (same 3 regions, all-samples) for + continuity. +- **Stores (already built):** `$W = /carter/users/dlaub/repos/for_loukik/svar2_mvp`: + `{germline,somatic}.{svar,svar2,gvl}` + `{chr21,gdc.chr21}.norm.filt.bcf`. Reuse the MVP + `benchmark.py` harness (`GenVarLoader worktree: tmp/svar2_mvp/benchmark.py`). + +## Experiments + +### E1 — Query-latency attribution (the primary experiment) + +For each **(backend × cohort)** — SVAR2 adapter `reconstruct` and SVAR1 `ds_hap[:R,:S]`, +germline + somatic — on the MVP workload (3 regions, all samples): + +1. **py-spy split.** `py-spy record --native --rate 500 -o .svg -- python ` + where `` loops the single path K times (warm first). Also capture + `py-spy record --format speedscope`. Extract: **% wall-clock in Python vs % in native + (Rust)**, and the top Python frames (numpy conversions, `overlap_batch` marshalling, FFI + boundary). This directly answers *"is SVAR2 adapter latency Python overhead?"*. +2. **perf Rust detail** (only where py-spy shows native dominates): `perf record -g + --call-graph dwarf -- python `; `perf report`. Read the top **Rust** symbols — + e.g. `svar2_codec` decode, the dense presence `get_bit` gather, `overlap_range` search, + memcpy/alloc. (Python frames will be opaque — that's expected; py-spy covered them.) + +**Deliverable E1:** a table per (backend × cohort) of `{Python %, native %, wall-clock}` + +the top ~10 Rust symbols for the SVAR2 path. **Decision output:** classify the SVAR2 gap as +(A) Python-adapter overhead → Task B Dataset wiring likely erases it; (B) Rust hot-path / +layout → Task B must include a kernel/layout fix; or (C) mix (quantify). + +### E2 — Same-cohort sample sweep (de-confound "scales with S") + +Hold the **dataset fixed** and vary sample count S. Pick the **somatic** cohort (widest +range, up to 16007). Subsample the filtered BCF at **S ∈ {1000, 2000, 4000, 8000, 16007}** +(`bcftools view -S --force-samples`), and for each S build **both** `.svar` and +`.svar2` (reuse `build_stores.py`; SLURM). Benchmark hap latency (warm, median N=5, same 3 +regions, all S samples) for both backends at each S. + +> Note the mild caveat: subsampling drops sites monomorphic in the subset, so variant count +> shrinks slightly with S; the AF *spectrum* is approximately preserved. Record per-S +> variant counts so the curves are interpretable. + +**Deliverable E2:** hap-latency-vs-S curves for both backends on one cohort, with slopes. +**Decision output:** confirm/refute that SVAR2 latency is less S-sensitive than SVAR1 — +the claim the MVP notes explicitly declined to make. + +### E3 — Dense-access layout probe (conditional on E1 showing dense gather hot) + +If E1 attributes significant Rust time to the dense presence gather: measure SVAR2 hap +latency vs **(a) region width** and **(b) `n_dense_variants`** (contig-wide dense count). +Optionally a Rust microbench of the `get_bit`-per-column gather vs a word-parallel variant. +**Deliverable E3:** evidence for/against the "contig-wide stride + bit-by-bit read" cost, +and a rough size of the win from a region-local / word-parallel presence layout — input to +how much layout work Task B should carry. + +### E4 — Conversion thread-allocation profiling (build side) + +1. **Confirm the bottleneck.** Profile a single-contig `run_conversion_pipeline` on a + compute node: py-spy `--native` (or perf on the Rust threads) to quantify time in + **htslib read/decompress** vs **encode** vs **Phase-2 merge**. (genoray already exposes a + sampler via `GENORAY_SAMPLE_INTERVAL` — capture its channel-fill / per-thread CPU% + alongside.) +2. **Sweep the split.** For a fixed single-contig input, vary the thread policy — htslib + decompression threads vs executor/writer threads (and total cores) — and measure build + wall-clock. Find the split minimizing single-contig time. + +**Deliverable E4:** build wall-clock vs thread-split table + a concrete recommended policy +for few-contig jobs, feeding +`genoray:docs/roadmap/architecture.md` → Open questions → *read-bound conversion / thread +allocation*. + +## Deliverables & out of scope + +- **Deliverable:** a notes file + `docs/superpowers/notes/2026-07-XX-svar2-profiling-results.md` with the E1–E4 tables and + two concrete recommendations: (1) where SVAR2 query latency actually goes (→ what Task B + must include), and (2) the conversion thread-split policy. Profiling driver scripts under + `tmp/svar2_mvp/`. +- **Out of scope (deliberately):** implementing Task B (Dataset wiring), any dense-layout + change, and the conversion thread rebalance. This spec produces the numbers that decide + *whether and how* to do those. + +## Suggested execution + +Small enough for a single focused session; a step-by-step plan can be derived from E1–E4 and +run via superpowers:subagent-driven-development. Start with **E1** (highest decision value: +it tells us if the whole latency story is just un-wired Python) and **E4** (independent, +build-side) — they need no new stores. E2 needs the SLURM sub-cohort builds; E3 is +conditional on E1. diff --git a/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md b/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md new file mode 100644 index 00000000..416659d4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-04-svar2-gvl-readbound-wiring-design.md @@ -0,0 +1,233 @@ +# SVAR2 gvl Read-Bound Dataset Wiring — Design Spec + +> **Purpose:** wire the SVAR2 format into `gvl.write()` and `gvl.Dataset.__getitem__` so all +> four output modes (haplotypes, tracks, variants, variant-windows) reconstruct **in Rust** off a +> write-time offsets cache, with **no interval-search tree rebuild and no dense-union rebuild at +> read**. This supersedes the gvl-side (Components B/C) of +> `2026-07-03-svar2-dataset-wiring-design.md`, now that genoray's search/gather split has shipped, +> and elects the deferred "fully read-bound" follow-up. + +**Date:** 2026-07-04 · **Status:** design, not yet implemented · **Ship:** **local-only, not +shipped** (genoray path-dep + local wheel; no crates.io / PyPI release in this spec). + +## Background & what changed + +The prior spec (`2026-07-03-svar2-dataset-wiring-design.md`) split genoray's fused `overlap_batch` +into search-only `find_ranges` + tree-free `gather_ranges` + fused `read_ranges`. **That split is +shipped** (merged to genoray `svar-2`): the interval-search trees are built once at write and not +rebuilt at read. Two things drive this follow-up spec: + +1. **The read path should be all-Rust.** `gvl.Dataset.__getitem__`'s hot loop is a single Rust FFI + call for SVAR1. SVAR2 reconstruction must match that — reconstruct in Rust, **not** via the + Python `SparseVar2.gather_ranges` API, and with no per-read numpy round-trip. +2. **The shipped `gather_ranges` still rebuilds a per-read dense union.** It merges the two on-disk + dense class tables (`dense/snp` 2-bit, `dense/indel` u32) into a transient position-sorted + `DenseUnion` **over the entire contig on every read** (genoray `src/query.rs` `dense_union()`) — + the O(N_contig) residual the prior spec flagged as "presence-bit cost at read." At 16k-sample + scale this is real. This spec eliminates it with a **per-class read-bound gather**. + +The `.svar2` dense store is **already** split and 2-bit-packed on disk (`src/dense.rs` +`DENSE_REGISTRY`: `dense/snp` `pack_snp: true`, `dense/indel` u32) — so **no genoray writer/format +change is needed**. The change is a new *gather* that slices those split tables' per-region windows +directly instead of unioning them. + +## Global constraints + +- **Byte-identical parity contract.** For any `contig, starts, ends, samples`: + `reconstruct(read-bound BatchResult)` ≡ `reconstruct(gather_ranges(find_ranges(...)))` (the shipped + union path) ≡ `reconstruct(overlap_batch(...))` ≡ genoray `decode` oracle — field-for-field / + byte-for-byte, for every output mode. +- **Additive.** The shipped union-based `find_ranges`/`gather_ranges`/`read_ranges`/`overlap_batch` + stay byte-unchanged and serve as the parity oracle and live-query API. The SVAR1 gvl path is + byte-unchanged (full SVAR1 regression green). Follows the rust-migration byte-identical parity + contract and the numba-oracle-bug policy. +- **Write caches only the dataset's samples.** `gvl.write` already selects which samples enter a + dataset (requested samples, plus any track-overlap resolution); the cache is sized to that + selection `S'`, not the full `.svar2` cohort — mirroring `_write_from_svar`. +- **Local-only.** gvl links `genoray_core` as a **path** dependency (`default-features = false`); + the Python genoray dep stays a **local wheel**. No crates.io/PyPI release. The path-dep and the + wheel MUST be built from the same genoray commit (the `RangesBundle` field layout is the contract). + +## Architecture — two deliverables + +**(A) genoray** — a second genoray PR, additive to the shipped split: +1. A **`conversion` cargo feature** (default-on) gating the htslib-tainted modules so the query core + builds htslib-free. +2. A **per-class read-bound path**: `find_ranges` emits per-class dense ranges; a read-bound gather + returns a **split-dense `BatchResult`** without building the contig-wide union. + +**(B) gvl** — the write cache + all-Rust read wiring targeting (A). + +Linkage: gvl adds `genoray_core = { path = "../genoray", default-features = false }` to `Cargo.toml` +(alongside the existing `svar2-codec` path-dep). Only `src/vcf_reader.rs` references `rust-htslib`, +so `default-features = false` yields a query-only, htslib-free core — gvl's build stays +toolchain-light. gvl's Rust opens `genoray_core::query::ContigReader` and calls the read-bound +gather directly; the Python `gather_ranges` API is **not** on gvl's read path. + +## Component A — genoray query-only feature + read-bound gather (dependency PR) + +### A1. `conversion` cargo feature (query-only build) +- `Cargo.toml`: make `rust-htslib` **optional**; add `[features] conversion = ["rust-htslib", ...]`, + `default = ["conversion", "extension-module"]` (keep `extension-module` behavior for the wheel). +- `src/lib.rs`: `#[cfg(feature = "conversion")]` on `vcf_reader` and its conversion-only dependents + (`writer`, `orchestrator`, `normalize` pipeline, and the `py_*` conversion entry points — the set + that transitively needs htslib). The query core (`query`, `search`, `spine`, `bits`, `nrvk`, + `rvk`, `layout`, `dense`, `types`, `py_query*`) builds with `default-features = false`. +- **Verify** the query core compiles with `--no-default-features` and that the Python wheel (built + with defaults) is byte-behavior-unchanged. The full genoray test suite stays green. + +### A2. Per-class read-bound `find_ranges` + gather + split-dense `BatchResult` +- **`find_ranges`** additionally emits **`dense_snp_range (R,2)`** and **`dense_indel_range (R,2)`** + — each a per-region `[ds,de]` into the corresponding on-disk dense table, computed by a per-class + overlap search (a `SearchTree` per class per region, at **write**, cold). These join the existing + `vk_snp_range`, `vk_indel_range` (per-hap) — the **4 offset arrays**. (The shipped union + `dense_range` may remain for the oracle path; the read-bound path uses the two per-class ranges.) +- **Read-bound gather** (`gather_ranges_readbound` or a bundle mode — genoray maintainer's naming): + slices `dense/snp[ds..de]` (2-bit) and `dense/indel[ds..de]` (u32) directly, applies the + per-element `q_start < v_end` left-overlap re-check, and computes **per-class presence bits** + (`DenseView::carried(hap, col)` over each class's window). Returns a **read-bound `BatchResult`**: + the var_key channel as today (snp+indel merged per hap via `merge_keys`, cheap over a small window) + **plus split `dense_snp` and `dense_indel` channels** (positions, keys, presence bits, and range), + **never** the contig-wide union. Exact channel factoring is a genoray-internal detail; gvl consumes + whatever the read-bound `BatchResult` exposes. +- **Parity:** a genoray test asserts `read-bound BatchResult` reconstructs identically to the shipped + union `BatchResult` and to the `decode` oracle (SNP/INS/DEL × samples × ploids, plus real chr21). + +## Component B — gvl write (`_write.py`, `_write_from_svar2`, `_svar2_link.py`) + +Mirror the SVAR1 write path (`_write_from_svar`, `_write.py:961`): +- **Detect `.svar2`.** Add a `.svar2` arm to variant-source path coercion (`_write.py:~217`) and a + `SparseVar2` arm to genotype-writing dispatch (`_write.py:~315`), alongside the `.svar`/`SparseVar` + branches. +- **`_write_from_svar2(path, bed, svar2, samples, extend_to_length)`.** Per contig, call + `svar2.find_ranges(c, df["chromStart"], df["chromEnd"], samples=samples, out=memmaps)` — for the + **dataset's samples only** — streaming the 4 offset arrays plus `region_starts`/`sample_cols` into + memmaps under `/genotypes/svar2_ranges/`. Write `svar2_meta.json` (shapes/dtypes) and a + `Svar2Link` into `metadata.json`; set `metadata["ploidy"] = svar2.ploidy`. +- **`_svar2_link.py`** mirrors `_svar_link.py`: a `Svar2Link` pydantic model + (`relative_path`, `absolute_path`, `fingerprint`) + `_resolve_svar2(gvl_path, link, override)` + + `_verify_fingerprint`. Fingerprint the `.svar2` on stable identity (n_variants from its index + a + canonical store-file byte count), analogous to `SvarFingerprint`. +- **Reject unsupported variants** (symbolic/breakend) as SVAR1 does; upstream `-V other,bnd` / + genoray M13 already filters these in the MVP build scripts. + +## Component C — gvl read (all Rust; `_open.py`, `_impl.py`/`_query.py`, `_haps.py`, kernels) + +- **`Svar2Store` (gvl pyclass).** Wraps a `genoray_core::query::ContigReader` per contig, opened + **once** at `Dataset.open` from the resolved `.svar2` path (the analog of SVAR1's once-built + `ffi_static` global table). Holds the memmapped cached 4-array ranges bundle. `Dataset.open` + resolves + fingerprints the `Svar2Link` (with a `svar2=` override paralleling `svar=`). +- **One FFI call per read.** `Dataset.__getitem__` → `_query.py` → a gvl Rust pyfunction that, for the + requested `(region, sample)` block: reconstructs a `genoray_core::RangesBundle` from the cached + memmap slice → calls the genoray **read-bound gather** → gets the split-dense `BatchResult` → + feeds the reconstruct core **in Rust** (no numpy round-trip, no Python `gather_ranges`). +- **Reconstruct kernel (read-bound variant).** A read-bound form of + `reconstruct_haplotypes_from_svar2` consumes the split-dense `BatchResult` and **merges + `var_key ⋈ dense_snp ⋈ dense_indel` by position** during assembly (extending the current + two-source splice), decoding keys inline via `svar2-codec`. `LongAlleleReader` lookups resolve + through the `ContigReader`'s LUT (no `lut_bytes`/`lut_off` marshalling). Tracks route + `shift_and_realign_tracks_from_svar2` off the **same** `BatchResult`. +- **Dispatch discriminant.** A dataset carries a source discriminant (`svar2_link` present) selecting + the SVAR2 reconstructor over the SVAR1 one; retires `TODO(svar2-dataset-dispatch)` in + `_svar2_source.py` (its live `overlap_batch` marshalling is removed for the cached path). + +## Output modes — all four, all Rust (Phase 1) + +All four modes read the **same** cached ranges and the **same** read-bound `BatchResult`; they differ +only in the final assembly kernel: +- **Haplotypes** (`with_seqs("haplotypes")` / default) — `reconstruct_haplotypes_from_svar2` + (read-bound). +- **Tracks** (`with_tracks`) — `shift_and_realign_tracks_from_svar2` (needs only `ilen`/`deletion_len`, + no alleles). +- **Variants** and **variant-windows** (`with_seqs("variants"/"variant-windows")`) — a gvl Rust + kernel decodes the per-hap overlapping keys from the read-bound `BatchResult` into `RaggedVariants` + (via `svar2-codec` `decode_key`: `Inline`/`PureDel`/`Lookup`, LUT via the `ContigReader`), mirroring + genoray's `decode_hap`. **No** Python `gather_ranges`/`decode` path. Same static-table Rust route + gvl already uses for SVAR1 variants. + +## Cache format (`.gvl/genotypes/svar2_ranges/`) + +The 4 offset arrays + 2 index vectors from `find_ranges(samples=dataset)`, as in-place memmaps +(`out=`), region/sample-sharded to the dataset layout; `svar2_meta.json` records shapes/dtypes +(mirrors SVAR1's `svar_meta.json`); `Svar2Link` → shared `.svar2`. + +``` +vk_snp_range (R, S', P, 2) -> vk_snp packed positions/keys (2-bit) +vk_indel_range (R, S', P, 2) -> vk_indel packed positions/keys (u32) +dense_snp_range (R, 2) -> dense/snp window (2-bit) +dense_indel_range (R, 2) -> dense/indel window (u32) +region_starts (R,) -> q_start per region (left-overlap re-check) +sample_cols (S',) -> selected slot -> original sample index +``` +**Size:** O(offsets) — same order as SVAR1's `offsets.npy`. The bulk (var_key + dense positions/keys, +genotype bitmatrices, LUT) stays in the compressed `.svar2`; SVAR2's on-disk size advantage +(1.46–5.67× smaller than `.svar`, per the MVP benchmark) is preserved. + +## Data flow + +- **WRITE:** `bed + .svar2` → per contig `find_ranges(..., samples=dataset, out=memmaps)` (search + once, per-class) → 4-array ranges cache + `Svar2Link` + `svar2_meta.json`. +- **READ:** `dataset[regions, samples]` → slice cached 4-array block → **read-bound gather** + (tree-free, union-free, per-class windows only) → split-dense `BatchResult` → Rust reconstruct + (haps / tracks / variants / variant-windows). **No interval search and no contig-wide union at + read.** + +## Parity & testing + +- **Byte-identical:** read-bound reconstruct ≡ shipped union reconstruct ≡ `decode` oracle, for all + four output modes, on the M6b matrix (SNP/INS/DEL × samples × ploids) + real chr21 germline & + somatic stores. Track re-alignment matched the same way. +- **Additive guarantee:** SVAR1 path byte-unchanged — `pixi run -e dev pytest tests -q` + + `cargo test`. genoray: query-only build compiles; full genoray suite green; the shipped union path + unchanged. +- **Perf verification (same-session, shared-node caveat):** a warm SVAR2 `Dataset` read shows + **neither** `SearchTree::build` **nor** the dense-union rebuild — the perf DSO split flips from + ~80% genoray to gvl-kernel-bound, like SVAR1. Report as a relative before/after within one + allocation (absolute wall-clock is not comparable across allocations on shared Carter nodes). +- **Docs/roadmaps:** update the genoray roadmap (read-bound per-class gather; conversion feature) and + `docs/roadmaps/rust-migration.md`; update user-facing docs for `.svar2` as a `write` variant source + — `skills/genvarloader/SKILL.md`, `docs/source/{api.md,write.md,format.md,faq.md}`, `README.md` + (docs-audit + skill-maintenance gates), and keep `api.md` in sync with any new `__all__` symbols. + +## Benchmark — relocate & re-run + +- **Relocate:** `mv /carter/users/dlaub/repos/for_loukik/svar2_mvp /carter/users/dlaub/projects/svar2_mvp` + (4.1 GB, not a git repo — plain move; update any absolute paths in `build_source.sh` / + benchmark driver). +- **Re-run** the SVAR1-vs-SVAR2 `gvl.Dataset.__getitem__` benchmark on chr21 germline (3202) + + somatic (16007) after wiring: latency (same-session before/after) + store size. **Success:** the + SVAR2 read is gvl-kernel-bound (no `SearchTree::build`, no union rebuild), and SVAR2's store-size + advantage holds. + +## Out of scope + +- **Annotated** output mode (extends from the same cache later). +- Shipping / release (crates.io / PyPI) — local-only path-dep + wheel; the release gate is a later, + separate step. +- Any change to the on-disk `.svar2` format itself (already split + 2-bit-packed; genoray-owned). +- Profiling experiments **E2** (same-cohort sample sweep) and **E4** (conversion build-thread + allocation) — parked; **E3** (dense-access probe) dropped (gvl dense-gather ≈1%). + +## Deliverables & sequencing + +1. **genoray PR** — `conversion` query-only feature + per-class read-bound `find_ranges`/gather + + split-dense `BatchResult` + parity (vs union & `decode`); build local wheel + crate. +2. **gvl PR** — write: `_write_from_svar2` + `_svar2_link.py` (4-array cache, dataset samples); + read: `Svar2Store` + read-bound reconstruct kernels for all four output modes + + `Dataset.open`/`__getitem__` dispatch + `_svar2_source.py` retirement; `genoray_core` path-dep. +3. **Relocate + re-run** the MVP benchmark. +4. **Docs/roadmap** updates. + +## Open questions / risks + +- **read-bound `BatchResult` channel factoring** (keep var_key merged + dense split, vs. fully + 4-channel unmerged) is a genoray-internal detail settled in the genoray PR; gvl consumes the + payload the read-bound gather exposes, staying agnostic. +- **wheel ↔ path-dep sync:** the Python wheel that writes the cache and the Rust path-dep that reads + it must be the same genoray commit; document the local-dev sync discipline (a fingerprint/version + check at `Dataset.open` is a possible guard). +- **format version:** adding `svar2_link` to `metadata.json` is additive; confirm + `_check_dataset_format_version` tolerates it and any bump is handled by `_migrate.py`. +- **`conversion` feature partition:** confirm the exact module set that transitively needs + `rust-htslib` (only `vcf_reader.rs` references it directly; verify `writer`/`orchestrator`/ + `normalize` and their py-entry points are the complete gated set). diff --git a/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md new file mode 100644 index 00000000..f9986501 --- /dev/null +++ b/docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md @@ -0,0 +1,167 @@ +# Profile & optimize the SVAR2 read-bound `Dataset.__getitem__` path + +> **Status:** design approved · **Date:** 2026-07-05 · **Repos/branches:** +> GenVarLoader `svar2-m6b-kernel` (PR #266, draft) + genoray `svar-2` (@ `aaf44fd`) +> +> Big picture: genoray `docs/roadmap/svar-2.md` (milestones M6b/M6d/M6e — the +> read-bound gather this work profiles). + +## 1. Motivation + +`Dataset.__getitem__` for a `.svar2`-backed dataset now dispatches through +`Svar2Haps` (`python/genvarloader/_dataset/_svar2_haps.py`) → the +`*_from_svar2_readbound` FFI kernels → genoray_core `gather_haps_readbound`. This +path builds **zero** interval-search trees and **zero** dense-union per read (the +structural win M6d/M6e were built for). + +The only profiling artifacts on disk (`tmp/svar2_mvp/prof_out/e1`) measure the +**retired union oracle** — `SparseVar2Source.reconstruct` → genoray +`overlap_batch` → 68% `SearchTree::build`. That path is no longer on any live +read. **We are profiling the real read-bound path effectively from scratch.** + +The PR bench also noted the small-workload latency is dominated by "per-read +Python/numpy cache-slicing overhead," never attributed to specific functions. +This effort attributes it and removes what is removable. + +Goal: identify the hottest functions in the live read-bound `Dataset.__getitem__` +path across all three supported modes, then optimize — Python by static +analysis/inspection, hot Rust by inspecting `cargo asm` — with parity preserved. + +## 2. Scope + +**In scope** — the live read path for the three supported output modes: + +- **haplotypes** — `Svar2Haps.get_haps_and_shifts` → `hap_diffs_from_svar2_readbound` + + `reconstruct_haplotypes_from_svar2_readbound`. +- **variants** — `Svar2Haps._reconstruct_variants` → `decode_variants_from_svar2_readbound`. +- **tracks** — `Svar2Haps.realign_track_block` → `intervals_to_tracks` + + `shift_and_realign_tracks_from_svar2_readbound`. + +Shared Rust spine (candidates for `cargo asm`): gvl-side +`svar2::split_to_flat` / `hap_diffs_svar2` / `reconstruct_haplotypes_from_svar2` +(`src/svar2/mod.rs`, `src/reconstruct.rs`, `src/ffi/mod.rs`); genoray-side +`query::gather_haps_readbound` / `spine::merge_keys` +(`genoray/src/query.rs`, `genoray/src/spine.rs`). + +**Out of scope:** the guarded-`NotImplementedError` modes (annotated, spliced, +`min_af`/`max_af`, in-kernel RC, `unphased_union`, variant-windows, +`max_jitter>0` variants — `unphased_union` and variant-windows have since been +*implemented*, see `2026-07-06-svar2-variant-windows-design.md`; the perf/fusion +work this doc scopes remains deferred for both); any on-disk **format** or +**public API** change; the union oracle (`SparseVar2Source`, `overlap_batch`) +except as the parity oracle; `gvl.write` (the write-time ranges cache producer). + +## 3. Landing targets (two repos) + +| Change site | Repo / branch | Rebuild to profile | +| --- | --- | --- | +| gvl Rust kernels (`src/svar2`, `src/reconstruct`, `src/ffi`) + Python (`_svar2_haps.py`) | GenVarLoader `svar2-m6b-kernel` (PR #266) | `maturin develop --release` | +| genoray Rust kernels (`gather_haps_readbound`, `merge_keys`) | genoray `svar-2` | gvl `maturin develop --release` rebuilds the `genoray_core` **crate path-dep** automatically | + +**Wheel caveat:** the Python `genoray` package is a pre-built cp310 wheel +(`pixi.toml`). It only needs rebuilding if a change touches genoray's **Python +API** (e.g. a `find_ranges` dict key). Pure hot-path kernel edits consumed by gvl +through the `genoray_core` crate do **not** need a genoray wheel rebuild. + +## 4. Tooling — forced substitution for py-spy + +Empirically verified on the Carter node this runs on: + +- **py-spy is unusable** — `ptrace_scope=2` + no sudo → "Permission Denied" for + `dump`/`record`, `--native` or not. +- **Python is 3.10** in `-e dev` → no `perf` trampoline (`-Xperf` is 3.12+), so + `perf` **cannot resolve Python frames** (Python stays a single opaque DSO). + +We recover both halves py-spy would have given from two ptrace-free tools: + +| Layer | Tool | Output | +| --- | --- | --- | +| Python fns ("python fns") | **cProfile** (stdlib) + **pyinstrument** (added to pixi deps for a low-overhead statistical wall-clock call-tree cross-check) | per-function cumulative/total time → the Python functions to inspect | +| Native/Rust ("total native %") | **perf** `record -g --call-graph fp -F 199`, run on `.pixi/envs/dev/bin/python` **directly** (not via `pixi run` — the launcher eats ~60% of samples), built with `-C force-frame-pointers=yes` | DSO split (native % = "total native %"), Rust symbol self-time, and the fp call-graph → the Rust fns to `cargo asm` | + +`pyinstrument` is added to `pixi.toml` deps (gvl, and genoray if it profits). + +## 5. Benchmark harness (rebuilt, real path) + +The existing `tmp/svar2_mvp` drivers profile the **wrong** (union) path and +reference a **stale** store path (`repos/for_loukik/...`). Stores actually live at +`/carter/users/dlaub/projects/svar2_mvp/{germline,somatic}.{svar,svar2}`; +reference FASTA `/carter/shared/data/gdc/resources/GRCh38.d1.vd1.fa` is present. + +Add a driver under `tmp/svar2_mvp/` that exercises the **live** +`Dataset.open(path, reference=REF, svar2=).with_seqs(mode)[regions, samples]` +warm loop for `mode ∈ {haplotypes, variants, tracks}`, over both cohorts +(germline 3202-smp, somatic 16007-smp), same fair workload as the PR bench +(fixed region set × all samples, warm caches, median of N). One code path per +capture (mirrors `prof_driver.py`) so cProfile/perf attribute cleanly. A tracks +run needs a BigWig or table track attached to the dataset; haplotypes/variants +reuse the PR bench setup. + +## 6. Optimization method (measure → confirm → fix → re-measure) + +Profile first; fix only what ranks hot. Static inspection already surfaces +concrete candidates to **confirm against the profile before touching**: + +- **Redundant double gather (Python/FFI, high confidence):** + `get_haps_and_shifts` calls `hap_diffs_from_svar2_readbound` *and then* + `reconstruct_haplotypes_from_svar2_readbound` per contig group — **both** run + the full `gather_haps_readbound` + `split_to_flat` internally, so genoray's + gather + AoS→SoA marshalling executes **twice per haplotype/track read**. + Candidate fix: gather once (fuse diffs into the reconstruct gather, or return + diffs from a single kernel), so the read draws shifts and reconstructs off one + gather. +- **Python FFI-shaping (`_svar2_haps.py`):** per-contig-group Python loops, + repeated `ascontiguousarray` copies in `_gather_inputs`, the + `_ragged_arange_gather` permutation passes — vectorize / drop copies where the + profile justifies. +- **Rust hot fns via `cargo asm`:** likely `svar2::split_to_flat` (AoS→SoA copy), + `svar2::merge_hap`/`decode_alt`, `gather_haps_readbound`, `merge_keys`, + `hap_diffs_svar2`, and the `reconstruct_haplotypes_from_svar2` inner kernel. + `cargo asm` targets: bounds-check elision, autovectorization of copy/merge + loops, alloc churn (`SpecFromIter`/`_int_malloc` were hot even in the old run). + Because these functions are independent, the `cargo asm` phase **fans out one + subagent per hot function in parallel** (Sonnet implementers, each in its own + git worktree so same-file edits don't clobber), and **each fix carries its own + per-function parity test** (a focused `cargo test` asserting byte-identical + output on representative inputs) alongside the instruction-count delta. The + branch owner then merges the worktree fixes sequentially behind the full-tree + parity gate, dropping any that regress or fail parity. + +## 7. Measurement discipline & parity gate + +Per hard-won project lessons (shared-node noise): + +- **No cross-session absolute wall-clock claims.** Gate each optimization on + **same-session before/after** plus a **deterministic `perf stat -e + instructions,cycles`** delta (instruction count is noise-free). Record the + instruction-count delta per change. +- **Parity is a hard gate.** The read-bound kernels are byte-identical to the + union oracle. After every change: `pixi run -e dev pytest tests -q` (full tree + — the svar2 suite is 31/31, plus the parity oracle), and for Rust changes + `maturin develop --release` **first** (pytest imports the stale `.so` + otherwise), plus `cargo test` on both repos. Any divergence blocks the change. +- The two documented intentional non-identities (pure-DEL ALT `b""`; SVAR1 + `max_ends` tie under-extension) are pre-existing and untouched. + +## 8. Deliverables + +1. A profiling report (cProfile + pyinstrument + perf DSO/symbol/callgraph + tables, per mode × cohort) under `tmp/svar2_mvp/prof_out/`. +2. Confirmed optimizations implemented: gvl-side on `svar2-m6b-kernel` (#266), + genoray-side on `svar-2` — each parity-gated, each with a recorded + same-session instruction-count delta. The `cargo asm` phase is executed as a + **parallel subagent fan-out** (one worktree-isolated Sonnet subagent per hot + function, each with a per-function parity test), merged sequentially behind + the full-tree parity gate. +3. `pyinstrument` added to `pixi.toml`. +4. No format/API/doc-surface changes (read-path internals only), so the + skill/api.md/docs gates do not apply; the genoray roadmap needs no milestone + flip (this is perf work under shipped M6b/M6d/M6e, noted in passing if a + kernel signature changes). + +## 9. Open questions + +- Whether the double-gather fix needs a **new fused kernel signature** (a genoray + `svar-2` API touch → genoray wheel rebuild) or can be done gvl-side by having + the reconstruct kernel also return diffs (no genoray API change). Resolve after + the profile confirms the gather is actually the dominant cost. diff --git a/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md new file mode 100644 index 00000000..afc4aafb --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-svar2-variant-windows-design.md @@ -0,0 +1,310 @@ +# Design: variant-windows mode for svar2-backed gvl + +> **Status:** design approved · **Date:** 2026-07-06 · **Branch:** `svar2-m6b-kernel` (PR #266, draft) +> +> Related: `docs/superpowers/specs/2026-06-25-target7-variant-windows-rust-assembly-design.md` +> (the `assemble_variant_buffers` kernel this reuses), and +> `docs/superpowers/specs/2026-07-05-svar2-readbound-getitem-perf-design.md` +> (which lists variant-windows as a guarded, out-of-scope svar2 mode — this +> design removes that guard). + +## 1. Problem + +`Dataset.open(path, reference=REF, svar2=).with_seqs("variant-windows", +VarWindowOpt(...))[regions, samples]` currently raises `NotImplementedError`. +`Svar2Haps.__call__` (`python/genvarloader/_dataset/_svar2_haps.py`, ~lines +282–285) guards `_FlatVariantWindows`: + +```python +if issubclass(self.kind, _FlatVariantWindows): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('variant-windows') yet." + ) +``` + +Every other piece needed already exists and is parity-validated: + +- The svar2 **`variants` decode** (`decode_variants_from_svar2_readbound`) is + byte-identical to the svar2 decode oracle and to SVAR1 for the variant SET + (`tests/dataset/test_svar2_dataset.py::test_svar2_variants_positions_match_svar1`, + `::test_svar2_variants_match_svar2_oracle`). +- The **window-assembly Rust kernel** `assemble_variant_buffers` + (`src/variants/windows.rs`, shimmed by `_assemble_variant_buffers_rust` in + `python/genvarloader/_dataset/_flat_variants.py`) is validated end-to-end for + SVAR1 and under both rust/numba backends (target-7 work). + +The **only** thing svar2 variant-windows adds over the working svar2 `variants` +decode is the window-assembly step (reference fetch + flank + tokenize). So the +implementation composes two already-validated kernels; **no new Rust**. + +## 2. Scope + +**In scope** — the live `variant-windows` read path for svar2 datasets: + +- `ref="window"` with `alt ∈ {"window", "allele"}`. +- Single-contig (fast path) and multi-contig (contig-group stitch). +- Empty (region, sample, ploid) groups with `dummy_variant` fill. +- **`unphased_union`** for **both** svar2 decode modes (`variants` and + `variant-windows`) — the ploidy-1 union view (issue #222). The same offset + fold serves both, so `variants` mode gets it too (avoids a confusing + "union works for windows but not plain variants" gap). SVAR1 supports union + for these two modes; svar2 now matches. Haplotypes/annotated + union remain + blocked at `with_seqs` (a union of phased sequences is ill-defined) — that + guard is in `_impl.py`, unchanged. + +**Out of scope** (existing guards stay in force; each must raise, not silently +diverge): + +- `ref="allele"` — **already** rejected upstream in `with_seqs` + (`_impl.py` ~line 720: raises `ValueError` when `window_opt.ref == "allele"` + and `self._seqs.variants.ref is None`, which is always true for `Svar2Haps` — + its dummy `_Variants` carries `ref=None`, and svar2 stores no REF allele + bytes). No new guard needed; a test pins it. +- `max_jitter > 0` at write or `jitter > 0` at read — the read-bound decode has + no right-clip, so a padded/slid window over-includes variants past the + (unpadded) read window. Same issue and same guard as svar2 `variants`. +- `min_af`/`max_af`, spliced, annotated, in-kernel reverse-complement — all + already guarded (`_guard_unsupported` + `__call__`). (`unphased_union` is now + **in scope** — see above — so it is removed from `_guard_unsupported`; the + haplotypes/annotated + union block stays in `_impl.py`.) +- No on-disk **format** change, no **public API** signature change + (`with_seqs`/`VarWindowOpt` are unchanged; this only makes an existing kind + reachable for svar2). +- A **fused** single-call svar2 windows kernel (decode+assemble in one FFI + crossing) is a perf optimization, deferred — see §7. + +## 3. Architecture — compose two validated kernels per contig group + +Add `Svar2Haps._reconstruct_variant_windows(idx, regions) -> _FlatVariantWindows`, +structured exactly like the existing `Svar2Haps._reconstruct_variants`: group +queries by contig (store readers are per-contig), process each group, then +stitch back to global `(b, P)` row order with the single-group identity fast +path and the multi-group inverse-row-permutation. + +The window config fields (`token_lut`, `token_dtype`, `window_opt`, +`unknown_token`, `flank_length`, `dummy_variant`) are set on the `Svar2Haps` +instance by `with_seqs("variant-windows", ...)` via `replace(self._seqs, ...)` +— `Svar2Haps` is a dataclass subclass of `Haps`, so this already works +(`_impl.py` ~lines 729–744). No change to `Svar2Haps.from_path` construction is +required for the config to arrive. + +### Per contig group `(ci, qsel)` + +1. **Decode** (existing): cache-slice via `_gather_inputs`, then + `decode_variants_from_svar2_readbound(...)` → + `(pos, ilen, alt_bytes, str_off, var_off)`. These per-variant arrays are the + already-gathered analog of SVAR1's global arrays. + +1a. **unphased_union fold** (when `self.unphased_union`): the group's `var_off` + has `len(qsel) * P + 1` entries in row = `q*P + p` order (the P haps of a + query are contiguous). Fold to `len(qsel) + 1` by keeping every P-th offset: + `row_offsets = np.ascontiguousarray(var_off[::P])`, and set the group's + effective ploidy `p_eff = 1`. This concatenates hap-0's variants then + hap-1's per query — no sort, no dedup, `v_idxs`/`pos`/`ilen`/`alt_bytes` + untouched. Identical to SVAR1 `get_variants_flat` (lines ~842–845). When not + union, `row_offsets = var_off` and `p_eff = P`. The window/scalar output + `shape` uses `p_eff` (`(b, p_eff, None[, None])`) and the group stitch uses + `_inverse_row_perm(cat_query_order, b, p_eff)`. + +2. **Assemble windows** (existing kernel, identity gather): call + `_assemble_variant_buffers_rust` with: + + | kernel arg | svar2 value | + |---|---| + | `mode` | `1` (windows) | + | `v_idxs` | `np.arange(n_var, dtype=np.int32)` (identity — data is pre-gathered; spans ALL variants incl. both haps, unaffected by the union fold) | + | `row_offsets` | the (possibly folded) `row_offsets` from step 1a | + | `alt_global`, `alt_off_global` | `alt_bytes`, `str_off` | + | `ref_global`, `ref_off_global` | `None`, `None` (`ref="allele"` blocked upstream) | + | `want_ref_bytes`, `want_flank` | `False`, `False` | + | `ref_mode` | `1` (window) | + | `alt_mode` | `1` if `window_opt.alt == "window"` else `2` | + | `flank_len` | `window_opt.flank_length` | + | `lut` | `self.token_lut` | + | `v_contigs` | `np.zeros(n_var, np.int32)` (single-contig slice) | + | `v_starts`, `ilens` | `pos`, `ilen` | + | `reference`, `ref_offsets` | `_ref_for_contig(ci)` slice + `[0, len]` | + | `pad_char` | `self.reference.pad_char` | + + Coordinate note: `pos` is 0-based within-contig (proven `==` SVAR1 `start`), + and the per-group reference is the single-contig slice starting at contig + position 0, so the kernel's `contig_offset(=0) + start` indexing and its + `[start-L, end+L)` OOB-padded read are correct — identical math to the SVAR1 + path, just fed pre-gathered arrays. + + Returned `bufs` keys: `ref_window` (always, since `ref="window"`) and either + `alt_window` (alt="window") or `alt` (alt="allele"), each `(data, seq_off)`. + +3. **Wrap** each `(data, seq_off)` into a per-group `_FlatWindow` + (`data`, `seq_offsets=seq_off`, `var_offsets=var_off`, `shape=(b,P,None,None)` + at group scope). + +### Stitch groups → global `(b, P)` order + +- **Single group** (the common single-contig read): grouped order already equals + global `(b, P)` order → return directly, no reorder, no concatenate. Mirrors + the `_reconstruct_variants` / `_assemble_haps` fast path. +- **Multiple groups**: compute `perm = _inverse_row_perm(cat_query_order, b, P)` + once, then + - scalar fields (`start`, `ilen`): reorder via `_ragged_arange_src(grouped_row_offsets, perm)` → `src`, index `pos_c[src]` / `ilen_c[src]` (exactly as `_reconstruct_variants` does for pos/ilen); + - each window token buffer: reorder via the existing + `_ragged_arange_gather_2level(token_data, grouped_row_offsets, grouped_seq_offsets, perm)` + → `(new_data, new_var_off, new_seq_off)`. The returned `new_var_off` equals + the reordered scalar `var_off_g`, so scalar and window offsets stay + consistent by construction. + +### Build the result + +- Scalar `fields`: `start` always; `ilen` when `"ilen" in self.var_fields` + (mirrors `get_variants_flat`; svar2 has no dosage/info/custom fields — + `available_var_fields = ["alt", "ilen", "start"]`). Each wrapped as `_Flat` + with the global `row_offsets` and `shape=(b, P, None)`. +- Assemble `_FlatVariantWindows(fields, ref_window=..., alt_window=... | alt=...)` + (set the present window fields per `window_opt`). +- If `self.dummy_variant is not None`, apply + `win.fill_empty_groups(self.dummy_variant, unk=self.unknown_token, flank_length=window_opt.flank_length)` + (the existing `_FlatVariantWindows.fill_empty_groups`). + +### `Svar2Haps.__call__` wiring + +Replace the NotImplementedError branch with: + +1. the **same jitter/max_jitter guard** already used for svar2 `variants` + (raise `NotImplementedError(... "right-clip" ...)` when + `self.max_jitter > 0 or jitter > 0`); +2. `return cast(_H, self._reconstruct_variant_windows(idx, regions))`. + +`_guard_unsupported(splice_plan)` is called first (as today), covering +splice/exonic/min_af/max_af (the `unphased_union` clause is **removed** from +`_guard_unsupported` — see below). `to_rc` is not applicable to variant-windows +(reference-oriented; RC intentionally unsupported for the kind). + +### `_reconstruct_variants` gets the same fold + +Because `unphased_union` is enabled for svar2 `variants` too, add step-1a's fold +to the existing `Svar2Haps._reconstruct_variants` as well: within each group, +`row_offsets = var_off[::P]` and `p_eff = 1` when `self.unphased_union` (else +unchanged), use `p_eff` for the `shape` and the stitch perm. The decoded +`pos`/`ilen`/`alt` data and the identity relationship are untouched — only the +per-row grouping changes. Remove the `unphased_union` clause from +`_guard_unsupported` so both decode modes reach the fold; the haplotypes/tracks +paths never set the flag (blocked at `with_seqs`), so removing the guard cannot +mis-route them. + +## 4. Parity strategy + +Both halves gated; a matched SVAR1 dataset cannot be the oracle for the alt side +because svar2 and SVAR1 encode a deletion's ALT differently (svar2 `""` vs SVAR1 +anchor base, e.g. `G` for `GTA>G`) — so `alt`/`alt_window` bytes legitimately +differ. Split the gate: + +1. **`ref_window` → byte-identical to a matched SVAR1 variant-windows dataset.** + `ref_window` is a pure reference read over the same decoded variant SET + (positions identical to SVAR1), so it must match exactly. Free, strong check + on the reference-read + tokenize half. Compare `data` and both offset levels + (`var_offsets`, `seq_offsets`) after `to_ragged()`. + +2. **`alt`/`alt_window` (and the full bundle) → independent numpy oracle.** + A small (~40-line) reference that consumes the *already-validated* svar2 + `variants` output (`ds2.with_seqs("variants")[:, :]`, i.e. the decoded + `start`/`ilen`/`alt`) and, per variant, reproduces the kernel exactly: + - `ends = start - min(ilen, 0) + 1`; + - fetch `reference[start-L : end+L)` with absolute-coordinate `pad_char` OOB + padding; + - `alt="window"`: `flank5 (first L) · alt_bytes · flank3 (last L)`; + `alt="allele"`: bare `alt_bytes`; + - tokenize via the same LUT (built from `window_opt.token_alphabet` / + `unknown_token`), out-of-alphabet bytes → `unknown_token`. + Assert the oracle's `(data, seq_offsets, var_offsets)` equals the svar2 + `_FlatVariantWindows` window buffers. This is the true correctness gate for + the alt side and independently re-derives `ref_window` too. + +3. **Multi-contig stitch** — reuse the interleaved chr2/chr1 fixture + (`_src2`/`svar2_fixture2`) so the single-contig fast path is bypassed and the + inverse-row-perm reorder of both scalar and 2-level window buffers is + exercised; assert against both oracles above. + +4. **Empty-group / dummy fill** — a variant-free tail region (the existing bed's + `chr1 [20, 40)`) plus a `dummy_variant`; assert each empty `(b*p)` row gets + one all-`unknown_token` entry of the right width (`2L + len(dummy allele)` for + `alt_window`, `2L + len(dummy ref)` for `ref_window`, `len(dummy alt)` for + bare `alt`). Also assert the no-fill path is unchanged. + +5. **Guard contracts** — tests that `ref="allele"` raises `ValueError` at + `with_seqs`, and that `max_jitter>0`/`jitter>0` raises `NotImplementedError` + with the "right-clip" message for variant-windows (mirrors the existing + `test_svar2_variants_jitter_guard_raises`). + +6. **unphased_union** (both modes) — against a matched SVAR1 dataset with + `with_settings(unphased_union=True)`: + - `variants` mode: `start`/`ilen` byte-identical to SVAR1 union (the fold is + order-preserving and drops nothing, so hap-0-then-hap-1 concat matches); + ALT compared to the svar2 decode oracle folded the same way (SVAR1 ALT + differs for deletions, per §4.2). Ploidy axis folds 2→1 (`shape[-2] == 1`). + Also assert the union row equals hap-0's decode concatenated with hap-1's. + - `variant-windows` mode: `ref_window` byte-identical to SVAR1 union + `ref_window`; `alt`/`alt_window` vs the numpy oracle applied to the folded + svar2 decode. `to_ragged()` succeeds (offsets consistent post-fold). + - Multi-contig + union together (interleaved fixture) to lock the + `p_eff=1` stitch perm. + +Backends: run the parity tests under both `GVL_BACKEND` values where the shim +dispatches (as the existing svar2 suite does), since `_assemble_variant_buffers` +is registered with rust + numba. + +## 5. Testing & gates + +- `maturin develop --release` is **not** needed (no Rust change) — but the + branch may carry unbuilt Rust from prior work; run it once if the `.so` is + stale, per CLAUDE.md. +- `pixi run -e dev pytest tests/dataset/test_svar2_dataset.py -q` (new tests), + then the full svar2 suite `pixi run -e dev pytest tests -k svar2 -q`. +- Full tree before push: `pixi run -e dev pytest tests -q` (scoped runs skip + `tests/unit/`), and `pixi run -e dev cargo-test` (no Rust change, but cheap + insurance). +- Lint/format/typecheck: `ruff check python/ tests/ && ruff format python/ tests/ + && typecheck`. +- HPC gotcha: `--basetemp=$(pwd)/.pytest_tmp` so the write path's `os.link` + hardlink does not fail cross-device (Errno 18). + +## 6. Docs / skill / roadmap + +Public API surface is unchanged, but svar2's **supported-mode matrix** changes +(variant-windows becomes supported), so: + +- `skills/genvarloader/SKILL.md` — svar2 mode support / gotchas. +- `docs/source/*.md` where svar2 output-mode support is enumerated + (`dataset.md`, `faq.md` as applicable). +- The svar2 read-bound perf design's "out of scope" list — note variant-windows + is now implemented (feature complete; perf/fusion still deferred). +- No `api.md`/`__all__` change (no new public symbol). + +## 7. Deferred / future + +- **Fused svar2 windows kernel.** The two-call compose (decode then assemble, + per group) mirrors the existing svar2 variants/tracks split-kernel structure + and reuses validated code. A single fused + `assemble_variant_windows_from_svar2` kernel could cut one FFI crossing + + intermediate buffers, but that is a perf optimization to pursue only after + profiling the live path (consistent with the read-bound perf design's + measure-first discipline). Not in this work. +- `ref="allele"` for svar2 (would require the decode to also return REF bytes, + or reconstructing REF from `reference[start:start+ref_len]` where + `ref_len = len(alt) - ilen`; a genoray-touching or extra-work change, deferred). + +## 8. Files + +- **Edit** `python/genvarloader/_dataset/_svar2_haps.py`: + - new `_reconstruct_variant_windows(self, idx, regions)`; + - `__call__`: replace the NotImplementedError branch with the jitter guard + + dispatch; + - `_reconstruct_variants`: add the unphased_union fold (`p_eff`); + - `_guard_unsupported`: remove the `unphased_union` clause; + - imports: `_assemble_variant_buffers_rust`, `_FlatWindow` (from + `_flat_variants`), and reuse existing `_ragged_arange_src` / + `_ragged_arange_gather_2level` / `_inverse_row_perm`. +- **Edit** `tests/dataset/test_svar2_dataset.py`: ref_window-vs-SVAR1, + alt-vs-numpy-oracle, multi-contig, empty-group/dummy, `ref="allele"` guard, + jitter guard. Add a variant-windows bigwig/track combo only if trivial; + otherwise seqs-only. +- **Edit** docs/skill per §6. diff --git a/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md b/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md new file mode 100644 index 00000000..c570d28b --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-svar2-info-format-field-routing-design.md @@ -0,0 +1,210 @@ +# SVAR2 INFO/FORMAT field routing → `RaggedVariants` / `FlatVariants` + +**Date:** 2026-07-12 +**Worktree/branch:** `svar2-m6b-kernel` (draft PR #266) +**Status:** design approved; ready for implementation plan + +## Goal + +Route arbitrary scalar-numeric INFO/FORMAT fields stored in an SVAR2 (`.svar`) +store all the way through gvl's read-bound SVAR2 path into the `variants` +(`RaggedVariants`) and variant-windows (`_FlatVariantWindows`, the "flat" +variant output) outputs. Today gvl's SVAR2 path surfaces only `alt`/`start`/ +`ilen`; the field values genoray now persists (`SparseVar2.from_vcf(info_fields=, +format_fields=)` / `from_pgen`) are dropped. + +**Scope (approved):** both INFO and FORMAT fields; both `RaggedVariants` and +variant-windows outputs. Scalar-numeric only (`Integer`/`Float`, `Flag` for +INFO; `Number` `1` or `A`) — exactly what genoray's SVAR2 store can hold. + +## Background — where the seam is + +gvl's SVAR2 path does **not** call genoray's Python `SparseVar2.decode()`. It +uses gvl's own read-bound Rust kernel `decode_variants_from_svar2_readbound` +(`src/ffi/mod.rs:1330`), which links `genoray_core` as a crate, calls +`genoray_core::query::gather_haps_readbound(reader, &HapRanges) -> +BatchResultSplit`, and runs its own `var_key ⋈ dense_snp ⋈ dense_indel` +position-merge (`src/svar2/mod.rs:decode_variants_from_split`) over plain +`KeyRef { position, key }`. To attach a field value to a decoded variant we need +that variant's **source index** in the store, which plain `KeyRef` does not +carry. + +genoray (current main `acc59cb`) already exports everything needed to recover it +— **no genoray-side code change is required**: + +- `gather_haps_readbound_src` — same as `gather_haps_readbound` but populates + `BatchResultSplit.vk_src` (packed var_key provenance). The plain variant + leaves `vk_src` empty (it does not pay for provenance). +- `pack_vk_src` / `unpack_vk_src` / `VK_SRC_INDEL_BIT` — `vk_src[i]` → + `(is_indel, call_idx)`, where `call_idx` is the absolute call index into + `var_key/{snp,indel}`. +- `dense_abs_row(on_disk_range, out_range, i)` — recovers a dense variant's + absolute source row from the `HapRanges` on-disk dense range + the + `BatchResultSplit` output dense range. (No `dense_src` array in the split + path; it's pure arithmetic.) +- `FieldView` (+ `FieldValue`) with `value_at(i)` / `format_at(dense_row, + orig_sample)` / `bytes_at(i)`; `crate::layout::{ContigPaths, FieldSub}`, + `crate::field::StorageDtype`. `FieldSub::all()` order is `[VkSnp, VkIndel, + DenseSnp, DenseIndel]`. + +Reference implementation to mirror: genoray's own `gather_batch_fields` +(`src/py_query_decode.rs:156`), which does this gather for the whole-cohort +batch path. **The one difference in the read-bound path:** the split has +`n_samples == 1`, so a dense FORMAT lookup must stride by the real cohort sample +`HapRanges::orig_samples[q]`, i.e. `format_at(dense_row, orig_samples[q])` — +*not* the split's sample slot (always 0). + +## Prerequisite — genoray dependency (build a local wheel) + +The field-read API is **unreleased**: it is 87 commits past the `3.0.0` tag and +in no tag. The existing `genoray-3.0.0` wheel lacks it. Plan (approved): + +1. **Drop gvl's genoray version pin.** `pyproject.toml:14` `"genoray>=3,<4"` → + `"genoray"`. (Re-pin to a real release later, when genoray ships svar-2.) +2. **Build a genoray manylinux wheel from current main** (`acc59cb`, has the + field-read API): + ``` + cd /carter/users/dlaub/projects/genoray + pixi run --manifest-path ci/wheel/pixi.toml build # -> wheelhouse/*.whl + pixi run --manifest-path ci/wheel/pixi.toml repair # auditwheel -> dist/*.whl + ``` + The wheel reports version `2.15.0` (genoray's `pyproject` version); that's + fine once gvl's pin is dropped. Mind the NFS build gotchas + (`CARGO_TARGET_DIR=/tmp/...`). +3. **Point gvl's pixi pin at the new wheel** (`pixi.toml:110`), then + `pixi install` / re-solve. This also fixes the currently-unsolvable env + (`pyproject` `>=3,<4` vs the pinned `2.15.0` wheel). + +The Rust side already links the live genoray repo via `Cargo.toml` +`genoray_core = { path = ".../genoray", ... }`, so it sees the exports at build +time regardless of the Python wheel. + +## Rust design (`genvarloader` crate) + +Files: `src/ffi/mod.rs`, `src/svar2/mod.rs`, `src/svar2/store.rs`, `src/lib.rs` +(registration unchanged unless signature changes require it). + +1. **`decode_variants_from_svar2_readbound` (`src/ffi/mod.rs:1330`)** gains a + `fields: Vec<(String /*category*/, String /*name*/, String /*dtype*/)>` + param (empty ⇒ current behavior, zero overhead). Returns the existing 5-tuple + plus: + - `field_bufs: Vec>>` — one flat little-endian byte buffer + per requested field, length `n_var * itemsize`, in the same variant order + as `pos` (so it shares `var_off`). + - `field_itemsizes: Vec` — parallel; Python asserts + `itemsize == dtype.itemsize` (mirrors genoray `_svar2_decode.py:70`). +2. When `fields` is non-empty, call **`gather_haps_readbound_src`** instead of + `gather_haps_readbound`, and keep the `HapRanges` alive to feed + `dense_abs_row`. +3. Open, per field, the four `FieldView`s (`FieldSub::all()`) from the store's + `ContigReader` paths. `Svar2Store`/`ContigReader` gain the accessor(s) needed + to reach `ContigPaths` + the store's cohort `n_samples` and each field's + `StorageDtype` (resolved from the store `meta.json` manifest — genoray's + `StorageDtype::from_meta_str` is public). If the store lacks a requested + field, error clearly in Python before calling the kernel (see Python §1). +4. **Thread provenance through the merge** in `decode_variants_from_split` + (`src/svar2/mod.rs`). For each emitted variant, in lockstep with `pos`: + - var_key entry `i` → `unpack_vk_src(split.vk_src[i]) = (is_indel, call_idx)`; + `is_dense=false`, `idx=call_idx`. + - dense entry `i` in the snp/indel window for query `q` → + `idx = dense_abs_row(hapranges.dense_{snp,indel}_range[q], + split.dense_{snp,indel}_range[q], i)`; `is_dense=true`, `is_indel` by + channel. + Then for each field pick the sub-view by `(is_dense, is_indel)` and append: + - INFO → `view.bytes_at(idx)` + - FORMAT → var_key: `view.bytes_at(idx)`; dense: + `view.bytes_at(dense_row * cohort_n_samples + orig_samples[q])` + (== `format_at(dense_row, orig_samples[q])` as bytes). + Copy bytes verbatim (no dtype dispatch in Rust; Python `.view()`s). Missing + values already carry genoray's stored default/sentinel — passed through. + +**Provenance-vs-decode ordering invariant (must-verify):** the merge already +emits variants in a defined tie order (`var_key < dense-snp < dense-indel` on +equal positions). The field append must use the provenance of *the same* entry +chosen at each merge step — i.e. provenance is carried on the merge cursor, not +recomputed. This is the single highest-risk part; the plan must add an identity +test that a field carrying "the variant's own source row index" decodes back to +a strictly-consistent mapping. + +## Python design (`Svar2Haps`, `python/genvarloader/_dataset/_svar2_haps.py`) + +1. **Field discovery** (`from_path`): after `sv = SparseVar2(str(svar2_path))`, + read the store manifest via `sv.available_fields` (genoray 3.x). Set + `self.available_var_fields = ["alt","ilen","start"] + list(store_field_keys)` + (replaces the hard-coded `:174`). Keep a `dict[key -> (category, name, + np.dtype)]` on the instance for the kernel call and dtype `.view()`. Thread + the user's `var_fields` into `from_path` and onto the instance (today it + defaults to `["alt","ilen","start"]` and is never set from the Dataset for + svar2 — wire it like the SVAR1 path). +2. **Request set:** `requested = [f for f in self.var_fields if f not in + {"alt","start","ref","ilen","dosage"}]`. Validate each is in + `available_var_fields`; raise a clear error otherwise. Map to `(category, + name, dtype)` triples for the kernel. +3. **`_reconstruct_variants`:** pass the triples to the kernel; receive + `field_bufs`/`itemsizes`. Each field is per-variant (one element per + variant), so it is **parallel to `pos`** and flows through the exact existing + machinery with `field_g = field_c[src]`: + - single-contig fast path: `Ragged.from_offsets(buf.view(dtype), shape, + var_off_g)`; + - multi-contig: index by the same `src` from `_ragged_arange_src`; + - `unphased_union`: the `var_off[::P]` fold needs **no special handling** + (field data stays whole, offsets reindex). + Add each as `RaggedVariants(**{key: field_ragged})`, sharing `alt`'s offsets. +4. **`_reconstruct_variant_windows`:** same kernel call; add each field to the + `_FlatVariantWindows.fields` dict as a `_Flat.from_offsets(buf.view(dtype), + shape, row_off)`, exactly like `start`/`ilen` (respecting the single- vs + multi-contig branches and the `include_*` gating by `var_fields`). +5. **`DummyVariant` / empty-group fill:** variant-windows fills empty groups via + `dummy_variant`. Extend the dummy to carry a per-field fill (genoray's stored + default/sentinel, or NaN/sentinel) so `fill_empty_groups` has a value for + each field. (`DummyVariant` already has an `info: dict` slot.) + +## Data-flow / alignment invariants + +- Field buffers are 1-level ragged, per-variant, sharing the variant-axis + offsets (`var_off`) with `pos`/`ilen`. Every reorder/fold applied to `pos` + applies identically to fields. +- Dtypes are preserved end-to-end (stored → raw bytes → `.view(dtype)`); no + widening. Ints keep genoray's lossless auto-narrowed width; floats stay + `f16`/`f32`; `Flag`→`bool`. +- Missing entries carry genoray's `default`/sentinel verbatim — gvl does not + reinterpret them. + +## Field key naming & dtypes + +Use the keys genoray's manifest exposes (`sv.available_fields` keys): bare field +name when unambiguous, else category-prefixed (genoray already disambiguates +INFO vs FORMAT collisions). These become the `RaggedVariants(**fields)` kwargs +and `_FlatVariantWindows.fields` keys. Never collide with builtin keys +(`alt`/`start`/`ref`/`ilen`/`dosage`) — guard in discovery. + +## Testing + +Round-trip oracle test mirroring genoray's `test_svar2_fields_read.py`: + +1. Build a small VCF with ≥2 INFO fields (e.g. an `Integer` and a `Float=AF`) + and ≥1 FORMAT field (e.g. `DP` Integer). Convert with + `SparseVar2.from_vcf(info_fields=..., format_fields=...)`. +2. `gvl.write` a dataset over it; read `variants` and variant-windows with + `var_fields` including the field names. +3. Assert field **values** (not just shape/dtype) against the VCF ground truth, + across: + - var_key-routed vs dense-routed variants (both present in the fixture); + - multi-contig (exercise the `_ragged_arange_src` reorder); + - `unphased_union=True`; + - VCF-missing entries (assert the configured default / sentinel). +4. Rust unit test on `decode_variants_from_split` asserting provenance identity + (the ordering invariant above). + +## Docs / skill updates + +- Update `skills/genvarloader/SKILL.md` if the public `var_fields` surface for + svar2 datasets changes (it will: svar2 `variants` now honors arbitrary store + fields). +- Add a `CHANGELOG.md` (Unreleased) entry. + +## Out of scope (unchanged guards) + +`min_af`/`max_af`, `filter=="exonic"`, spliced output, annotated haps, in-kernel +RC — all still `NotImplementedError` for svar2. Non-scalar / `Number`-other +fields are rejected at the genoray write boundary, so they never reach here. diff --git a/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md b/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md new file mode 100644 index 00000000..967fe49b --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-svar2-m6b-kernel-final-pass-design.md @@ -0,0 +1,267 @@ +# SVAR2 M6b branch — final pre-merge pass (design) + +**Date:** 2026-07-13 +**Branch:** `svar2-m6b-kernel` +**Goal:** A finishing pass over the SVAR2 read-bound work — correctness/safety fixes, +shipping hygiene, and doc consistency — so the branch is mergeable modulo the genoray +release gate (documented, not resolved here). + +## Context + +The branch adds SVAR2 (`.svar2` sparse variant format) support to GenVarLoader: as a +`gvl.write` variant source, a live `Dataset` read-bound backend (all-Rust FFI kernels), +INFO/FORMAT field routing into `variants`/`variant-windows` outputs, and `unphased_union`. +It is ~23k insertions across 86 files. The full test suite is green at branch HEAD +(`pixi run -e dev pytest tests -x` exits 0; SVAR2 parity gate 31/31), and `ruff`/`ruff +format` are clean. + +Three focused audits (docs / Python / Rust) plus manual verification produced the +work-list below. This pass does **not** add features or change the parity contract; every +change is either a correctness fix, a hygiene cleanup, or a doc correction. The SVAR1 path +must remain byte-unchanged (additive-only), and the SVAR2 parity gate must stay 31/31. + +## Out of scope / the release gate (documented, not fixed) + +The branch is dev-wired to build only on this machine, and that **cannot** be resolved in +this pass — it depends on a genoray release: + +- `Cargo.toml`: `svar2-codec` and `genoray_core` are `path = "/carter/users/dlaub/projects/genoray..."`. +- `pixi.toml` (`feature.py310`): `genoray` installed from a local `dist/*.whl`. +- `pyproject.toml`: genoray version constraint dropped (`"genoray"`, unpinned). + +PyPI's newest genoray is 2.15.0; the INFO/FORMAT field-read + read-bound gather API this +branch consumes lives on genoray `main`, unreleased. **Decision: leave the dev pins in +place** (they are required to build/test) and add a prominent **RELEASE-GATE checklist** so +nothing ships silently un-pinned. The checklist lives in two places: + +1. The PR body (a "⛔ Do not merge until" section). +2. `docs/roadmaps/rust-migration.md` Phase 6a (a `Release gate` subsection). + +Checklist contents (exact lines to flip at genoray release): +- `Cargo.toml`: `svar2-codec`/`genoray_core` path-deps → published crates.io versions. +- `pixi.toml` `feature.py310`: `genoray = { path = ".../dist/*.whl" }` → `genoray = "=="`. +- `pyproject.toml`: `"genoray"` → `"genoray>=,"`. +- Re-run the full py3xx matrix once the wheel is on PyPI. + +Also confirm/adjust the version pins already touched but not gated: `numpy 0.28→0.29`, +`pyo3 0.28.3→0.29`, `seqpro 0.20.0→0.21.1` — verify these are the intended floors at merge. + +## Workstreams + +### 1. Correctness & safety (Rust + Python) — must-fix + +**1a. Unsafe serial path missing its guard.** +`src/reconstruct/mod.rs` — the raw-pointer slices at `:557,564,572` and `:857,864` carve +`out_e - out_s` from caller-supplied `out_offsets`. The `debug_assert!(out_e >= out_s)` +monotonicity guard exists only on the parallel path (`:442,:742`); the serial fallback has +none, so a non-monotonic offsets array underflows to a multi-GB slice (UB) in debug and +silently in release. Fix: hoist the same `debug_assert!` into the serial loops before each +`from_raw_parts_mut`. Confirm the SAFETY comment text matches what is actually asserted. + +**1b. Doc comment describing a reverted optimization.** +`src/svar2/mod.rs:800` documents a `get_unchecked` read of `dense_present`. `get_unchecked` +appears nowhere in `src/` (verified by grep) — all three `present_bit` closures +(`svar2/mod.rs:88`, `reconstruct/mod.rs:677`, `tracks/mod.rs:761`) use checked indexing. The +optimization was reverted and the comment left stale. Fix: delete/rewrite the comment to +describe the checked read that is actually there. (A doc comment asserting a false unsafe +invariant is a correctness hazard for the next reader.) + +**1c. Python-reachable panics → `PyErr`.** +Arrays arriving from Python are `.as_slice().unwrap()` / `.expect("must be contiguous")`'d in +the SVAR2 kernels (`ffi/mod.rs:821-829`, `reconstruct/mod.rs:639-645,692`, +`tracks/mod.rs:725-734,806-808,834-836`). A non-contiguous view (`a[::2]`) panics instead of +raising. Fix: validate contiguity once at the `#[pyfunction]` boundary and return +`PyValueError`. Also bounds-check the pure-DEL anchor index `contig_ref_s[pos..pos+1]` +(`reconstruct/mod.rs:703`) → `PyValueError` for a variant at/past contig end. Also +`svar2/mod.rs:358-365` `assert_eq!(vk_src.len(), ...)` fires in release from a +Python-reachable path — hoist the check to the FFI boundary (`ffi/mod.rs:1411`, where +`has_fields` is known) as a `PyValueError`. + +**1d. `extend_to_length` silently ignored for `.svar2`.** +`_write.py:_write_from_svar2` accepts the flag and never reads it; `chromEnd` is always +extended. Passing `extend_to_length=False` yields a different dataset than requested, with no +signal. **Fix: raise `NotImplementedError`** when `extend_to_length is False` for a `.svar2` +source (consistent with the branch's Phase-1 guard-matrix policy of failing loudly on +unsupported combinations), and document the limitation. Do **not** silently honor `True` +only — make the unsupported case explicit. + +### 2. Shipping hygiene — should-fix + +**2a. Test-only oracle code out of the library.** +`python/genvarloader/_dataset/_svar2_source.py` (`SparseVar2Source`) and +`_svar2_store_py.py` (`build_readbound_*`) have zero importers under `python/` — only +`tests/` uses them (they are the parity oracle + FFI-input builders). Move both under +`tests/` (e.g. `tests/_oracles/`), update test imports, and confirm nothing in the shipped +package references them. Rename on the way: `_svar2_store_py.py` holds no store class (the +Rust `Svar2Store` is the store) — the `_py` suffix is meaningless; name it for what it does +(`svar2_readbound_inputs.py` or similar). `SparseVar2Source` may keep its name once it lives +in tests. + +**2b. Drop dead FFI capability.** +`reconstruct_haplotypes_from_svar2_readbound`'s `annot_v_idxs`/`annot_ref_pos` params are +`None` at both call sites (`ffi/mod.rs:889-890,1045-1046`); 3 of the 4 match arms (~60 lines) +and the `:605-608` doc are unreachable. Remove the params and the dead arms (annotated-hap +output for `.svar2` is guarded `NotImplementedError` anyway). If there's a near-term plan to +wire them, leave a one-line note instead — but default to removal (YAGNI). + +**2c. Vectorize `_svar2_region_max_ends`.** +`_write.py:1092-1138` is an `O(regions × samples × ploidy)` Python triple-loop over decoded +records at write time; its own docstring flags it as a scalability follow-up. The semantics +are a per-region max over haplotypes of `(pos, end)` with a `pos`-then-`end` tie-break. Fix: +vectorize with a scatter-reduce over the decoded ragged offsets — encode `(pos<<32) | end` +into an int64, `np.maximum.reduceat` (or `np.maximum.at` on a region-index scatter) to get +the per-region max, then unpack `end`. Must stay byte-identical to the current loop +(`test_write_svar2.py` locks the cache contents + same-POS-tie behavior). + +**2d. Drop unused `region_starts`.** +`_write.py:1162,1205,1214` writes/memmaps `region_starts`; `_svar2_haps.py:86-88,95` loads it +and its own docstring says it is "kept for parity/debug, NOT fed to the FFI." Remove the +array, its `svar2_meta.json` entry, and the loader. (Confirm no test asserts its presence; if +one does, drop that assertion.) + +**2e. Fix the `typecheck` pixi task (also helps every worktree).** +`pixi run -e dev typecheck` = `pyrefly check` with no paths → inside any `.claude/worktrees/` +checkout pyrefly matches **zero files** (root `.gitignore` ignores `.claude/`, pyrefly honors +ignore files) and exits 0. Typecheck has effectively never run on this branch. Fix: change the +task to `pyrefly check python/genvarloader tests` (explicit paths). Then clear the one real +finding it surfaces: unused `# pyrefly: ignore[no-matching-overload]` at `_ragged.py:325`. + +**2f. Relocate `tmp/svar2_mvp/` into `tests/benchmarks/`.** +19 tracked files with hardcoded absolute paths (`/carter/shared/data/gdc/...`), and a +self-contradicting `.gitignore` entry (`tmp/svar2_mvp/prof_out/` ignored while its `.md` +files are tracked). `tests/benchmarks/` already has the right shape: `profiling/` for +`profile_*.py` + shell drivers, `data/build_*.py` for corpus builders, and session-scoped +path fixtures in `conftest.py`. Plan: +- Benchmark/profiling drivers (`benchmark.py`, `bench_gvl_svar1_vs_svar2.py`, `prof_*.py`, + `prof_*.sh`, `build_stores.py`, `validate.py`, `split_folded.py`) → `tests/benchmarks/` + (drivers) / `tests/benchmarks/profiling/` (perf shells), with hardcoded paths replaced by + the existing `data_dir`/`kg_dir` fixtures or a module-level constant + CLI arg. +- `.sbatch` files and `env_baseline.txt` → drop (machine/cluster-specific scratch; the perf + numbers they produced are already captured in `docs/superpowers/notes/`). +- `prof_out/*.md` reports → drop from git (superseded by the roadmap Phase-6a results + + notes); keep on disk locally. +- Remove the `tmp/svar2_mvp/prof_out/` `.gitignore` line; add `tmp/` if we want scratch + ignored going forward. +Nothing of value is lost — perf conclusions live in the roadmap and notes; only +machine-specific scratch is dropped. + +### 3. Rust duplication extraction — approved, highest-risk + +Guarded by the parity/oracle suite (31/31) + full-tree regression. Do this **last**, rebuild +and re-run parity after each extraction, and keep each extraction its own commit so a +regression bisects cleanly. + +**3a. Chunk-carving + serial/parallel dispatch helper.** +`reconstruct/mod.rs:424-578` and `:724-880` are ~150 lines verbatim (bounds build, +`split_at_mut` carve ×3, 4-arm `(av, ap)` match, serial raw-ptr path). Extract a +`carve_chunks(&mut [T], &[(usize,usize)]) -> Vec<&mut [T]>` and one dispatcher generic over +the per-chunk work closure. This is the hot path — byte-identical output is mandatory. + +**3b. Readbound FFI preamble helper.** +`ffi/mod.rs:934-998, 1086-1144, 1186-1250, 1358-1432` paste the same preamble 4× (reader +lookup → regions build → `arr2_to_ranges` ×4 → `HapRanges::new` → gather → `lut_arrays` → +`split_to_flat` → `dense_range` view). Extract one helper returning +`(FlatChannels, lut_bytes, lut_off, regions)`. The diffs→`out_offsets` prefix-sum loop +(`ffi/mod.rs:846-864, 1000-1019, 1252-1268`) is pasted 3× — extract +`offsets_from_diffs(...)`. Add a `type` alias for the three readbound return tuples to clear +the `clippy::type_complexity` warnings at `:930,1182,1344`. + +**3c. Shared `present_bit`.** +`reconstruct/mod.rs:675-678` == `tracks/mod.rs:759-762` (identical closure + re-documented +LSB-first invariant). Move to a `svar2::present_bit` fn documented once. + +### 4. Docs & comments — should-fix + +**4a. Roadmap (`docs/roadmaps/rust-migration.md`).** +- Phase 6a guard-matrix bullet (~`:812-816`) still lists `unphased_union` and + `"variant-windows"` as guarded `NotImplementedError`; both now ship. Move them to the + supported list. +- Gate footnote (~`:822-826`) claiming variant-windows parity is untested is false — remove + it (`test_svar2_readbound_variants.py`, `test_svar2_fields_read.py` cover it). +- The 2026-07-05 notes-log entry (~`:890`) repeats the stale exclusions — amend or add a + 2026-07-12/07-13 entry reflecting shipped scope. +- Add a ticked task line for the INFO/FORMAT field-routing work (plan + `2026-07-12-svar2-info-format-field-routing.md`). +- Fill the `_PR: TBD_` link once the PR exists (project rule: a ✅ phase carries a PR link) — + or leave 🚧 until the PR number is known, then update. + +**4b. `skills/genvarloader/SKILL.md`.** +- `:193-195`: `var_fields` on `.svar2` does **not** accept `ref`/`dosage`; allowed set is + `alt|ilen|start` + store INFO/FORMAT fields (`_svar2_haps.py:261` raises otherwise). Correct + the statement. +- `:66,170,437`: "`min_af`/`max_af` requires SVAR-backed genotypes" → "`.svar` only (not + `.svar2`)" — `.svar2` raises `NotImplementedError`. +- `:128,442`: note `extend_to_length` has no effect / is unsupported for a `.svar2` source + (matches the 1d fix). +- `:91`: "byte-identical … all four output modes" is contradicted by the pure-deletion ALT + paragraph below it — qualify with "except pure-deletion ALT bytes (see below)". + +**4c. Prose docs (`docs/source/`).** +- `index.md:51`: "Currently supports VCF, PGEN, and BigWig" — mirror README's updated + `.svar`/`.svar2` wording. +- `faq.md:81` + `write.md:98` point at `format.md` "for the full list" of unsupported + `.svar2` combinations, which does not exist there → add the guard-matrix list to + `format.md`, or repoint to the actual location. +- `write.md` §"Variants from a genoray sparse store": add a 2-line snippet showing how to + *build* a `.svar`/`.svar2` (`genoray` `dense2sparse` / `SparseVar2.from_vcf`), since + `faq.md:76` promises it. +- `dataset.md`: add a short "Variant fields (`var_fields`)" section — the branch's headline + feature (`.svar2` store INFO/FORMAT fields on `variants`/`variant-windows`, e.g. `rv["AF"]`) + is currently documented only in the skill. +- `format.md:145`: pin the `(unreleased)` changelog row to the target version at merge. + +**4d. Strip internal plan/task numbering from shipped code.** +Comments/docstrings referencing planning artifacts (reader has no access to them): +`_svar2_haps.py:23` (stale — lists `unphased_union` as unsupported; delete that entry), +`:384,486,489` ("tracks follow-up (7c)"), `_reconstruct.py:143` ("Task 7c"), `:399` ("FIX 1 +guard"), `_write.py:1196` ("Phase-1 wiring"); Rust `svar2/mod.rs:280` ("Task 1.3"), +`tracks/mod.rs:2467` ("Task 4 Part C"), `ffi/mod.rs:774` ("first cut minimal"). Rewrite each +in terms of behavior. The `_reconstruct.py:399` FlankSample fill-seed divergence that the +guard papers over needs a tracked GitHub issue (the comment is currently the only record) — +open one and reference it. + +**4e. Missing docstrings.** +Add numpydoc-style docstrings to public/semi-public surfaces lacking them: +`_svar2_link.py:make_svar2_link`, `_svar2_haps.py:_reconstruct_variants` (sibling +`_reconstruct_variant_windows` has one), `_write.py:_write_from_svar2` (SVAR1's +`_write_from_svar` has one); Rust `svar2/store.rs:16,19,26,46` (`reader`, `store_path`, +`#[new]`, `contigs` — PyO3-exposed, become Python docstrings). + +### 5. Clippy nits (`cargo clippy --all-targets`) — new-code only + +Clear the new-code-attributable warnings; leave pre-existing ones (`bigwig.rs`, +`reference/mod.rs`, etc.) alone as out-of-scope: +- `reconstruct/mod.rs:248,251` `explicit_auto_deref` → drop `as_deref_mut()`. +- `reconstruct/mod.rs:19-37,278` `doc_overindented_list_items` (4→2 spaces). +- `svar2/mod.rs:593,600,772` + `reconstruct/mod.rs:905` `single_range_in_vec_init` / + `redundant_closure` (tests). +- `type_complexity` aliases from 3b cover `:930,1182,1344`. + +## Sequencing + +1. Docs & comments (§4) + clippy nits (§5) — zero runtime risk, do first. +2. Correctness/safety (§1) — rebuild Rust (`maturin develop --release`), run SVAR2 parity. +3. Hygiene (§2) — oracle relocation, dead-capability removal, `max_ends` vectorization, + `region_starts` drop, typecheck-task fix, `tmp/` relocation. +4. Rust duplication (§3) — last, one extraction per commit, parity re-run after each. + +## Verification (gates — all must pass before PR) + +- `pixi run -e dev maturin develop --release` (rebuild before any Python test touching Rust). +- `pixi run -e dev pytest tests -x` — full tree green (SVAR1 byte-unchanged, SVAR2 gate 31/31). +- `pixi run -e dev cargo test` (compiles from source; LD_LIBRARY_PATH per pixi activation.env). +- `pixi run -e dev ruff check python/ tests/` + `ruff format --check python/ tests/`. +- `pixi run -e dev pyrefly check python/genvarloader tests` (the fixed task) — clean. +- `pixi run -e dev cargo clippy --all-targets` — no **new** warnings. +- `api.md` ↔ `__all__` check (should remain "none"; no public symbol added). +- Manual: confirm no `python/` module imports the relocated oracle code; confirm the + RELEASE-GATE checklist is present in both the PR body and the roadmap. + +## Non-goals + +- Resolving the genoray path-pins (release-gated; documented only). +- Any new SVAR2 feature, output mode, or change to the parity contract. +- Refactoring the unrelated genoray-3.0-API bump bundled into this branch (the + `ContigNormalizer` module moves + `_vcf_region_chunks` rewrite) — it's clean; splitting it + into its own commit is optional and noted for the reviewer, not required. +- Touching pre-existing clippy warnings outside the SVAR2 changes. diff --git a/pixi.lock b/pixi.lock index 158e8a89..1f39437e 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,7 +1,16 @@ version: 7 platforms: - name: linux-64 + virtual-packages: + - __unix=0=0 + - __linux=4.18 + - __glibc=2.28 + - __archspec=0=x86_64 - name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 environments: default: channels: @@ -88,6 +97,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -163,69 +173,66 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -326,6 +333,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -337,76 +345,73 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl dev: channels: @@ -484,6 +489,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -547,105 +553,103 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl @@ -746,6 +750,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -757,72 +762,69 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl docs: channels: @@ -907,6 +909,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -982,7 +985,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl @@ -1032,6 +1035,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl @@ -1068,7 +1072,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl @@ -1219,6 +1222,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -1230,7 +1234,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/03/78/843bcf0cf31f88d2f8a9a063d2d80817b1901657d83d65b89b3aa835732e/nbsphinx-0.9.8-py3-none-any.whl @@ -1275,6 +1279,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl @@ -1308,7 +1313,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -1448,6 +1452,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -1511,7 +1516,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.10.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl @@ -1569,6 +1574,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -1614,7 +1620,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl @@ -1779,6 +1784,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -1790,7 +1796,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/03/78/843bcf0cf31f88d2f8a9a063d2d80817b1901657d83d65b89b3aa835732e/nbsphinx-0.9.8-py3-none-any.whl @@ -1834,6 +1840,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl @@ -1867,7 +1874,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -1975,85 +1981,82 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.4.22-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-8_cp310.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/00/5820a3b1fe264b23770f77dbb3ac0f6fdadd21b640624791a05950f934da/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl osx-arm64: @@ -2085,86 +2088,83 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8f/fe/a4867bc2b8b81d9b1648992fb7e4a732b3db480ff2d02df2c7b59189c812/hypothesis-6.156.6-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl notebook: channels: @@ -2299,6 +2299,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.2.2-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.11.0-py310hb9dbd67_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -2432,65 +2433,62 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -2649,6 +2647,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.2.2-py310haea493c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pthread-stubs-0.4-hd74edd7_1002.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -2670,67 +2669,64 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h4818236_10.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py310: channels: @@ -2817,6 +2813,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.20-h3c07f61_0_cpython.conda @@ -2892,69 +2889,66 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -3055,6 +3049,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -3066,76 +3061,73 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl py311: channels: @@ -3219,6 +3211,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py311h49ec1c0_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py311haee01d2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.15-hd63d673_0_cpython.conda @@ -3294,7 +3287,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -3318,6 +3311,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/44/0b/0615dbedb98f5b32a35a53290fbdc6e22306968109278d7e58df82d7a9f6/numba-0.65.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/bb/3d3e0a9350a81f3cf38e0bf69c4ad0c17e2f4bfe142c98202294fe0cfa92/selectolax-0.4.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3337,7 +3331,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl @@ -3452,6 +3445,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py311hc949640_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py311h50b1a05_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.15-h8561d8f_0_cpython.conda @@ -3463,7 +3457,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py311hc949640_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/8f/16812ee742bbdda2746a15f5cc788cbaf4d329c35b2ca1f6cdf45685866c/ncls-0.0.70-cp311-cp311-macosx_11_0_arm64.whl @@ -3482,6 +3476,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5a/b0/a4ffc4ae74d2d822200dcc46898987d8eb6032d1e2b219cae39da6f5cbcc/pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3499,7 +3494,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/96/b3/650500c2eab4534d98e9166f4298e0f3c69c742afdf24e6eabccd1f16ad8/numba-0.65.1-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -3617,6 +3611,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py312h5253ce2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda @@ -3692,7 +3687,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -3713,6 +3708,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/55/0f/e7f1ff3a1cabc6c4486a7ee1b0506aedf2f5f8329760ac1f4e8032feef2b/pysam-0.24.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -3733,7 +3729,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/c9/65f89382870c8cf8277b06680f81ea7621324de4c49fbce7276e0fd17ce5/cyvcf2-0.32.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl @@ -3850,6 +3845,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py312hc28c600_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-h8561d8f_0_cpython.conda @@ -3861,7 +3857,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py312h2bbb03f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl @@ -3880,6 +3876,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -3896,7 +3893,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/98/c1/37f2fcccce3f1494147e46ccc04996226defe9ccae8251a9ce61296fa599/pysam-0.24.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl @@ -4015,6 +4011,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-7_hd590300_perl5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/prek-0.3.13-hb17b654_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py313h07c4f96_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py313h54dd161_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.13-h6add32d_100_cp313.conda @@ -4090,7 +4087,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/uc-micro-py-2.0.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -4110,6 +4107,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/42/2b2b00b0df934353c8e3ebfb2be68f63d7aa6dfe5eb5d5f6427b092e0814/awkward_cpp-52-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/e6/01ada46425bd120b36e7c2f6fe74a980de6912d416f07c9c85b22fadd515/pybigwig-0.3.25-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/5f/dd/0c6a5a36ec132665f85e5e33f0480b58cf5aa8af8fbe1d5971410d789558/ncls-0.0.70.tar.gz @@ -4130,7 +4128,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/22/b8d873f6466b20aa563fc9b33acd48dec89a07803ddaa2f1c8ca1cd33126/numba-0.65.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -4249,6 +4246,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/perl-5.32.1-7_h4614cfb_perl5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/prek-0.3.13-h6fdd925_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py313h0997733_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py313h6fa1262_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.13-h20e6be0_100_cp313.conda @@ -4260,7 +4258,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/wrapt-2.1.2-py313h0997733_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . + - pypi: ./ - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl @@ -4280,6 +4278,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5f/dd/0c6a5a36ec132665f85e5e33f0480b58cf5aa8af8fbe1d5971410d789558/ncls-0.0.70.tar.gz - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl @@ -4300,7 +4299,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl @@ -4465,6 +4463,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/psutil-7.2.2-py310h139afa4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/py-spy-0.4.2-he182f42_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.11.0-py310hb9dbd67_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -4598,100 +4597,98 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/widgetsnbextension-4.0.15-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/xarray-2025.6.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - - pypi: . + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/00/ba/8d8aa1df96e0666752e5c9d406d440495df2014d315b2a95bbef9856b23e/datafusion-50.1.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0a/59/69032bf511d51bbc2d45311110386042a7b6a62e6149f919e94a1b55979e/pybigwig-0.3.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/2d/ea/d6230cc9772edca0fd8c91638dc1c2e2def09b9a22cfabe6532906e96aea/basenji2_pytorch-0.1.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/35/7d/bc4080a0d94719a039a96b1b5fb5b9a12d0048fab9f56efd9324fa07a096/ncls-0.0.70-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/74/df/b1f009cb86e2d721ad8a1e9f64acb0df49743e15b62dad54276e863bc960/seqpro-0.20.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/ff/9d30128a88df6c795097b6f73218d4a5afcd0e2d74cf2dedd99b28d42cdc/cyvcf2-0.31.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/8e/0eccb528701273640dd4f13678a0c1352176166aecd1ee7f4fa29355132b/hirola-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c1/66/4f1f8decc506e7a561245982832e4ab87d95e6b08b1b9ec657f2c91ad5e1/polars_bio-0.20.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7d/0b8e265c43978292425249ae1fd63d5f7b0a719c78eee70055751b5bb8cb/oxbow-0.5.2-cp39-abi3-manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl @@ -4850,6 +4847,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/psutil-7.2.2-py310haea493c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pthread-stubs-0.4-hd74edd7_1002.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/py-spy-0.4.2-h748bcf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pytest-codspeed-5.0.3-py310ha69dea2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.20-h1b19095_0_cpython.conda @@ -4871,67 +4869,64 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h4818236_10.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-ng-2.3.3-hed4e4f5_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - - pypi: . - - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/07/9e/74c4c410837f223ee97183948e9f228433f7d802d45b6cc53286cc9627a2/genoray-2.12.3-py3-none-any.whl + - pypi: ./ + - pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0c/2c/d4d96c78e72031f3171fb3a584b557d79d191e9bb4e93747f793c18f8623/fast_histogram-0.14-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/4e/ca/03624e017e5ee2d7ce8a08d89f81c1e535eb3c30d7b2dc4a435ea3fbbeae/mkdocs_glightbox-0.5.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5b/bc/246f452431c592a2a424050e8bb9ccf494fb47613fd97c912f4d573a5e3b/phantom_types-3.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6b/a4/bd8ad7d1cf66268219f282d4268b6ba73f8ff51c5fba2c9663adab103615/sorted_nearest-0.0.41.tar.gz - - pypi: https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/58/2fce4fc87cfd75802e5f397470f00dfd15ec77a8164d457d67efa8d576f9/cyvcf2-0.31.4-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6e/ae/76fb528c6112a3df5a581a18f1a2ceee5983d54977d7f2b6bc883637fe4c/polars_config_meta-0.3.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/89/35ea267fb12e608529f0df315aff200171e555623cb38b2e4444592ce872/pyranges-0.1.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/b2/04438111b57e3591c09dfa9f220609ae1afacf436fba124a328dbdb9b7b2/genvarloader_cli-0.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/a6/aa38bddc9f8d90e5ce14023f06ccbf642ab5d507da1ffafb031c0f332dc6/numerary-0.4.4-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ae/e5/6dc3400bc4bea5e6576c2468536bad65b956483f2c32ec9af9c98e7c068d/polars_bio-0.20.1-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/16/9298cca16e26d2fc3751a2b93469cc14a6f8bb9fb5f4087977926d936383/hirola-0.3.0-py3-none-macosx_10_6_universal2.whl + - pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/39/2c9aea082383c8d41d0d8b8f9907dc6273f8cee47a8c83d0c8bad02488b4/pysam-0.24.0-cp310-cp310-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cc/e4/d71e71edc972d534f51cff72fa30edb8b0e5df109b2e5ae67a3985faebb9/joblib_progress-1.0.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cf/f4/672930ad51bbc135f51cac89577155f1d0c2d120375fc77978d3aa071bb7/pybigwig-0.3.25.tar.gz - - pypi: https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/db/58/2dc473240f552d3620186b527c04397f82b36f02243afaf49f0813c84a17/datafusion-50.1.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl packages: - conda: https://conda.anaconda.org/bioconda/linux-64/bcftools-1.23.1-hb2cee57_0.conda @@ -6805,7 +6800,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/maturin?source=compressed-mapping + - pkg:pypi/maturin?source=hash-mapping size: 9793377 timestamp: 1778496710620 - conda: https://conda.anaconda.org/conda-forge/linux-64/memray-1.19.3-py310hbdcf458_0.conda @@ -7367,6 +7362,66 @@ packages: purls: [] size: 1759386 timestamp: 1777312019188 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py310h7c4b9e2_0.conda + sha256: 5315f974ccb4a664aaacdf3cdbcf8619221f6fb211b44045a9bcf2ec6f2eec16 + md5: 3c8b1312ed395879899491168f503d00 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.10,<3.11.0a0 + - python_abi 3.10.* *_cp310 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 160187 + timestamp: 1767565456971 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py311h49ec1c0_0.conda + sha256: 37293d78effcfd7958d7e467bf961e0abb017ddd49eb75fa3cbdefe489d04b2f + md5: 9ec77f7ef80f25bc19d8eeb8ba12f142 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.11,<3.12.0a0 + - python_abi 3.11.* *_cp311 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 191363 + timestamp: 1767565475544 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py312h4c3975b_0.conda + sha256: d355a34b127d5ffbf716e19c8beaf4031251ccb029285618437e344c12a2d0b0 + md5: 0b365b8629970ebf8486485fefa5f8d5 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.12,<3.13.0a0 + - python_abi 3.12.* *_cp312 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 187624 + timestamp: 1767565456324 +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyinstrument-5.1.2-py313h07c4f96_0.conda + sha256: ba206c26c6555a08b8fce977a31f848c78fa30531688d3dd5789e99613ff311a + md5: 572bb387712a5ed27f860aec04d3f5a0 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 190051 + timestamp: 1767565452479 - conda: https://conda.anaconda.org/conda-forge/linux-64/pyrefly-1.0.0-h2b88eb6_0.conda sha256: 225d491c94a6962b92377154b33b424cb0cc30aa2fe0b433c6c110d05da7283d md5: 132a1bc9d31e17d6e12d7188ca7ddfb8 @@ -7401,7 +7456,8 @@ packages: - libxslt >=1.1.43,<2.0a0 license: LGPL-3.0-only purls: - - pkg:pypi/pyside6?source=compressed-mapping + - pkg:pypi/pyside6?source=hash-mapping + - pkg:pypi/shiboken6?source=hash-mapping size: 13559182 timestamp: 1778540466597 - conda: https://conda.anaconda.org/conda-forge/linux-64/pytest-codspeed-5.0.3-py310h139afa4_0.conda @@ -9061,7 +9117,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/matplotlib-inline?source=compressed-mapping + - pkg:pypi/matplotlib-inline?source=hash-mapping size: 15725 timestamp: 1778264403247 - conda: https://conda.anaconda.org/conda-forge/noarch/mdit-py-plugins-0.6.0-pyhd8ed1ab_0.conda @@ -9679,7 +9735,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/tomlkit?source=compressed-mapping + - pkg:pypi/tomlkit?source=hash-mapping size: 41169 timestamp: 1778423744478 - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.15.0-pyhcf101f3_0.conda @@ -11003,9 +11059,69 @@ packages: purls: [] size: 1176168 timestamp: 1777313031417 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda - sha256: 34dd964342267b5d93674eefe37055b8915943de60d43b4e36a08b9642319f46 - md5: b0b2e5d34dba70dfad3955a59e611b46 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py310h72544b6_0.conda + sha256: 5155281bd659b1c87a2621739e2cb4cad5643239b1238cd51ba344e029cd7377 + md5: 4fd4fd1a923ab4d04a31beffe986f0e3 + depends: + - __osx >=11.0 + - python >=3.10,<3.11.0a0 + - python >=3.10,<3.11.0a0 *_cpython + - python_abi 3.10.* *_cp310 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 159914 + timestamp: 1767565719097 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py311hc949640_0.conda + sha256: 963ced7b3506a1817c0db28cd647395d41d7cd4c9c3da9dfd7f107565c4c91a9 + md5: c6bda0caf8d63adbe088ac8181fb1960 + depends: + - __osx >=11.0 + - python >=3.11,<3.12.0a0 + - python >=3.11,<3.12.0a0 *_cpython + - python_abi 3.11.* *_cp311 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 189747 + timestamp: 1767565960069 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py312h2bbb03f_0.conda + sha256: b31076b97840fb2e983413e4034090cfaf7128cd06f32d3e94d0f2b397fac350 + md5: 84fd6824786da8cb8391c2038186b74c + depends: + - __osx >=11.0 + - python >=3.12,<3.13.0a0 + - python >=3.12,<3.13.0a0 *_cpython + - python_abi 3.12.* *_cp312 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 187474 + timestamp: 1767565653461 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyinstrument-5.1.2-py313h0997733_0.conda + sha256: ce2592818d1de99a561df62f505b2b8ed8a9bd2f6bc22f57ac517f3587afbf83 + md5: 36016d88b9275f390d39434d0c041662 + depends: + - __osx >=11.0 + - python >=3.13,<3.14.0a0 + - python >=3.13,<3.14.0a0 *_cp313 + - python_abi 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pyinstrument?source=hash-mapping + run_exports: {} + size: 188912 + timestamp: 1767565820081 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyrefly-1.0.0-h4dd0d4f_0.conda + sha256: 34dd964342267b5d93674eefe37055b8915943de60d43b4e36a08b9642319f46 + md5: b0b2e5d34dba70dfad3955a59e611b46 depends: - __osx >=11.0 constrains: @@ -11496,11 +11612,11 @@ packages: purls: [] size: 433413 timestamp: 1764777166076 -- pypi: . +- pypi: ./ name: genvarloader requires_dist: - seqpro>=0.20 - - genoray>=2.12.3,<3 + - genoray - numpy - loguru - natsort @@ -11525,6 +11641,39 @@ packages: - tbb ; extra == 'tbb' - pyomp ; extra == 'pyomp' requires_python: '>=3.10,<3.14' +- pypi: /carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl + name: genoray + version: 2.15.0 + requires_dist: + - seqpro>=0.21.1,<0.22 + - numpy>=1.26 + - pandas>=2.2.3 + - hirola>=0.3.0 + - pgenlib>=0.91.0 + - cyvcf2>=0.31.1 + - pysam>=0.22 + - polars>=1.37.1 + - polars-bio>=0.20.1,<0.34 + - pyranges>=0.1.3 + - typing-extensions>=4.14 + - pyarrow>=21 + - tqdm>=4.65 + - phantom-types>=3 + - more-itertools>=10 + - loguru>=0.7.0 + - attrs + - awkward + - numba + - cyclopts + - zstandard + - pydantic + - oxbow>=0.5.1,<0.6 + - joblib>=1.4.2,<2 + - joblib-progress>=1.0.6,<2 + - filelock>3,<4 + - scipy>=1.10 + - pooch>=1.7 + requires_python: '>=3.10,<3.14' - pypi: https://download-r2.pytorch.org/whl/cu126/torch-2.10.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl name: torch version: 2.10.0+cu126 @@ -11662,6 +11811,15 @@ packages: - ruff>=0.12.0 ; extra == 'dev' - cython-lint>=0.12.2 ; extra == 'dev' requires_python: '>=3.12' +- pypi: https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: yarl + version: 1.24.2 + sha256: f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90 + requires_dist: + - idna>=2.0 + - multidict>=4.0 + - propcache>=0.2.1 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/02/bf/6f506a37c7f8ecc4576caf9486e303c7af249f6d70447bb51dde9d78cb99/sphinx_book_theme-1.2.0-py3-none-any.whl name: sphinx-book-theme version: 1.2.0 @@ -11720,11 +11878,49 @@ packages: version: 4.1.0 sha256: 647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - name: nvidia-cuda-nvrtc-cu12 - version: 12.8.93 - sha256: a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994 - requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/05/00/5820a3b1fe264b23770f77dbb3ac0f6fdadd21b640624791a05950f934da/hypothesis-6.156.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: hypothesis + version: 6.156.6 + sha256: c7c3f067166bfc28b67f0042fc5fdcc87b3b58664da0c2f563fe544448b7ab3b + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-cusparse-cu12 version: 12.5.4.2 @@ -11815,30 +12011,6 @@ packages: - cudf-polars-cu12 ; extra == 'gpu' - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl - name: zipp - version: 3.23.1 - sha256: 0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc - requires_dist: - - pytest>=6,!=8.1.* ; extra == 'test' - - jaraco-itertools ; extra == 'test' - - jaraco-functools ; extra == 'test' - - more-itertools ; extra == 'test' - - big-o ; extra == 'test' - - pytest-ignore-flaky ; extra == 'test' - - jaraco-test ; extra == 'test' - - sphinx>=3.5 ; extra == 'doc' - - jaraco-packaging>=9.3 ; extra == 'doc' - - rst-linker>=1.9 ; extra == 'doc' - - furo ; extra == 'doc' - - sphinx-lint ; extra == 'doc' - - jaraco-tidelift>=1.4 ; extra == 'doc' - - pytest-checkdocs>=2.4 ; extra == 'check' - - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' - - pytest-cov ; extra == 'cover' - - pytest-enabler>=2.2 ; extra == 'enabler' - - pytest-mypy ; extra == 'type' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: scipy version: 1.17.1 @@ -11976,11 +12148,6 @@ packages: - pytest ; extra == 'test' - hypothesis[numpy] ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cuda-runtime-cu12 - version: 12.8.90 - sha256: adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/0e/93/c8c361bf0a2fe50f828f32def460e8b8a14b93955d3fd302b1a9b63b19e4/pytorch_lightning-2.6.1-py3-none-any.whl name: pytorch-lightning version: 2.6.1 @@ -12236,6 +12403,26 @@ packages: version: 0.5.2 sha256: 6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: triton + version: 3.7.1 + sha256: ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e + requires_dist: + - importlib-metadata ; python_full_version < '3.10' + - cmake>=3.20,<4.0 ; extra == 'build' + - lit ; extra == 'build' + - autopep8 ; extra == 'tests' + - isort ; extra == 'tests' + - numpy ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-forked ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - scipy>=1.7.1 ; extra == 'tests' + - llnl-hatchet ; extra == 'tests' + - matplotlib ; extra == 'tutorials' + - pandas ; extra == 'tutorials' + - tabulate ; extra == 'tutorials' + requires_python: '>=3.10,<3.15' - pypi: https://files.pythonhosted.org/packages/15/ef/7d57ceb0651af74194e97ed6583e148d352f03d696090221b8059cdfc90b/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: polars-runtime-32 version: 1.40.1 @@ -12269,39 +12456,6 @@ packages: - requests ; extra == 'telegram' - ipywidgets>=6 ; extra == 'notebook' requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl - name: torch - version: 2.10.0 - sha256: a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321 - requires_dist: - - filelock - - typing-extensions>=4.10.0 - - setuptools ; python_full_version >= '3.12' - - sympy>=1.13.3 - - networkx>=2.5.1 - - jinja2 - - fsspec>=0.8.5 - - cuda-bindings==12.9.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-nvrtc-cu12==12.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-runtime-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cuda-cupti-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cudnn-cu12==9.10.2.21 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cublas-cu12==12.8.4.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cufft-cu12==11.3.3.83 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-curand-cu12==10.3.9.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusolver-cu12==11.7.3.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusparse-cu12==12.5.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cusparselt-cu12==0.7.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nccl-cu12==2.27.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvshmem-cu12==3.4.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvtx-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-nvjitlink-cu12==12.8.93 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - nvidia-cufile-cu12==1.13.1.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - triton==3.6.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' - - optree>=0.13.0 ; extra == 'optree' - - opt-einsum>=3.3 ; extra == 'opt-einsum' - - pyyaml ; extra == 'pyyaml' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl name: pydantic-extra-types version: 2.11.1 @@ -12330,27 +12484,11 @@ packages: - semver>=3.0.2 ; extra == 'semver' - uuid-utils>=0.6.0 ; python_full_version < '3.14' and extra == 'uuid-utils' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl - name: pyarrow - version: 21.0.0 - sha256: e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26 - requires_dist: - - pytest ; extra == 'test' - - hypothesis ; extra == 'test' - - cffi ; extra == 'test' - - pytz ; extra == 'test' - - pandas ; extra == 'test' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl name: websockets version: '16.0' sha256: 9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - name: typing-extensions - version: 4.15.0 - sha256: f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/18/dc/1843828349729a86f8d9f79b19bd6e7eaa358a5682f13a0af667dae0c1d0/cyvcf2-0.32.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: cyvcf2 version: 0.32.1 @@ -12388,23 +12526,60 @@ packages: requires_dist: - numpy>=1.21.3 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cufft-cu12 - version: 11.3.3.83 - sha256: 4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 +- pypi: https://files.pythonhosted.org/packages/1e/51/f2d40a7ba9477f97cab5e1d06338c7f5f97f76479d1443ff4c6b5048e2bc/seqpro-0.21.1-cp39-abi3-macosx_11_0_arm64.whl + name: seqpro + version: 0.21.1 + sha256: e40d7b20697685694a0e4ab7864f7cc93d784950d19b7160402c304e15ce725f requires_dist: - - nvidia-nvjitlink-cu12 - requires_python: '>=3' -- pypi: https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: charset-normalizer - version: 3.4.7 - sha256: cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 - requires_python: '>=3.7' + - numba>=0.58.1 + - numpy>=1.26.0 + - polars>=1.21.0,<2 + - pyranges>=0.1.3,<0.2 + - pandera>=0.31.1 + - pandas + - pyarrow + - natsort + - narwhals>=2.20.0 + - setuptools>=70 + - awkward>=2.5.0 + - polars-config-meta[polars]>=0.3.2 + - attrs + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + name: idna + version: '3.18' + sha256: 7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/1f/21/f308e6f8b84f56cb441b7795f40080cb68a89f6430ea6f8f9e25daf83788/pgenlib-0.94.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: pgenlib + version: 0.94.1 + sha256: 1c267d592095c2363a3fb01e128740dea1b42e9a4a3a86bb0ab346ac7dda773b + requires_dist: + - numpy>=1.19.3 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/21/48/92dddc8df65b576c9d30752650c89301b5222d4ac10187724796cedfd723/pysam-0.24.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: pysam version: 0.24.0 sha256: 98a40440b16bbf67b696914a10acc241a0cd0fa02dcb14aee8e614eb2c5d8b37 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl + name: tqdm + version: 4.68.4 + sha256: 5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2 + requires_dist: + - colorama ; sys_platform == 'win32' + - requests ; extra == 'discord' + - envwrap ; extra == 'discord' + - slack-sdk ; extra == 'slack' + - envwrap ; extra == 'slack' + - requests ; extra == 'telegram' + - envwrap ; extra == 'telegram' + - ipywidgets>=6 ; extra == 'notebook' + requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl name: certifi version: 2026.4.22 @@ -12453,6 +12628,13 @@ packages: - trio>=0.10.0 ; extra == 'trio' - pyyaml>=6.0.1 ; extra == 'yaml' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/23/7c/df4a47340a003ab1aa57d1f8d44ba5eb7c91d42de8fd51a3322e33931642/pgenlib-0.94.1-cp310-cp310-macosx_10_9_universal2.whl + name: pgenlib + version: 0.94.1 + sha256: f6242c3afe0238d7180b6926e927c0626b3bded16c58af15732d829d6bd850c4 + requires_dist: + - numpy>=1.19.3 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl name: lightning-utilities version: 0.15.3 @@ -12466,11 +12648,6 @@ packages: - jsonargparse[signatures]>=4.38.0 ; extra == 'cli' - tomlkit ; extra == 'cli' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl - name: charset-normalizer - version: 3.4.7 - sha256: cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d - requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl name: sphinxcontrib-qthelp version: 2.0.0 @@ -12483,13 +12660,6 @@ packages: - pytest ; extra == 'test' - defusedxml>=0.7.1 ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/28/0f/ae28551a2cc20c87a0cd435045824501258a8564f12981802e5ad68a54c9/arro3_core-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - name: arro3-core - version: 0.8.0 - sha256: 524e2ce13ea3d2739df4d52ea03977a53d103c1fd73f0fcc6a713903ea6ad4fa - requires_dist: - - typing-extensions ; python_full_version < '3.12' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/28/53/21f7b97e82772caa61541348427f42435120b32961c92d16f9c8ce9757d6/cslug-1.0.0-py3-none-any.whl name: cslug version: 1.0.0 @@ -12521,6 +12691,22 @@ packages: - pytest-httpserver ; extra == 'test' - pytest-localftpserver ; extra == 'test' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2a/68/1fc93dd759605b5d00fc98b50200739e41ed32bd22d6ba35ca6c3932371b/rich_rst-2.1.0-py3-none-any.whl + name: rich-rst + version: 2.1.0 + sha256: 7ecd1343ee12c879d0e7ae74c3eb6d263b023d2929c6d114212eb1fd91057255 + requires_dist: + - rich>=12.0.0 + - pygments>=2.0.0 + - docutils ; extra == 'docs' + - sphinx ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - ruff ; extra == 'dev' + - mypy ; extra == 'dev' + - pre-commit ; extra == 'dev' - pypi: https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl name: mistune version: 3.2.1 @@ -12528,15 +12714,34 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.11' requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl - name: pandera - version: 0.31.1 - sha256: f9f1ff4852804e1a181a4cb968e732a492f4b6dbefe051a8c5500da43d5c326d +- pypi: https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: aiohttp + version: 3.14.1 + sha256: 23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb requires_dist: - - packaging>=20.0 - - pydantic - - typeguard - - typing-extensions + - aiohappyeyeballs>=2.5.0 + - aiosignal>=1.4.0 + - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' + - attrs>=17.3.0 + - frozenlist>=1.1.1 + - multidict>=4.5,<7.0 + - propcache>=0.2.0 + - typing-extensions>=4.4 ; python_full_version < '3.13' + - yarl>=1.17.0,<2.0 + - aiodns>=3.3.0 ; sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotli>=1.2 ; platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' + - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl + name: pandera + version: 0.31.1 + sha256: f9f1ff4852804e1a181a4cb968e732a492f4b6dbefe051a8c5500da43d5c326d + requires_dist: + - packaging>=20.0 + - pydantic + - typeguard + - typing-extensions - typing-inspect>=0.6.0 - numpy>=1.24.4 ; extra == 'pandas' - pandas>=2.1.1 ; extra == 'pandas' @@ -12640,6 +12845,11 @@ packages: - torchinfo>=1.8.0 - torchmetrics>=1.6.2 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cuda-runtime + version: 13.0.96 + sha256: 7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/2e/66/70786ee1cfdd03d36d456c4ef02a35506b7ae256c70a74bd7abf135daba0/arro3_core-0.8.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: arro3-core version: 0.8.0 @@ -12647,24 +12857,6 @@ packages: requires_dist: - typing-extensions ; python_full_version < '3.12' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: aiohttp - version: 3.13.5 - sha256: bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95 - requires_dist: - - aiohappyeyeballs>=2.5.0 - - aiosignal>=1.4.0 - - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' - - attrs>=17.3.0 - - frozenlist>=1.1.1 - - multidict>=4.5,<7.0 - - propcache>=0.2.0 - - yarl>=1.17.0,<2.0 - - aiodns>=3.3.0 ; extra == 'speedups' - - brotli>=1.2 ; platform_python_implementation == 'CPython' and extra == 'speedups' - - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' - - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and extra == 'speedups' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/2f/97/9214bd9b860e680a281232e218d10b718a7280b593f4ab56240a558dc975/pgenlib-0.94.0-cp312-cp312-macosx_10_13_universal2.whl name: pgenlib version: 0.94.0 @@ -12806,11 +12998,27 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl + name: nvidia-cuda-cupti + version: 13.0.85 + sha256: 4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/34/0b/b9d1911cfefa61399821dfb37f486d83e0f42630a8d12f7194270c417002/llvmlite-0.47.0-cp311-cp311-macosx_11_0_arm64.whl name: llvmlite version: 0.47.0 sha256: 74090f0dcfd6f24ebbef3f21f11e38111c4d7e6919b54c4416e1e357c3446b07 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: arro3-core + version: 0.8.1 + sha256: 31b966b0f9b2a6fb3f285339ed5ec4eac69591bbb92a9fdb2cf80ad995b759af + requires_dist: + - typing-extensions ; python_full_version < '3.12' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl + name: nvidia-cusparselt-cu13 + version: 0.8.1 + sha256: 786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0 - pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl name: sphinxcontrib-devhelp version: 2.0.0 @@ -12861,6 +13069,42 @@ packages: version: 0.5.2 sha256: b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl + name: zipp + version: 4.1.0 + sha256: 25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - jaraco-itertools ; extra == 'test' + - jaraco-functools ; extra == 'test' + - more-itertools ; extra == 'test' + - big-o ; extra == 'test' + - pytest-ignore-flaky ; extra == 'test' + - jaraco-test ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pytest-checkdocs>=2.14 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=3.4 ; extra == 'enabler' + - pytest-mypy>=1.0.1 ; platform_python_implementation != 'PyPy' and extra == 'type' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cublas + version: 13.1.1.3 + sha256: 37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436 + requires_dist: + - nvidia-cuda-nvrtc + requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-nvshmem-cu13 + version: 3.4.5 + sha256: 290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/3e/fe/1624eb5024e897bf4074bfc31f9e5e823160aed1ac14e7720e849a3d1109/selectolax-0.4.8-cp313-cp313-macosx_11_0_arm64.whl name: selectolax version: 0.4.8 @@ -12881,6 +13125,11 @@ packages: version: 4.0.15 sha256: 8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366 requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cufile + version: 1.15.1.6 + sha256: 08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/40/03/f5d0b979c6a1f8a8a11ba115a7c5b145671f092372a4ede164dc2597c466/ncls-0.0.70-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: ncls version: 0.0.70 @@ -13062,17 +13311,6 @@ packages: - numpy>=1.22 - numpy>=1.22,<2.5 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl - name: pyarrow - version: 21.0.0 - sha256: 26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51 - requires_dist: - - pytest ; extra == 'test' - - hypothesis ; extra == 'test' - - cffi ; extra == 'test' - - pytz ; extra == 'test' - - pandas ; extra == 'test' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: llvmlite version: 0.47.0 @@ -13083,6 +13321,11 @@ packages: version: 2.8.3 sha256: ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl + name: pyarrow + version: 22.0.0 + sha256: 710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/47/d4/dbacced3953544b9a93088cc10ef2b596d348c983d5c67a404fa41ec51ba/fonttools-4.62.1-cp312-cp312-macosx_10_13_universal2.whl name: fonttools version: 4.62.1 @@ -13122,6 +13365,11 @@ packages: version: 12.6.80 sha256: 6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + name: typing-extensions + version: 4.16.0 + sha256: 481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/4b/82/14fed4543ed4ddb4fa582f04bd50e9c2dacad4f6c2aa38de4cf8b32ea252/seqpro-0.20.0-cp39-abi3-macosx_11_0_arm64.whl name: seqpro version: 0.20.0 @@ -13165,15 +13413,6 @@ packages: requires_dist: - selectolax>=0.3.29 requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: yarl - version: 1.23.0 - sha256: 31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 - requires_dist: - - idna>=2.0 - - multidict>=4.0 - - propcache>=0.2.1 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/50/3e/f0dba6333dbe5c5a338d1466939c8733256a5f6d7e10615b8f96a90277e5/fast_histogram-0.14-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: fast-histogram version: '0.14' @@ -13183,6 +13422,29 @@ packages: - pytest ; extra == 'test' - hypothesis[numpy] ; extra == 'test' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl + name: torch + version: 2.13.0 + sha256: c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4 + requires_dist: + - filelock + - typing-extensions>=4.10.0 + - setuptools>=77.0.3 + - sympy>=1.13.3 + - networkx>=2.5.1 + - jinja2 + - fsspec>=0.8.5 + - cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.3 ; sys_platform == 'linux' + - cuda-bindings>=13.0.3,<14 ; python_full_version < '3.15' and sys_platform == 'linux' + - nvidia-cudnn-cu13==9.20.0.48 ; sys_platform == 'linux' + - nvidia-cusparselt-cu13==0.8.1 ; sys_platform == 'linux' + - nvidia-nccl-cu13==2.29.7 ; sys_platform == 'linux' + - nvidia-nvshmem-cu13==3.4.5 ; sys_platform == 'linux' + - triton==3.7.1 ; python_full_version < '3.15' and sys_platform == 'linux' + - optree>=0.13.0 ; extra == 'optree' + - opt-einsum>=3.3 ; extra == 'opt-einsum' + - pyyaml ; extra == 'pyyaml' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl name: sphinxcontrib-serializinghtml version: 2.0.0 @@ -13239,6 +13501,16 @@ packages: version: 0.24.0 sha256: 4a642f18649e59817de272173e9c27c031dceaca199809e4f8b338ebfc5d6698 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/55/5e/f8f4f7cc9b24b35103eb9e19f0f69935d16b878b4c5e4511ecd2261403fb/vcfixture-0.6.0-py3-none-any.whl + name: vcfixture + version: 0.6.0 + sha256: 62e59f9ea7a1c8847057cff8f97c9e74588445afac15bd9dd43f34b7f8bb4ff9 + requires_dist: + - numpy>=1.24 + - pysam>=0.22 + - hypothesis>=6.100 + - typing-extensions>=4.4 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl name: ipywidgets version: 8.1.8 @@ -13298,13 +13570,6 @@ packages: - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and extra == 'speedups' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/57/f6/a92704f33af317ce33c2bbda4a63f902f088d24b92a89fb5cdc52148e7cb/arro3_core-0.8.0-cp310-cp310-macosx_11_0_arm64.whl - name: arro3-core - version: 0.8.0 - sha256: 29b3d1cbd2c4bac787f473d071e1eb02b71b2701a7118bb5d0a274ffbd26b16c - requires_dist: - - typing-extensions ; python_full_version < '3.12' - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl name: tornado version: 6.5.5 @@ -13400,6 +13665,14 @@ packages: - xlsxwriter>=3.2.0 ; extra == 'all' - zstandard>=0.23.0 ; extra == 'all' requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl + name: typeguard + version: 4.5.2 + sha256: fcf9de18bd945cdb4c7b996e12b4c51ce83f92f191314a6d7cf1739586ec98cf + requires_dist: + - importlib-metadata>=3.6 ; python_full_version < '3.10' + - typing-extensions>=4.14.0 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl name: torch version: 2.10.0 @@ -13467,6 +13740,62 @@ packages: version: 0.3.25 sha256: ba907b069ad498647661cc376ba79438728a321afb61a9eca333d60ca9c6d9a6 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl + name: setuptools + version: 83.0.0 + sha256: 29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - virtualenv>=13.0.0 ; extra == 'test' + - wheel>=0.44.0 ; extra == 'test' + - pip>=19.1 ; extra == 'test' + - packaging>=24.2 ; extra == 'test' + - jaraco-envs>=2.2 ; extra == 'test' + - pytest-xdist>=3 ; extra == 'test' + - jaraco-path>=3.7.2 ; extra == 'test' + - build[virtualenv]>=1.0.3 ; extra == 'test' + - filelock>=3.4.0 ; extra == 'test' + - ini2toml[lite]>=0.14 ; extra == 'test' + - tomli-w>=1.0.0 ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' + - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' + - pytest-home>=0.5 ; extra == 'test' + - pytest-subprocess ; extra == 'test' + - pyproject-hooks!=1.1 ; extra == 'test' + - jaraco-test>=5.5 ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pygments-github-lexers==0.0.5 ; extra == 'doc' + - sphinx-favicon ; extra == 'doc' + - sphinx-inline-tabs ; extra == 'doc' + - sphinx-reredirects ; extra == 'doc' + - sphinxcontrib-towncrier ; extra == 'doc' + - sphinx-notfound-page>=1,<2 ; extra == 'doc' + - pyproject-hooks!=1.1 ; extra == 'doc' + - towncrier<24.7 ; extra == 'doc' + - packaging>=24.2 ; extra == 'core' + - more-itertools>=8.8 ; extra == 'core' + - jaraco-text>=3.7 ; extra == 'core' + - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' + - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' + - wheel>=0.43.0 ; extra == 'core' + - jaraco-functools>=4 ; extra == 'core' + - more-itertools ; extra == 'core' + - pytest-checkdocs>=2.14 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - ruff>=0.13.0 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=3.4 ; extra == 'enabler' + - pytest-mypy>=1.0.1 ; platform_python_implementation != 'PyPy' and extra == 'type' + - mypy==1.18.* ; extra == 'type' + - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' + - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl name: sphinxcontrib-applehelp version: 2.0.0 @@ -13483,54 +13812,20 @@ packages: version: 1.8.0 sha256: f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/5f/1c/ed568eca3a963dc3e447b01961ae653e0d6f107c2cfd77b3f2b1a5cfc520/hypothesis-6.152.6-py3-none-any.whl - name: hypothesis - version: 6.152.6 - sha256: b20ffc532e5f2901229348d10ed7cb37fd9723ebf4799df663d2dce1cdce4e32 - requires_dist: - - exceptiongroup>=1.0.0 ; python_full_version < '3.11' - - sortedcontainers>=2.1.0,<3.0.0 - - click>=7.0 ; extra == 'cli' - - black>=20.8b0 ; extra == 'cli' - - rich>=9.0.0 ; extra == 'cli' - - libcst>=0.3.16 ; extra == 'codemods' - - black>=20.8b0 ; extra == 'ghostwriter' - - pytz>=2014.1 ; extra == 'pytz' - - python-dateutil>=1.4 ; extra == 'dateutil' - - lark>=0.10.1 ; extra == 'lark' - - numpy>=1.21.6 ; extra == 'numpy' - - pandas>=1.1 ; extra == 'pandas' - - pytest>=4.6 ; extra == 'pytest' - - dpcontracts>=0.4 ; extra == 'dpcontracts' - - redis>=3.0.0 ; extra == 'redis' - - hypothesis-crosshair>=0.0.27 ; extra == 'crosshair' - - crosshair-tool>=0.0.102 ; extra == 'crosshair' - - tzdata>=2026.1 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') - - django>=4.2 ; extra == 'django' - - watchdog>=4.0.0 ; extra == 'watchdog' - - black>=20.8b0 ; extra == 'all' - - click>=7.0 ; extra == 'all' - - crosshair-tool>=0.0.102 ; extra == 'all' - - django>=4.2 ; extra == 'all' - - dpcontracts>=0.4 ; extra == 'all' - - hypothesis-crosshair>=0.0.27 ; extra == 'all' - - lark>=0.10.1 ; extra == 'all' - - libcst>=0.3.16 ; extra == 'all' - - numpy>=1.21.6 ; extra == 'all' - - pandas>=1.1 ; extra == 'all' - - pytest>=4.6 ; extra == 'all' - - python-dateutil>=1.4 ; extra == 'all' - - pytz>=2014.1 ; extra == 'all' - - redis>=3.0.0 ; extra == 'all' - - rich>=9.0.0 ; extra == 'all' - - tzdata>=2026.1 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') - - watchdog>=4.0.0 ; extra == 'all' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl name: imagesize version: 2.0.0 sha256: 5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96 requires_python: '>=3.10,<3.15' +- pypi: https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cusolver + version: 12.0.4.66 + sha256: 0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112 + requires_dist: + - nvidia-cublas + - nvidia-nvjitlink + - nvidia-cusparse + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: pydantic-core version: 2.46.4 @@ -13549,11 +13844,33 @@ packages: - isort ; extra == 'dev' - pip-tools ; extra == 'dev' - pytest ; extra == 'dev' +- pypi: https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl + name: filelock + version: 3.29.7 + sha256: 987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: rpds-py version: 0.30.0 sha256: 47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/60/62/ef188bdbd0c455c63f5ee399386fb01675a65b50f15c1e1b49949fb9b662/polars_bio-0.28.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: polars-bio + version: 0.28.0 + sha256: 3cbff526d4af4d06bef3bd25f70b0c77a12c95b4e578a4831199cd92c79f85e7 + requires_dist: + - polars>=1.37.1 + - pyarrow>=21.0.0,<23 ; python_full_version < '3.14' + - pyarrow>=22.0.0,<23 ; python_full_version >= '3.14' + - datafusion>=50.0.0,<51 + - tqdm>=4.67.0,<5 + - typing-extensions>=4.14.0,<5 + - polars-config-meta>=0.3.0,<1 + - pandas ; extra == 'pandas' + - pysam ; extra == 'test' + - bioframe ; extra == 'viz' + - matplotlib ; extra == 'viz' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl name: comm version: 0.2.3 @@ -13561,13 +13878,6 @@ packages: requires_dist: - pytest ; extra == 'test' requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/63/2f/ed68a7ee0f76b20b3d8ea3c3dcb06f9c2a2725a95c83cbe0eee65939e750/awkward_cpp-52-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: awkward-cpp - version: '52' - sha256: cd7c09f29781805dc050329418168cc7568e0837d0adb2d523b7c3f45c187b90 - requires_dist: - - numpy>=1.21.3 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: zstandard version: 0.25.0 @@ -13695,6 +14005,11 @@ packages: - multidict>=4.0 - propcache>=0.2.1 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl + name: nvidia-nccl-cu13 + version: 2.29.7 + sha256: edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl name: nbconvert version: 7.17.1 @@ -13852,6 +14167,13 @@ packages: - coloredlogs - click requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-cudnn-cu13 + version: 9.20.0.48 + sha256: 0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304 + requires_dist: + - nvidia-cublas + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-nccl-cu12 version: 2.27.5 @@ -13866,6 +14188,11 @@ packages: - polars>=1.30 ; extra == 'polars' - pyarrow>=22.0.0 ; extra == 'pyarrow' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl + name: aiohappyeyeballs + version: 2.7.1 + sha256: 9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl name: beartype version: 0.22.9 @@ -14048,11 +14375,6 @@ packages: version: 12.6.77 sha256: 35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53 requires_python: '>=3' -- pypi: https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl - name: platformdirs - version: 4.9.6 - sha256: e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/77/39/4d8414260c3d83f22029a39e51553c173611b378d62ca391e5ca68e65cfa/awkward-2.9.0-py3-none-any.whl name: awkward version: 2.9.0 @@ -14116,22 +14438,6 @@ packages: version: 1.1.0 sha256: 1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - name: cuda-bindings - version: 12.9.4 - sha256: 4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5 - requires_dist: - - cuda-pathfinder~=1.1 - - nvidia-cuda-nvcc-cu12 ; extra == 'all' - - nvidia-cuda-nvrtc-cu12 ; extra == 'all' - - nvidia-nvjitlink-cu12>=12.3 ; extra == 'all' - - nvidia-cufile-cu12 ; sys_platform == 'linux' and extra == 'all' - - cython>=3.1,<3.2 ; extra == 'test' - - setuptools>=77.0.0 ; extra == 'test' - - numpy>=1.21.1 ; extra == 'test' - - pytest>=6.2.4 ; extra == 'test' - - pytest-benchmark>=3.4.1 ; extra == 'test' - - pyglet>=2.1.9 ; extra == 'test' - pypi: https://files.pythonhosted.org/packages/7a/f6/51d8a97116de23c9280c1fa3b813bc088f8571ce5936ba84af1ecf13ed45/pybigwig-0.3.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: pybigwig version: 0.3.25 @@ -14147,26 +14453,40 @@ packages: version: 1.5.3 sha256: 5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/7c/1b/477e180dd01e407dbddb8a18fad670b743c6d3f563b2baffa737b8c5f844/awkward_cpp-52-cp310-cp310-macosx_11_0_arm64.whl - name: awkward-cpp - version: '52' - sha256: 610cec406c3f7c214e3e7673aaf55b5963e6c9695b97bd687b2f0f450ca66b64 - requires_dist: - - numpy>=1.21.3 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl name: polars-runtime-32 version: 1.37.1 sha256: c682bf83f5f352e5e02f5c16c652c48ca40442f07b236f30662b22217320ce76 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl - name: alabaster - version: 1.0.0 - sha256: fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b - requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - name: urllib3 - version: 2.7.0 +- pypi: https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl + name: narwhals + version: 2.24.0 + sha256: 42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489 + requires_dist: + - cudf-cu12>=24.10.0 ; sys_platform == 'linux' and extra == 'cudf' + - dask[dataframe]>=2024.8 ; extra == 'dask' + - duckdb>=1.1 ; extra == 'duckdb' + - ibis-framework>=6.0.0 ; extra == 'ibis' + - packaging>=21.3 ; extra == 'ibis' + - pyarrow-hotfix>=0.7 ; extra == 'ibis' + - modin>=0.22.0 ; extra == 'modin' + - pandas>=1.3.4 ; extra == 'pandas' + - polars>=0.20.4 ; extra == 'polars' + - pyarrow>=13.0.0 ; extra == 'pyarrow' + - pyspark>=3.5.0 ; extra == 'pyspark' + - pyspark[connect]>=3.5.0 ; extra == 'pyspark-connect' + - narwhals[duckdb] ; extra == 'sql' + - sqlparse>=0.5.5 ; extra == 'sql' + - sqlframe>=3.22.0,!=3.39.3 ; extra == 'sqlframe' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl + name: alabaster + version: 1.0.0 + sha256: fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + name: urllib3 + version: 2.7.0 sha256: 9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 requires_dist: - brotli>=1.2.0 ; platform_python_implementation == 'CPython' and extra == 'brotli' @@ -14321,10 +14641,10 @@ packages: version: 0.1.4 sha256: 502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c requires_python: '>=3.6' -- pypi: https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl - name: filelock - version: 3.29.0 - sha256: 96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258 +- pypi: https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl + name: platformdirs + version: 4.10.0 + sha256: fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl name: rich @@ -14464,15 +14784,6 @@ packages: - testpath ; extra == 'test' - xmltodict ; extra == 'test' requires_python: '>=3.10.0' -- pypi: https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-cusolver-cu12 - version: 11.7.3.90 - sha256: 4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450 - requires_dist: - - nvidia-cublas-cu12 - - nvidia-nvjitlink-cu12 - - nvidia-cusparse-cu12 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/85/dc/bf8a9b7e289dd9b0b550b9964786231fe48264583eecd733f7ab77b374b7/ncls-0.0.70-cp310-cp310-macosx_11_0_arm64.whl name: ncls version: 0.0.70 @@ -14511,26 +14822,102 @@ packages: - typing-extensions>=4.6.0 ; python_full_version < '3.13' - pytest>=6 ; extra == 'test' requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - name: triton - version: 3.6.0 - sha256: a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea +- pypi: https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl + name: pytorch-lightning + version: 2.6.5 + sha256: 62d9c8549b2278fedc3364f0a5607a56c6063d18635008f8cf3fae8d802b0d76 requires_dist: - - importlib-metadata ; python_full_version < '3.10' - - cmake>=3.20,<4.0 ; extra == 'build' - - lit ; extra == 'build' - - autopep8 ; extra == 'tests' - - isort ; extra == 'tests' - - numpy ; extra == 'tests' - - pytest ; extra == 'tests' - - pytest-forked ; extra == 'tests' - - pytest-xdist ; extra == 'tests' - - scipy>=1.7.1 ; extra == 'tests' - - llnl-hatchet ; extra == 'tests' - - matplotlib ; extra == 'tutorials' - - pandas ; extra == 'tutorials' - - tabulate ; extra == 'tutorials' - requires_python: '>=3.10,<3.15' + - torch>=2.1.0 + - tqdm>=4.57.0 + - pyyaml>5.4 + - fsspec[http]>=2022.5.0 + - torchmetrics>0.7.0 + - packaging>=23.0 + - typing-extensions>4.5.0 + - lightning-utilities>=0.10.0 + - coverage==7.13.4 ; python_full_version >= '3.10' and extra == 'test' + - coverage==7.10.7 ; python_full_version < '3.10' and extra == 'test' + - pytest==9.0.2 ; extra == 'test' + - pytest-cov==7.0.0 ; extra == 'test' + - pytest-timeout==2.4.0 ; extra == 'test' + - pytest-rerunfailures==16.0.1 ; python_full_version < '3.10' and extra == 'test' + - pytest-rerunfailures==16.1 ; python_full_version >= '3.10' and extra == 'test' + - pytest-random-order==1.2.0 ; extra == 'test' + - cloudpickle>=1.3 ; extra == 'test' + - scikit-learn>0.22.1 ; extra == 'test' + - numpy>1.21.0 ; python_full_version < '3.12' and extra == 'test' + - numpy>2.1.0 ; python_full_version >= '3.12' and extra == 'test' + - onnx>1.12.0 ; extra == 'test' + - onnxruntime>=1.12.0 ; extra == 'test' + - onnxscript>=0.1.0 ; extra == 'test' + - onnx-ir<0.1.16 ; extra == 'test' + - psutil<7.3.0 ; extra == 'test' + - pandas>2.0 ; extra == 'test' + - fastapi ; extra == 'test' + - uvicorn ; extra == 'test' + - tensorboard>=2.11 ; extra == 'test' + - huggingface-hub ; extra == 'test' + - requests<2.33.0 ; extra == 'examples' + - torchvision>=0.16.0 ; extra == 'examples' + - ipython[all]>=8.0.0 ; extra == 'examples' + - torchmetrics>=0.10.0 ; extra == 'examples' + - matplotlib>3.1 ; extra == 'extra' + - omegaconf>=2.2.3 ; extra == 'extra' + - hydra-core>=1.2.0 ; extra == 'extra' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'extra' + - rich>=12.3.0 ; extra == 'extra' + - tensorboardx>=2.2 ; extra == 'extra' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'extra' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'strategies' + - torch-tensorrt ; python_full_version >= '3.12' and sys_platform != 'darwin' and extra == 'test-gpu' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'deepspeed' + - matplotlib>3.1 ; extra == 'all' + - omegaconf>=2.2.3 ; extra == 'all' + - hydra-core>=1.2.0 ; extra == 'all' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'all' + - rich>=12.3.0 ; extra == 'all' + - tensorboardx>=2.2 ; extra == 'all' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'all' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'all' + - requests<2.33.0 ; extra == 'all' + - torchvision>=0.16.0 ; extra == 'all' + - ipython[all]>=8.0.0 ; extra == 'all' + - torchmetrics>=0.10.0 ; extra == 'all' + - matplotlib>3.1 ; extra == 'dev' + - omegaconf>=2.2.3 ; extra == 'dev' + - hydra-core>=1.2.0 ; extra == 'dev' + - jsonargparse[jsonnet,signatures]>=4.39.0 ; extra == 'dev' + - rich>=12.3.0 ; extra == 'dev' + - tensorboardx>=2.2 ; extra == 'dev' + - bitsandbytes>=0.45.2 ; sys_platform != 'darwin' and extra == 'dev' + - deepspeed>=0.15.0,<0.17.0 ; sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'dev' + - requests<2.33.0 ; extra == 'dev' + - torchvision>=0.16.0 ; extra == 'dev' + - ipython[all]>=8.0.0 ; extra == 'dev' + - torchmetrics>=0.10.0 ; extra == 'dev' + - coverage==7.13.4 ; python_full_version >= '3.10' and extra == 'dev' + - coverage==7.10.7 ; python_full_version < '3.10' and extra == 'dev' + - pytest==9.0.2 ; extra == 'dev' + - pytest-cov==7.0.0 ; extra == 'dev' + - pytest-timeout==2.4.0 ; extra == 'dev' + - pytest-rerunfailures==16.0.1 ; python_full_version < '3.10' and extra == 'dev' + - pytest-rerunfailures==16.1 ; python_full_version >= '3.10' and extra == 'dev' + - pytest-random-order==1.2.0 ; extra == 'dev' + - cloudpickle>=1.3 ; extra == 'dev' + - scikit-learn>0.22.1 ; extra == 'dev' + - numpy>1.21.0 ; python_full_version < '3.12' and extra == 'dev' + - numpy>2.1.0 ; python_full_version >= '3.12' and extra == 'dev' + - onnx>1.12.0 ; extra == 'dev' + - onnxruntime>=1.12.0 ; extra == 'dev' + - onnxscript>=0.1.0 ; extra == 'dev' + - onnx-ir<0.1.16 ; extra == 'dev' + - psutil<7.3.0 ; extra == 'dev' + - pandas>2.0 ; extra == 'dev' + - fastapi ; extra == 'dev' + - uvicorn ; extra == 'dev' + - tensorboard>=2.11 ; extra == 'dev' + - huggingface-hub ; extra == 'dev' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl name: accessible-pygments version: 0.0.5 @@ -14546,6 +14933,25 @@ packages: - hypothesis ; extra == 'tests' - pytest ; extra == 'tests' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/8d/78/831596c30d1520a89b367fde4857d2e51aade8fad072616a076e6c441900/seqpro-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: seqpro + version: 0.21.1 + sha256: 6b6720c75abff0f9e5826c4e6c09000d988b0b738a81172fdf226f6763b01170 + requires_dist: + - numba>=0.58.1 + - numpy>=1.26.0 + - polars>=1.21.0,<2 + - pyranges>=0.1.3,<0.2 + - pandera>=0.31.1 + - pandas + - pyarrow + - natsort + - narwhals>=2.20.0 + - setuptools>=70 + - awkward>=2.5.0 + - polars-config-meta[polars]>=0.3.2 + - attrs + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl name: pure-eval version: 0.2.3 @@ -14595,13 +15001,6 @@ packages: - doit>=0.36.0 ; extra == 'dev' - pydevtool ; extra == 'dev' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/8e/85/f4f06a30e63bbbaa42685122b7b0718c43ccb37b6e58f8a160b563b6fc37/selectolax-0.4.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: selectolax - version: 0.4.8 - sha256: 239fb539f67a43cbd543d48a09314eb955b5f3fe1b9ede4e1a31feb7ccfdd6b0 - requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' - pypi: https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-cufft-cu12 version: 11.3.0.4 @@ -14609,6 +15008,49 @@ packages: requires_dist: - nvidia-nvjitlink-cu12 requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/8f/fe/a4867bc2b8b81d9b1648992fb7e4a732b3db480ff2d02df2c7b59189c812/hypothesis-6.156.6-cp310-cp310-macosx_11_0_arm64.whl + name: hypothesis + version: 6.156.6 + sha256: 583a658162ee7e1a82155e3f146932a3a2269030294f3c83f5cf52fcaa0562c3 + requires_dist: + - exceptiongroup>=1.0.0 ; python_full_version < '3.11' + - sortedcontainers>=2.1.0,<3.0.0 + - black>=20.8b0 ; extra == 'all' + - click>=7.0 ; extra == 'all' + - crosshair-tool>=0.0.107 ; extra == 'all' + - django>=5.2 ; extra == 'all' + - dpcontracts>=0.4 ; extra == 'all' + - hypothesis-crosshair>=0.0.28 ; extra == 'all' + - lark>=0.10.1 ; extra == 'all' + - libcst>=0.3.16 ; extra == 'all' + - numpy>=1.21.6 ; extra == 'all' + - pandas>=1.1 ; extra == 'all' + - pytest>=4.6 ; extra == 'all' + - python-dateutil>=1.4 ; extra == 'all' + - pytz>=2014.1 ; extra == 'all' + - redis>=3.0.0 ; extra == 'all' + - rich>=9.0.0 ; extra == 'all' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32' and extra == 'all') + - watchdog>=4.0.0 ; extra == 'all' + - click>=7.0 ; extra == 'cli' + - black>=20.8b0 ; extra == 'cli' + - rich>=9.0.0 ; extra == 'cli' + - libcst>=0.3.16 ; extra == 'codemods' + - hypothesis-crosshair>=0.0.28 ; extra == 'crosshair' + - crosshair-tool>=0.0.107 ; extra == 'crosshair' + - python-dateutil>=1.4 ; extra == 'dateutil' + - django>=5.2 ; extra == 'django' + - dpcontracts>=0.4 ; extra == 'dpcontracts' + - black>=20.8b0 ; extra == 'ghostwriter' + - lark>=0.10.1 ; extra == 'lark' + - numpy>=1.21.6 ; extra == 'numpy' + - pandas>=1.1 ; extra == 'pandas' + - pytest>=4.6 ; extra == 'pytest' + - pytz>=2014.1 ; extra == 'pytz' + - redis>=3.0.0 ; extra == 'redis' + - watchdog>=4.0.0 ; extra == 'watchdog' + - tzdata>=2026.2 ; (sys_platform == 'emscripten' and extra == 'zoneinfo') or (sys_platform == 'win32' and extra == 'zoneinfo') + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl name: typeguard version: 4.5.1 @@ -14627,16 +15069,6 @@ packages: - coloredlogs - click requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/91/fc/861c6c2322a1202eb35eb24cb3b520dc0015b7624c1a39a45f48757ed034/vcfixture-0.5.0-py3-none-any.whl - name: vcfixture - version: 0.5.0 - sha256: 9d9823022e11e8d20b3191046627dbaf0be21a7db578f49636089fdb70cbe3a7 - requires_dist: - - numpy>=1.24 - - pysam>=0.22 - - hypothesis>=6.100 - - typing-extensions>=4.4 - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl name: pyzmq version: 27.1.0 @@ -14920,62 +15352,13 @@ packages: requires_dist: - cython ; extra == 'cython' requires_python: '>=3.9,<3.15' -- pypi: https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl - name: setuptools - version: 82.0.1 - sha256: a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb +- pypi: https://files.pythonhosted.org/packages/9d/66/47cf0a44c768792154c665eedeb6a33201d89d75e5fba62c7b4b585d08c4/awkward_cpp-54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: awkward-cpp + version: '54' + sha256: 0c9fae1b68b9821b375e6da9de00b90387509c1ec0be8c3b19e7d0d8f54a79b0 requires_dist: - - pytest>=6,!=8.1.* ; extra == 'test' - - virtualenv>=13.0.0 ; extra == 'test' - - wheel>=0.44.0 ; extra == 'test' - - pip>=19.1 ; extra == 'test' - - packaging>=24.2 ; extra == 'test' - - jaraco-envs>=2.2 ; extra == 'test' - - pytest-xdist>=3 ; extra == 'test' - - jaraco-path>=3.7.2 ; extra == 'test' - - build[virtualenv]>=1.0.3 ; extra == 'test' - - filelock>=3.4.0 ; extra == 'test' - - ini2toml[lite]>=0.14 ; extra == 'test' - - tomli-w>=1.0.0 ; extra == 'test' - - pytest-timeout ; extra == 'test' - - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' - - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' - - pytest-home>=0.5 ; extra == 'test' - - pytest-subprocess ; extra == 'test' - - pyproject-hooks!=1.1 ; extra == 'test' - - jaraco-test>=5.5 ; extra == 'test' - - sphinx>=3.5 ; extra == 'doc' - - jaraco-packaging>=9.3 ; extra == 'doc' - - rst-linker>=1.9 ; extra == 'doc' - - furo ; extra == 'doc' - - sphinx-lint ; extra == 'doc' - - jaraco-tidelift>=1.4 ; extra == 'doc' - - pygments-github-lexers==0.0.5 ; extra == 'doc' - - sphinx-favicon ; extra == 'doc' - - sphinx-inline-tabs ; extra == 'doc' - - sphinx-reredirects ; extra == 'doc' - - sphinxcontrib-towncrier ; extra == 'doc' - - sphinx-notfound-page>=1,<2 ; extra == 'doc' - - pyproject-hooks!=1.1 ; extra == 'doc' - - towncrier<24.7 ; extra == 'doc' - - packaging>=24.2 ; extra == 'core' - - more-itertools>=8.8 ; extra == 'core' - - jaraco-text>=3.7 ; extra == 'core' - - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' - - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' - - wheel>=0.43.0 ; extra == 'core' - - jaraco-functools>=4 ; extra == 'core' - - more-itertools ; extra == 'core' - - pytest-checkdocs>=2.4 ; extra == 'check' - - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' - - ruff>=0.13.0 ; sys_platform != 'cygwin' and extra == 'check' - - pytest-cov ; extra == 'cover' - - pytest-enabler>=2.2 ; extra == 'enabler' - - pytest-mypy ; extra == 'type' - - mypy==1.18.* ; extra == 'type' - - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' - - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' - requires_python: '>=3.9' + - numpy>=1.21.3 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl name: nvidia-nvjitlink-cu12 version: 12.6.85 @@ -15087,6 +15470,18 @@ packages: version: 1.6.0 sha256: 87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c requires_python: '>=3.5' +- pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + name: requests + version: 2.34.2 + sha256: 2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 + requires_dist: + - charset-normalizer>=2,<4 + - idna>=2.5,<4 + - urllib3>=1.26,<3 + - certifi>=2023.5.7 + - pysocks>=1.5.6,!=1.5.7 ; extra == 'socks' + - chardet>=3.0.2,<8 ; extra == 'use-chardet-on-py3' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl name: sympy version: 1.14.0 @@ -15096,11 +15491,13 @@ packages: - pytest>=7.1.0 ; extra == 'dev' - hypothesis>=6.70.0 ; extra == 'dev' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-nvtx-cu12 - version: 12.8.90 - sha256: 5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f - requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl + name: arro3-core + version: 0.8.1 + sha256: 0501662cb3c7c6a05dcd6b998db55f9a97743aa571da42203733366c42166a7a + requires_dist: + - typing-extensions ; python_full_version < '3.12' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl name: watchfiles version: 1.1.1 @@ -15108,6 +15505,11 @@ packages: requires_dist: - anyio>=3.0.0 requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl + name: nvidia-curand + version: 10.4.0.35 + sha256: 1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl name: semver version: 3.0.4 @@ -15132,6 +15534,24 @@ packages: - pydoctor>=25.4.0 ; extra == 'docs' - pytest ; extra == 'test' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + name: cuda-bindings + version: 13.3.1 + sha256: 9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0 + requires_dist: + - cuda-pathfinder>=1.4.2 + - cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm]==13.* ; extra == 'all' + - cuda-toolkit[cufile]==13.* ; sys_platform == 'linux' and extra == 'all' + - cuda-toolkit==13.* ; extra == 'all' + - nvidia-cudla==13.* ; platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cufft + version: 12.0.0.61 + sha256: 6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3 + requires_dist: + - nvidia-nvjitlink + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl name: nbformat version: 5.10.4 @@ -15243,6 +15663,11 @@ packages: - pytest-cov ; extra == 'test' - coverage-conditional-plugin ; extra == 'test' requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl + name: charset-normalizer + version: 3.4.9 + sha256: cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl name: pyarrow version: 21.0.0 @@ -15254,13 +15679,6 @@ packages: - pytz ; extra == 'test' - pandas ; extra == 'test' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl - name: click - version: 8.3.3 - sha256: a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613 - requires_dist: - - colorama ; sys_platform == 'win32' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl name: pydantic-core version: 2.46.4 @@ -15400,6 +15818,45 @@ packages: - pytest-timeout ; extra == 'testing' - requests ; extra == 'testing' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/b4/f6/1c671874560a70d902dd57028a75411c176910de2df3d6a6a533de424131/cyclopts-4.21.0-py3-none-any.whl + name: cyclopts + version: 4.21.0 + sha256: ded3ddb15b0c815f44d245011fc4cdd5a4809a3bb8202869e9e02195a87c0e18 + requires_dist: + - attrs>=23.1.0 + - docstring-parser>=0.15,<4.0 + - rich-rst>=1.3.1,<3.0.0 + - rich>=13.6.0 + - tomli>=2.0.0 ; python_full_version < '3.11' + - typing-extensions>=4.8.0 ; python_full_version < '3.11' + - ipdb>=0.13.9 ; extra == 'debug' + - line-profiler>=3.5.1 ; extra == 'debug' + - coverage[toml]>=5.1 ; extra == 'dev' + - mkdocs>=1.4.0 ; extra == 'dev' + - pexpect>=4.9.0 ; sys_platform != 'win32' and extra == 'dev' + - pre-commit>=2.16.0 ; extra == 'dev' + - pydantic>=2.11.2,<3.0.0 ; extra == 'dev' + - pytest-cov>=3.0.0 ; extra == 'dev' + - pytest-mock>=3.7.0 ; extra == 'dev' + - pytest>=8.2.0 ; extra == 'dev' + - pyyaml>=6.0.1 ; extra == 'dev' + - syrupy>=4.0.0 ; extra == 'dev' + - toml>=0.10.2,<1.0.0 ; extra == 'dev' + - trio>=0.10.0 ; extra == 'dev' + - gitpython>=3.1.31 ; extra == 'docs' + - myst-parser[linkify]>=3.0.1,<5.0.0 ; extra == 'docs' + - sphinx-autodoc-typehints>=1.25.2,<4.0.0 ; extra == 'docs' + - sphinx-copybutton>=0.5,<1.0 ; extra == 'docs' + - sphinx-rtd-dark-mode>=1.3.0,<2.0.0 ; extra == 'docs' + - sphinx-rtd-theme>=3.0.0,<4.0.0 ; extra == 'docs' + - sphinx>=7.4.7,<8.2.0 ; extra == 'docs' + - markdown>=3.3 ; extra == 'mkdocs' + - mkdocs>=1.4.0 ; extra == 'mkdocs' + - pymdown-extensions>=10.0 ; extra == 'mkdocs' + - tomli>=2.0.0 ; python_full_version < '3.11' and extra == 'toml' + - trio>=0.10.0 ; extra == 'trio' + - pyyaml>=6.0.1 ; extra == 'yaml' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: nvidia-nvshmem-cu12 version: 3.4.5 @@ -15578,11 +16035,6 @@ packages: - cffi~=1.17 ; python_full_version < '3.14' and platform_python_implementation != 'PyPy' and extra == 'cffi' - cffi>=2.0.0b0 ; python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and extra == 'cffi' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cufile-cu12 - version: 1.13.1.3 - sha256: 1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/bf/0f/f1c2e2592f995d301bc06901fbb2e947cbe4889397a418309710ccc3a1db/sphinx_autodoc_typehints-3.10.2-py3-none-any.whl name: sphinx-autodoc-typehints version: 3.10.2 @@ -15632,12 +16084,15 @@ packages: - flake8 ; extra == 'test' - mypy ; extra == 'test' requires_python: '>=3.5' -- pypi: https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cusparse-cu12 - version: 12.5.8.93 - sha256: 1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b - requires_dist: - - nvidia-nvjitlink-cu12 +- pypi: https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl + name: nvidia-nvtx + version: 13.0.85 + sha256: 4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4 + requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + name: nvidia-cuda-nvrtc + version: 13.0.88 + sha256: ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575 requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/c3/a2/c7f6ebf546f8f644edf0f999aa98ece106986a77a7b922316bf6414ff825/torchmetrics-1.9.0-py3-none-any.whl name: torchmetrics @@ -15805,6 +16260,13 @@ packages: version: 2.4.4 sha256: 23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/c6/1d/c550ead60f8d442f9e7d209d99b6c50afcf84da9e836148a84fb9df7b299/awkward_cpp-54-cp310-cp310-macosx_11_0_arm64.whl + name: awkward-cpp + version: '54' + sha256: 38c4d810a7ac7a8ed185a90a7e252e8bb240db20628284ef7a2b7bc8e046fda8 + requires_dist: + - numpy>=1.21.3 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/c6/3e/48599a38009ca60ff82a6f38c8a621ce3c0286aa7397c7d79e741bd9060e/polars_runtime_32-1.40.1-cp310-abi3-macosx_11_0_arm64.whl name: polars-runtime-32 version: 1.40.1 @@ -15844,6 +16306,67 @@ packages: version: 3.0.1 sha256: 6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064 requires_python: '!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl + name: pandera + version: 0.32.1 + sha256: 1a17a3ffa906174d19207715f4f082ec3db3709647927ad8c095c147d74d8454 + requires_dist: + - packaging>=20.0 + - pydantic + - typeguard + - typing-extensions + - typing-inspect>=0.6.0 + - numpy>=1.24.4 ; extra == 'pandas' + - pandas>=2.1.1 ; extra == 'pandas' + - hypothesis>=6.92.7 ; extra == 'strategies' + - scipy ; extra == 'hypotheses' + - pyyaml>=5.1 ; extra == 'io' + - frictionless<=4.40.8 ; extra == 'frictionless' + - pandas-stubs ; extra == 'mypy' + - scipy-stubs ; python_full_version >= '3.10' and extra == 'mypy' + - fastapi ; extra == 'fastapi' + - geopandas ; extra == 'geopandas' + - shapely ; extra == 'geopandas' + - pyspark[connect]>=3.2.0 ; extra == 'pyspark' + - modin ; extra == 'modin' + - ray ; extra == 'modin' + - dask[dataframe] ; extra == 'modin' + - distributed ; extra == 'modin' + - modin ; extra == 'modin-ray' + - ray ; extra == 'modin-ray' + - modin ; extra == 'modin-dask' + - dask[dataframe] ; extra == 'modin-dask' + - distributed ; extra == 'modin-dask' + - dask[dataframe] ; extra == 'dask' + - distributed ; extra == 'dask' + - ibis-framework>=9.0.0 ; extra == 'ibis' + - pyarrow-hotfix ; extra == 'ibis' + - polars>=0.20.0 ; extra == 'polars' + - xarray>=2024.10.0 ; extra == 'xarray' + - numpy>=1.24.4 ; extra == 'xarray' + - narwhals>=1.26.0 ; extra == 'narwhals' + - hypothesis>=6.92.7 ; extra == 'all' + - scipy ; extra == 'all' + - scipy-stubs ; python_full_version >= '3.10' and extra == 'all' + - pyyaml>=5.1 ; extra == 'all' + - black ; extra == 'all' + - frictionless<=4.40.8 ; extra == 'all' + - pyspark[connect]>=3.2.0 ; extra == 'all' + - modin ; extra == 'all' + - ray ; extra == 'all' + - dask[dataframe] ; extra == 'all' + - distributed ; extra == 'all' + - pandas-stubs ; extra == 'all' + - fastapi ; extra == 'all' + - geopandas ; extra == 'all' + - shapely ; extra == 'all' + - ibis-framework>=9.0.0 ; extra == 'all' + - pyarrow-hotfix ; extra == 'all' + - polars>=0.20.0 ; extra == 'all' + - xarray>=2024.10.0 ; extra == 'all' + - numpy>=1.24.4 ; extra == 'all' + - narwhals>=1.26.0 ; extra == 'all' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl name: pyarrow version: 21.0.0 @@ -15873,6 +16396,23 @@ packages: - pytest-benchmark ; extra == 'devel' - pytest-cache ; extra == 'devel' - validictory ; extra == 'devel' +- pypi: https://files.pythonhosted.org/packages/cc/67/c1d029268051e6cfd973cd4558baee7688808920c9aa0d879a2fd4672ba8/polars_bio-0.28.0-cp39-abi3-macosx_11_0_arm64.whl + name: polars-bio + version: 0.28.0 + sha256: f66e12c5e3f5fccc2f6b7124f348f6ce47fcae0be18b21dd2c11ea6350b0a55f + requires_dist: + - polars>=1.37.1 + - pyarrow>=21.0.0,<23 ; python_full_version < '3.14' + - pyarrow>=22.0.0,<23 ; python_full_version >= '3.14' + - datafusion>=50.0.0,<51 + - tqdm>=4.67.0,<5 + - typing-extensions>=4.14.0,<5 + - polars-config-meta>=0.3.0,<1 + - pandas ; extra == 'pandas' + - pysam ; extra == 'test' + - bioframe ; extra == 'viz' + - matplotlib ; extra == 'viz' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: contourpy version: 1.3.3 @@ -15933,11 +16473,6 @@ packages: - fsspec>=2025.5.1 - typing-extensions ; python_full_version < '3.10' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl - name: tzdata - version: '2026.2' - sha256: bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 - requires_python: '>=2' - pypi: https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl name: pandas version: 3.0.3 @@ -16045,6 +16580,75 @@ packages: version: 0.24.0 sha256: b9445a4c3be5ed4b60202690af3890a444452276372e3abb58564308cc6d5a45 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl + name: cuda-toolkit + version: 13.0.3.0 + sha256: d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f + requires_dist: + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cccl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-culibos==13.0.85.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cupti==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cuxxfilt==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-nvcc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-opencl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-profiler-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-sanitizer-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cufft==12.0.0.61.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cufile==1.15.1.6.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-curand==10.4.0.35.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cusolver==12.0.4.66.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-npp==13.0.1.2.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvfatbin==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvjpeg==13.0.1.86.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvml-dev==13.0.87.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvptxcompiler==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvtx==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') + - nvidia-cuda-cccl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cccl') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cccl') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cccl') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'crt') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'crt') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'crt') + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cublas') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cublas') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cublas') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cublas') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cublas') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cublas') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cudart') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cudart') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cudart') + - nvidia-cufft==12.0.0.61.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cufft') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufft') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufft') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cufft') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufft') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufft') + - nvidia-cufile==1.15.1.6.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cufile') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cufile') + - nvidia-cuda-culibos==13.0.85.* ; (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'culibos') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'culibos') + - nvidia-cuda-cupti==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cupti') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cupti') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cupti') + - nvidia-curand==10.4.0.35.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'curand') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'curand') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'curand') + - nvidia-cublas==13.1.1.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusolver==12.0.4.66.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusolver') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusolver') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusolver') + - nvidia-cusparse==12.6.3.3.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusparse') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusparse') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusparse') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cusparse') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cusparse') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cusparse') + - nvidia-cuda-cuxxfilt==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'cuxxfilt') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cuxxfilt') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'cuxxfilt') + - nvidia-npp==13.0.1.2.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'npp') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'npp') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'npp') + - nvidia-cuda-crt==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-cuda-nvcc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-cuda-runtime==13.0.96.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvcc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvcc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvcc') + - nvidia-nvfatbin==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvfatbin') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvfatbin') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvfatbin') + - nvidia-nvjitlink>=13.0.88,<14 ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvjitlink') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvjitlink') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvjitlink') + - nvidia-nvjpeg==13.0.1.86.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvjpeg') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvjpeg') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvjpeg') + - nvidia-nvml-dev==13.0.87.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvml') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvml') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvml') + - nvidia-nvptxcompiler==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvptxcompiler') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvptxcompiler') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvptxcompiler') + - nvidia-cuda-nvrtc==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvrtc') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvrtc') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvrtc') + - nvidia-nvtx==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvtx') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvtx') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvtx') + - nvidia-nvvm==13.0.88.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'nvvm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'nvvm') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'nvvm') + - nvidia-cuda-opencl==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'opencl') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'opencl') + - nvidia-cuda-profiler-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'profiler') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'profiler') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'profiler') + - nvidia-cuda-sanitizer-api==13.0.85.* ; (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'sanitizer') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'sanitizer') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'sanitizer') +- pypi: https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl + name: cuda-pathfinder + version: 1.5.6 + sha256: 7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/d2/89/b6d8c4ec1425b81ed588fa3ecc1b4ed5b9b8da21894eba5ac5d5288881b2/cyvcf2-0.32.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl name: cyvcf2 version: 0.32.1 @@ -16286,13 +16890,6 @@ packages: - trove-classifiers>=2024.10.12 ; extra == 'tests' - defusedxml ; extra == 'xmp' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/d8/b2/10b296fc0e1c3a7c058a39e424a66f7ae32cd9d414bb0433b8f28a4c0d38/pgenlib-0.94.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: pgenlib - version: 0.94.0 - sha256: 5a9c26fc0de6282c82d95697b08b9f5fba2473b048952d810c9d01e094058861 - requires_dist: - - numpy>=1.19.3 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/d9/11/81484d5ca1041b5c32fa1714c8862a2955fb15fbed3624963a3222eb9705/oxbow-0.5.2-cp39-abi3-macosx_11_0_arm64.whl name: oxbow version: 0.5.2 @@ -16312,6 +16909,11 @@ packages: requires_dist: - pygments requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl + name: pyarrow + version: 22.0.0 + sha256: 77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl name: anyio version: 4.13.0 @@ -16346,11 +16948,6 @@ packages: - pyarrow>=11.0.0 - typing-extensions ; python_full_version < '3.13' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-cublas-cu12 - version: 12.8.4.1 - sha256: 8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl name: typing-inspection version: 0.4.2 @@ -16476,13 +17073,119 @@ packages: - babel ; extra == 'i18n' - jinja2 ; extra == 'i18n' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/e6/1d/a8457a0fb898d9803aabdbe2028841f03889ba1d95771164c1bdce9fd1ef/selectolax-0.4.8-cp310-cp310-macosx_11_0_arm64.whl - name: selectolax - version: 0.4.8 - sha256: 4542a35f5ea993ae8bdb5850346f65848794afabc83976e576dcab07a6699020 +- pypi: https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl + name: fsspec + version: 2026.6.0 + sha256: 02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1 requires_dist: - - cython ; extra == 'cython' - requires_python: '>=3.9,<3.15' + - adlfs ; extra == 'abfs' + - adlfs ; extra == 'adl' + - pyarrow>=1 ; extra == 'arrow' + - dask ; extra == 'dask' + - distributed ; extra == 'dask' + - pre-commit ; extra == 'dev' + - ruff>=0.5 ; extra == 'dev' + - numpydoc ; extra == 'doc' + - sphinx ; extra == 'doc' + - sphinx-design ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - yarl ; extra == 'doc' + - dropbox ; extra == 'dropbox' + - dropboxdrivefs ; extra == 'dropbox' + - requests ; extra == 'dropbox' + - adlfs ; extra == 'full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'full' + - dask ; extra == 'full' + - distributed ; extra == 'full' + - dropbox ; extra == 'full' + - dropboxdrivefs ; extra == 'full' + - fusepy ; extra == 'full' + - gcsfs>2024.2.0 ; extra == 'full' + - libarchive-c ; extra == 'full' + - ocifs ; extra == 'full' + - panel ; extra == 'full' + - paramiko ; extra == 'full' + - pyarrow>=1 ; extra == 'full' + - pygit2 ; extra == 'full' + - requests ; extra == 'full' + - s3fs>2024.2.0 ; extra == 'full' + - smbprotocol ; extra == 'full' + - tqdm ; extra == 'full' + - fusepy ; extra == 'fuse' + - gcsfs>2024.2.0 ; extra == 'gcs' + - pygit2 ; extra == 'git' + - requests ; extra == 'github' + - gcsfs ; extra == 'gs' + - panel ; extra == 'gui' + - pyarrow>=1 ; extra == 'hdfs' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'http' + - libarchive-c ; extra == 'libarchive' + - ocifs ; extra == 'oci' + - s3fs>2024.2.0 ; extra == 's3' + - paramiko ; extra == 'sftp' + - smbprotocol ; extra == 'smb' + - paramiko ; extra == 'ssh' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test' + - numpy ; extra == 'test' + - pytest ; extra == 'test' + - pytest-asyncio!=0.22.0 ; extra == 'test' + - pytest-benchmark ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytest-recording ; extra == 'test' + - pytest-rerunfailures ; extra == 'test' + - requests ; extra == 'test' + - aiobotocore>=2.5.4,<3.0.0 ; extra == 'test-downstream' + - dask[dataframe,test] ; extra == 'test-downstream' + - moto[server]>4,<5 ; extra == 'test-downstream' + - pytest-timeout ; extra == 'test-downstream' + - xarray ; extra == 'test-downstream' + - adlfs ; extra == 'test-full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test-full' + - backports-zstd ; python_full_version < '3.14' and extra == 'test-full' + - cloudpickle ; extra == 'test-full' + - dask ; extra == 'test-full' + - distributed ; extra == 'test-full' + - dropbox ; extra == 'test-full' + - dropboxdrivefs ; extra == 'test-full' + - fastparquet ; extra == 'test-full' + - fusepy ; extra == 'test-full' + - gcsfs ; extra == 'test-full' + - jinja2 ; extra == 'test-full' + - kerchunk ; extra == 'test-full' + - libarchive-c ; extra == 'test-full' + - lz4 ; extra == 'test-full' + - notebook ; extra == 'test-full' + - numpy ; extra == 'test-full' + - ocifs ; extra == 'test-full' + - pandas<3.0.0 ; extra == 'test-full' + - panel ; extra == 'test-full' + - paramiko ; extra == 'test-full' + - pyarrow ; extra == 'test-full' + - pyarrow>=1 ; extra == 'test-full' + - pyftpdlib ; extra == 'test-full' + - pygit2 ; extra == 'test-full' + - pytest ; extra == 'test-full' + - pytest-asyncio!=0.22.0 ; extra == 'test-full' + - pytest-benchmark ; extra == 'test-full' + - pytest-cov ; extra == 'test-full' + - pytest-mock ; extra == 'test-full' + - pytest-recording ; extra == 'test-full' + - pytest-rerunfailures ; extra == 'test-full' + - python-snappy ; extra == 'test-full' + - requests ; extra == 'test-full' + - smbprotocol ; extra == 'test-full' + - tqdm ; extra == 'test-full' + - urllib3 ; extra == 'test-full' + - zarr<3.2.0 ; extra == 'test-full' + - zstandard ; python_full_version < '3.14' and extra == 'test-full' + - tqdm ; extra == 'tqdm' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + name: tzdata + version: '2026.3' + sha256: dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931 + requires_python: '>=2' - pypi: https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl name: tinycss2 version: 1.4.0 @@ -16531,6 +17234,16 @@ packages: - pytest-timeout ; extra == 'test' - pytest<9 ; extra == 'test' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + name: more-itertools + version: 11.1.0 + sha256: 4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.9 + sha256: 9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/e8/b3/7b0845952f65ecd3f02e8573ece4dd17c97ca27604cd28302c0467293e83/pgenlib-0.94.0-cp311-cp311-macosx_10_9_universal2.whl name: pgenlib version: 0.94.0 @@ -16656,6 +17369,23 @@ packages: name: pytz version: '2026.2' sha256: 04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 +- pypi: https://files.pythonhosted.org/packages/ed/d2/fdeb7e555891b1299405954365574e4fbc64ad448360d13cd933073c5b37/awkward-2.10.0-py3-none-any.whl + name: awkward + version: 2.10.0 + sha256: 808c9536790865cbbcde07cb3678caa86c0369c443a3212a4552ec10dd187140 + requires_dist: + - awkward-cpp==54 + - fsspec>=2022.11.0 + - importlib-metadata>=4.13.0 ; python_full_version < '3.12' + - numpy>=1.21.3 + - packaging + - typing-extensions>=4.1.0 ; python_full_version < '3.11' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl + name: certifi + version: 2026.6.17 + sha256: 2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db + requires_python: '>=3.7' - pypi: https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl name: natsort version: 8.4.0 @@ -16699,13 +17429,11 @@ packages: - nvidia-nvjitlink-cu12 - nvidia-cusparse-cu12 requires_python: '>=3' -- pypi: https://files.pythonhosted.org/packages/f1/26/2c4e3e57055d5c3460b353caa899a6af5b6e44b81425433b765529d72990/pgenlib-0.94.0-cp310-cp310-macosx_10_9_universal2.whl - name: pgenlib - version: 0.94.0 - sha256: ffaf1e9b8baa05da40213e12b950d4e612097d8fa81a921f2a2a797880b5f871 - requires_dist: - - numpy>=1.19.3 - requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + name: nvidia-nvjitlink + version: 13.3.33 + sha256: 26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5 + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl name: stack-data version: 0.6.3 @@ -16737,11 +17465,6 @@ packages: requires_dist: - colorama>=0.4.6 ; extra == 'windows-terminal' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - name: nvidia-nvjitlink-cu12 - version: 12.8.93 - sha256: 81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: scipy version: 1.18.0 @@ -16839,11 +17562,6 @@ packages: requires_dist: - numpy>=1.21.3 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - name: nvidia-cuda-cupti-cu12 - version: 12.8.90 - sha256: ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182 - requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl name: pyzmq version: 27.1.0 @@ -16851,6 +17569,13 @@ packages: requires_dist: - cffi ; implementation_name == 'pypy' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: nvidia-cusparse + version: 12.6.3.3 + sha256: 2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b + requires_dist: + - nvidia-nvjitlink + requires_python: '>=3' - pypi: https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl name: llvmlite version: 0.47.0 @@ -16864,11 +17589,13 @@ packages: - frozenlist>=1.1.0 - typing-extensions>=4.2 ; python_full_version < '3.13' requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl - name: nvidia-curand-cu12 - version: 10.3.9.90 - sha256: b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 - requires_python: '>=3' +- pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + name: click + version: 8.4.2 + sha256: e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + requires_dist: + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/fb/ec/b0c23ec7fc9df5af527b2d63f15a92699f7fd0515986763ed8e50489a755/ncls-0.0.70-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: ncls version: 0.0.70 diff --git a/pixi.toml b/pixi.toml index e05e09ed..d0cd35ec 100644 --- a/pixi.toml +++ b/pixi.toml @@ -19,6 +19,13 @@ splice = { features = ["notebook", "basenji2", "py310", "splice"] } [target.osx-arm64.activation.env] RUSTFLAGS = "-C link-arg=-rpath -C link-arg=$CONDA_PREFIX/lib" +# Prepend the env's lib dir so its libstdc++/libpython win over any system libs +# an HPC module may have placed earlier on LD_LIBRARY_PATH (e.g. an older gcc +# libstdc++ lacking GLIBCXX_3.4.30, which breaks llvmlite in spawned workers and +# libpython loading in `cargo test` binaries). +[target.linux-64.activation.env] +LD_LIBRARY_PATH = "$CONDA_PREFIX/lib:$LD_LIBRARY_PATH" + [dependencies] python = "<3.14" uv = "*" @@ -40,6 +47,7 @@ icecream = "*" hypothesis = "*" filelock = "*" memray = "*" +pyinstrument = "*" patchelf = "*" pyrefly = "*" @@ -50,7 +58,7 @@ plink2 = "*" genvarloader = { path = ".", editable = true, extras = ["cli"] } # Test-only: needed by the property/fixture tests across the whole py3xx matrix # (not a genvarloader runtime dep, so it must live here, not just in py310). -vcfixture = ">=0.5.0" +vcfixture = ">=0.6.0,<0.7" [feature.docs.dependencies] pandoc = "*" @@ -93,8 +101,15 @@ numba = "==0.59.1" [feature.py310.pypi-dependencies] pyarrow = ">=21" hirola = "==0.3" -seqpro = "==0.20.0" -genoray = "==2.12.3" +seqpro = "==0.21.1" +# Dev-wiring (release-gated): genoray MAIN (unreleased INFO/FORMAT field-read API) as a +# pre-built abi3 wheel. Editable path-install would require genoray's rust-htslib C toolchain +# (cxx-compiler/clangdev/zlib/LIBCLANG_PATH) inside this env; we never edit genoray, so a +# frozen wheel is equivalent. Rebuild via, in the genoray checkout: +# pixi run --manifest-path ci/wheel/pixi.toml build && ... repair # -> dist/*.whl +# The wheel reports 2.15.0 (genoray's version was never bumped past the 3.0.0 tag), hence the +# dropped version constraint in pyproject.toml. Flip both to the PyPI release at merge. +genoray = { path = "/carter/users/dlaub/projects/genoray/dist/genoray-2.15.0-cp310-abi3-manylinux_2_28_x86_64.whl" } polars = "==1.37.1" loguru = "*" natsort = "*" @@ -112,7 +127,7 @@ pooch = "*" awkward = "*" pydantic = ">=2,<3" hypothesis = "*" -vcfixture = ">=0.5.0" +vcfixture = ">=0.6.0,<0.7" filelock = "*" [feature.py311.dependencies] @@ -144,7 +159,7 @@ test-join-audit = { cmd = "pytest tests -p tests._join_audit_plugin", depends-on "gen", "gen-1kg", ] } -typecheck = { cmd = "pyrefly check" } +typecheck = { cmd = "pyrefly check python/genvarloader tests" } bench = { cmd = "pytest tests/benchmarks --codspeed -p no:cov" } bench-local = { cmd = "pytest tests/benchmarks --benchmark-only -p no:cov" } # perf on the Python process (NOT py-spy --native, which slows deep-stack paths ~10x). diff --git a/pyproject.toml b/pyproject.toml index 2e3a0e50..c037c802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ license = { file = "LICENSE.txt" } requires-python = ">=3.10,<3.14" # >= 3.14 blocked by pyarrow/genoray dependencies = [ "seqpro>=0.20", - "genoray>=2.12.3,<3", + # Unpinned for dev: gvl is being built against genoray main (unreleased field-read API), + # whose pyproject still reports 2.15.0. Re-pin (>=3,<4 or the new floor) at genoray release. + "genoray", "numpy", "loguru", "natsort", diff --git a/python/genvarloader/_dataset/_impl.py b/python/genvarloader/_dataset/_impl.py index ee2e7927..2bea3986 100644 --- a/python/genvarloader/_dataset/_impl.py +++ b/python/genvarloader/_dataset/_impl.py @@ -102,6 +102,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[MaybeRSEQ, MaybeRTRK]: ... @staticmethod @overload @@ -120,6 +121,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[RaggedSeqs, MaybeRTRK]: ... @staticmethod def open( @@ -137,6 +139,7 @@ def open( var_filter: Literal["exonic"] | None = None, *, svar: str | Path | None = None, + svar2: str | Path | None = None, ) -> RaggedDataset[MaybeRSEQ, MaybeRTRK]: """Open a dataset from a path. If no reference genome is provided, the dataset cannot yield sequences. Will initialize the dataset such that it will return tracks and haplotypes (reference sequences if no genotypes) if possible. @@ -179,6 +182,10 @@ def open( Override the recorded SVAR location. Use when the original SVAR has moved and the dataset cannot find it via the stored relative/absolute path or by sibling discovery. + svar2 + Override the recorded ``.svar2`` location. Use when the original + ``.svar2`` store has moved and the dataset cannot find it via the + stored relative/absolute path or by sibling discovery. """ from ._open import OpenRequest @@ -196,6 +203,7 @@ def open( splice_info=splice_info, var_filter=var_filter, svar=svar, + svar2=svar2, ).resolve() def with_settings( @@ -331,34 +339,47 @@ def with_settings( missing = list(set(var_fields) - set(self.available_var_fields)) if missing or not isinstance(self._seqs, Haps): raise ValueError(f"Missing variant fields: {missing}") - haps = to_evolve.get("_seqs", self._seqs) - # Discover custom FORMAT fields so we don't try to load them as INFO. - custom_fmt = _svar_format_fields(haps.variants.path.parent) - # Lazily load any newly-requested info columns into the existing - # _Variants struct (mutates haps.variants.info in place). - builtin = {"alt", "ilen", "start", "ref", "dosage"} - new_info_fields = [ - f - for f in var_fields - if f not in builtin - and f not in haps.variants.info - and f not in custom_fmt - ] - if new_info_fields: - haps.variants.load_info(new_info_fields) - # Lazily memmap dosages if newly requested. - if "dosage" in var_fields and haps.dosages is None: - haps = _lazy_load_dosages(self, haps) - # Lazily memmap custom FORMAT fields if newly requested. - new_custom_fields = { - f: custom_fmt[f] - for f in var_fields - if f in custom_fmt and f not in haps.var_field_data - } - if new_custom_fields: - haps = _lazy_load_custom_fields(self, haps, new_custom_fields) - haps = replace(haps, var_fields=var_fields) - to_evolve["_seqs"] = haps + + from ._svar2_haps import Svar2Haps + + if isinstance(self._seqs, Svar2Haps): + # SVAR2 field values are read on demand by the decode kernel + # (decode_variants_from_svar2_readbound); there is no SVAR1 variants + # table to lazily load INFO/dosage/custom-FORMAT columns from — this + # reconstructor's `variants` is a dummy placeholder. + haps = replace( + to_evolve.get("_seqs", self._seqs), var_fields=var_fields + ) + to_evolve["_seqs"] = haps + else: + haps = to_evolve.get("_seqs", self._seqs) + # Discover custom FORMAT fields so we don't try to load them as INFO. + custom_fmt = _svar_format_fields(haps.variants.path.parent) + # Lazily load any newly-requested info columns into the existing + # _Variants struct (mutates haps.variants.info in place). + builtin = {"alt", "ilen", "start", "ref", "dosage"} + new_info_fields = [ + f + for f in var_fields + if f not in builtin + and f not in haps.variants.info + and f not in custom_fmt + ] + if new_info_fields: + haps.variants.load_info(new_info_fields) + # Lazily memmap dosages if newly requested. + if "dosage" in var_fields and haps.dosages is None: + haps = _lazy_load_dosages(self, haps) + # Lazily memmap custom FORMAT fields if newly requested. + new_custom_fields = { + f: custom_fmt[f] + for f in var_fields + if f in custom_fmt and f not in haps.var_field_data + } + if new_custom_fields: + haps = _lazy_load_custom_fields(self, haps, new_custom_fields) + haps = replace(haps, var_fields=var_fields) + to_evolve["_seqs"] = haps if splice_info is not None: if splice_info is False: @@ -1433,7 +1454,14 @@ def _output_bytes_per_instance( total += per_ploid.reshape(-1, ploidy).sum(-1) else: # INFO column: numeric, known dtype from on-disk schema. - info_dtype = haps_obj.variants.info[f].dtype + # Svar2Haps.variants is a dummy placeholder (info={}) -- + # store fields' dtypes live in the store manifest instead. + from ._svar2_haps import Svar2Haps + + if isinstance(haps_obj, Svar2Haps) and f in haps_obj.store_fields: + info_dtype = haps_obj.store_fields[f].dtype + else: + info_dtype = haps_obj.variants.info[f].dtype total += n_vars_total * info_dtype.itemsize if include_offsets: # RaggedVariants (kind=2) writes, per field: outer offsets diff --git a/python/genvarloader/_dataset/_open.py b/python/genvarloader/_dataset/_open.py index c720a266..03b59441 100644 --- a/python/genvarloader/_dataset/_open.py +++ b/python/genvarloader/_dataset/_open.py @@ -15,7 +15,7 @@ import numpy as np import polars as pl import seqpro as sp -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from loguru import logger from numpy.typing import NDArray @@ -56,6 +56,7 @@ class OpenRequest: splice_info: str | tuple[str, str] | None = None var_filter: Literal["exonic"] | None = None svar: str | Path | None = None + svar2: str | Path | None = None var_fields: list[str] | None = None def resolve(self) -> RaggedDataset: @@ -149,19 +150,38 @@ def _build_seqs( if self._has_genotypes(): if metadata.ploidy is None: raise ValueError("Malformed dataset: found genotypes but not ploidy.") - seqs = Haps.from_path( - path=self.path, - reference=reference, - regions=regions, - samples=metadata.samples, - ploidy=metadata.ploidy, - version=metadata.version, - svar_link=metadata.svar_link, - svar_override=self.svar, - min_af=self.min_af, - max_af=self.max_af, - var_fields=self.var_fields, - ) + svar2_meta = self.path / "genotypes" / "svar2_ranges" / "svar2_meta.json" + seqs: Haps | Ref | None + if svar2_meta.exists(): + from ._svar2_haps import Svar2Haps + + seqs = Svar2Haps.from_path( + path=self.path, + reference=reference, + samples=metadata.samples, + ploidy=metadata.ploidy, + svar2_link=metadata.svar2_link, + svar2_override=self.svar2, + contigs=metadata.contigs, + min_af=self.min_af, + max_af=self.max_af, + max_jitter=metadata.max_jitter, + var_fields=self.var_fields, + ) + else: + seqs = Haps.from_path( + path=self.path, + reference=reference, + regions=regions, + samples=metadata.samples, + ploidy=metadata.ploidy, + version=metadata.version, + svar_link=metadata.svar_link, + svar_override=self.svar, + min_af=self.min_af, + max_af=self.max_af, + var_fields=self.var_fields, + ) if reference is None: logger.warning( "No reference: dataset only has genotypes but no reference was given." diff --git a/python/genvarloader/_dataset/_reconstruct.py b/python/genvarloader/_dataset/_reconstruct.py index 0d6b80e5..8ca005c0 100644 --- a/python/genvarloader/_dataset/_reconstruct.py +++ b/python/genvarloader/_dataset/_reconstruct.py @@ -41,7 +41,7 @@ from ._utils import _ffi_array from .._threads import should_parallelize -# Fused tracks entry (Task 14): intervals → scratch → realign, one FFI crossing. +# Fused tracks entry: intervals → scratch → realign, one FFI crossing. # Imported at module level so the spy in test_fused_tracks_parity can monkeypatch it. from ..genvarloader import ( intervals_and_realign_track_fused as intervals_and_realign_track_fused, @@ -140,6 +140,24 @@ def __call__( flat: bool = False, to_rc: "NDArray[np.bool_] | None" = None, ) -> tuple[_H, _T]: + # SVAR2 read path: route to the split materialize→realign + # kernel. The isinstance guard keeps the SVAR1 body below byte-unchanged. + from ._svar2_haps import Svar2Haps + + if isinstance(self.haps, Svar2Haps): + return self._call_svar2( + idx, + r_idx, + regions, + output_length, + jitter, + rng, + deterministic, + splice_plan, + flat, + to_rc, + ) + if splice_plan is not None: raise NotImplementedError( "Splicing of haplotypes + tracks (shape (b, t, p, ~l)) is not " @@ -286,6 +304,155 @@ def __call__( return haps, tracks + def _call_svar2( + self, + idx: NDArray[np.integer], # (b) + r_idx: NDArray[np.integer], # (b) + regions: NDArray[np.int32], # (b 3) + output_length: Literal["ragged", "variable"] | int, + jitter: int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: SplicePlan | None = None, + flat: bool = False, + to_rc: "NDArray[np.bool_] | None" = None, + ) -> tuple[_H, _T]: + """SVAR2 haplotype-realigned tracks (see :class:`Svar2Haps`). + + Produces the SAME ``(b, t, p, ~l)`` ``_Flat`` layout the SVAR1 + :meth:`__call__` above produces — the per-track ``_out`` slices are filled + byte-identically, only the interval→realign step is SPLIT into the two + standalone SVAR2 kernels (``intervals_to_tracks`` + + ``shift_and_realign_tracks_from_svar2_readbound``) instead of the fused + SVAR1 kernel. Haps come from the shared ``get_haps_and_shifts`` 7-tuple. + """ + from ._svar2_haps import Svar2Haps + + haps_recon = cast(Svar2Haps, self.haps) + + if splice_plan is not None: + raise NotImplementedError( + "Splicing of haplotypes + tracks is not supported for svar2 " + "datasets yet." + ) + # Annotated haps are out of scope for svar2 (matches Svar2Haps.__call__). + # HapsTracks routes here BEFORE Svar2Haps's own annotated guard, and + # get_haps_and_shifts returns plain Ragged[S1] regardless of kind, so + # without this an annotated view would silently yield non-annotated haps. + if issubclass(self.haps.kind, RaggedAnnotatedHaps): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('annotated') with tracks yet." + ) + # The realign kernel has no in-kernel reverse-complement. + if to_rc is not None and bool(np.asarray(to_rc).any()): + raise NotImplementedError( + "In-kernel reverse-complement is not supported for svar2 " + "haplotype-realigned tracks." + ) + + lengths = regions[:, 2] - regions[:, 1] + + # ragged (b p l), (b p), (b p), (b p), (b p), None, None + haps, _geno_idx, shifts, diffs, hap_lengths, _keep, _keep_offsets = ( + haps_recon.get_haps_and_shifts( + idx=idx, + regions=regions, + output_length=output_length, + rng=rng, + deterministic=deterministic, + to_rc=to_rc, + ) + ) + + if issubclass(self.tracks.kind, RaggedTracks): + # The readbound track kernel always sizes each hap to ref_len + diff + # (no output_length override), so a fixed-length request cannot be + # honored byte-identically. Guard rather than silently mis-size. + if isinstance(output_length, int): + raise NotImplementedError( + "Fixed-length (int output_length) haplotype-realigned tracks " + "are not supported for svar2 datasets yet; use ragged/variable " + "output." + ) + # (b p) — ragged output: hap output length == hap_lengths. + out_lengths = hap_lengths + # (b) = lengths (b) + max deletion length across ploidy (b p) -> (b) + track_lengths = lengths - diffs.clip(max=0).min(1) + + # (b*p+1) + out_ofsts_per_t = lengths_to_offsets(out_lengths) + n_per_track: int = out_ofsts_per_t[-1] + # ragged (b t p l) + out = np.empty(len(self.tracks.active_tracks) * n_per_track, np.float32) + out_lens = repeat( + out_lengths, "b p -> b t p", t=len(self.tracks.active_tracks) + ) + out_offsets = lengths_to_offsets(out_lens) + + # Lower per-track strategies into numba-friendly arrays. + strat_list = [ + self.tracks.insertion_fill.get(name, Repeat5p()) + for name in self.tracks.active_tracks + ] + strat_ids, strat_params = _lower_insertion_fills(strat_list) + + # Seed-dependent (FlankSample) fills stay byte-identical to the single + # fused SVAR1 call across a multi-contig batch: SVAR1 realigns the + # whole batch in ONE call, so the fill hash `hash4(base_seed, query, + # hap, out_idx+i)` uses the GLOBAL row `query`. `_call_svar2` calls the + # readbound kernel once PER CONTIG GROUP where `k / ploidy` is + # contig-LOCAL, so `realign_track_block` passes the group's global row + # indices into the FFI (`global_query`) and the kernel seeds with those + # instead of the local index. `base_seed` already matches (both derive + # from the full idx). Fixed in issue #267. + # Base seed identical to the SVAR1 path (idx-xor when deterministic). + if deterministic: + base_seed = np.uint64( + np.bitwise_xor.reduce(idx.astype(np.uint64, copy=False)) + ) + else: + base_seed = np.uint64( + rng.integers(0, np.iinfo(np.uint64).max, dtype=np.uint64) + ) + + for track_ofst, (name, tracktype) in enumerate( + self.tracks.active_tracks.items() + ): + intervals = self.tracks.intervals[name] + o_idx = idx if tracktype is TrackType.SAMPLE else r_idx + + _out = out[track_ofst * n_per_track : (track_ofst + 1) * n_per_track] + block_data, _block_off = haps_recon.realign_track_block( + idx=idx, + o_idx=o_idx, + regions=regions, + shifts=shifts, + track_lengths=track_lengths, + intervals=intervals, + params=np.ascontiguousarray(strat_params[track_ofst], np.float64), + strategy_id=int(strat_ids[track_ofst]), + base_seed=int(base_seed), + ) + # block_data is (b, P) C-ordered with per-hap lengths == hap_lengths, + # so its offsets equal out_ofsts_per_t; copy into the track slice. + _out[:] = block_data + + out_shape = ( + len(idx), + len(self.tracks.active_tracks), + self.haps.genotypes.shape[-2], + None, + ) + + # flat (b t p l) + tracks = _Flat.from_offsets(out, out_shape, out_offsets) + + else: + tracks = self.tracks._call_intervals(idx) + + tracks = cast(_T, tracks) + return cast(_H, haps), tracks + def _build_reconstructor( seqs: Haps | Ref | None, diff --git a/python/genvarloader/_dataset/_reference.py b/python/genvarloader/_dataset/_reference.py index 4d95f794..8019e4ab 100644 --- a/python/genvarloader/_dataset/_reference.py +++ b/python/genvarloader/_dataset/_reference.py @@ -7,7 +7,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from hirola import HashTable from numpy.typing import ArrayLike, NDArray from seqpro.rag import Ragged, lengths_to_offsets diff --git a/python/genvarloader/_dataset/_svar2_haps.py b/python/genvarloader/_dataset/_svar2_haps.py new file mode 100644 index 00000000..4b903ba4 --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_haps.py @@ -0,0 +1,1082 @@ +"""SVAR2-backed haplotype/variant reconstructor (dataset read dispatch). + +``Svar2Haps`` is a *separate* reconstructor from the SVAR1 :class:`Haps` — the +SVAR1 path is left byte-unchanged. It subclasses :class:`Haps` only so the many +``isinstance(_, Haps)`` / ``case Haps()`` checks throughout the dataset +machinery keep working; every read method is overridden. + +For a query block of ``n_q`` rows, each row ``q = (region r_q, sample slot si_q)`` +with post-jitter bounds ``[start_q, end_q)``, the cache (written by +``_write._write_from_svar2`` under ``genotypes/svar2_ranges/``) is sliced by +fancy-indexing — NO per-read interval search, NO dense-union rebuild — and fed to +the read-bound Rust kernels (``reconstruct_haplotypes_from_svar2_readbound`` / +``decode_variants_from_svar2_readbound`` / ``hap_diffs_from_svar2_readbound``). + +The FFI-input shaping + output wrapping mirror +``tests/_oracles/svar2_readbound_inputs.build_readbound_*`` (test oracle) exactly; the only difference is the source +of the per-query ranges (this module slices the on-disk cache for the specific +``(r_q, si_q)`` block, whereas the helpers call ``SparseVar2._find_ranges`` over +the full cohort). + +Out of scope (guarded with ``NotImplementedError``): spliced output, +``filter == "exonic"`` (keep mask), ``min_af``/``max_af``, annotated haps, and +in-kernel reverse-complement. (``unphased_union`` and ``variant-windows`` ARE +supported.) +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal, cast + +import numpy as np +from genoray._types import POS_TYPE, V_IDX_TYPE +from numpy.typing import NDArray +from seqpro.rag import Ragged + +from .._flat import _Flat +from .._ragged import RaggedAnnotatedHaps +from .._threads import should_parallelize +from .._utils import lengths_to_offsets +from .._variants._records import RaggedAlleles +from ..genvarloader import ( + Svar2Store, + decode_variants_from_svar2_readbound, + hap_diffs_from_svar2_readbound, + reconstruct_haplotypes_from_svar2_readbound, + shift_and_realign_tracks_from_svar2_readbound, +) +from ._flat_variants import ( + _FlatVariantWindows, + _FlatWindow, + _assemble_variant_buffers_rust, +) +from ._intervals import intervals_to_tracks +from ._haps import _H, Haps, _Variants +from ._rag_variants import RaggedVariants +from ._reference import Reference +from ._svar2_link import Svar2Link, _resolve_svar2, _verify_svar2_fingerprint + +if TYPE_CHECKING: + from genoray._svar2_fields import StoredField + + from ._splice import SplicePlan + + +_BUILTIN_VAR_FIELDS: frozenset[str] = frozenset( + {"alt", "ilen", "start", "ref", "dosage"} +) +"""Variant-field keys the reconstructors handle natively (never store fields).""" + + +def _field_spec(sf: "StoredField") -> tuple[str, str, str]: + """(category, name, dtype_str) as the Rust FFI expects it.""" + from genoray._svar2_fields import _META_DTYPE + + return (sf.category, sf.name, _META_DTYPE[sf.dtype]) + + +@dataclass(slots=True) +class _Svar2Cache: + """The six memmapped ``svar2_ranges/`` arrays (all int64), sliced per query. + + ``vk_*_range`` are ``(R, S, P, 2)`` (per region/sample/ploid byte windows into + the store's var_key tables); ``dense_*_range`` are ``(R, 2)`` (per-region, + sample-independent); ``sample_cols`` is ``(S,)`` (selected slot -> original + store sample index). Per-query starts are recomputed post-jitter at read time, + so they are not cached here. + """ + + vk_snp_range: NDArray[np.int64] + vk_indel_range: NDArray[np.int64] + dense_snp_range: NDArray[np.int64] + dense_indel_range: NDArray[np.int64] + sample_cols: NDArray[np.int64] + + +def _ragged_arange_src( + offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray[np.int64], NDArray[np.int64]]: + """Source-row index + new offsets for a 1-level ragged reorder by ``perm``. + + ``new_data == data[src]``; ``src`` and ``new_off`` depend only on + ``(offsets, perm)`` — so callers reordering several parallel data arrays by + the same key compute this ONCE and index each array. + """ + offsets = np.asarray(offsets, np.int64) + lens = np.diff(offsets) + new_lens = lens[perm] + new_off = lengths_to_offsets(new_lens, np.int64) + n = int(new_off[-1]) + if n == 0: + return np.zeros(0, np.int64), new_off + within = np.arange(n, dtype=np.int64) - np.repeat(new_off[:-1], new_lens) + src = np.repeat(offsets[perm], new_lens) + within + return src, new_off + + +def _ragged_arange_gather( + data: NDArray, offsets: NDArray[np.integer], perm: NDArray[np.integer] +) -> tuple[NDArray, NDArray[np.int64]]: + """Reorder the rows of a 1-level ragged array ``(data, offsets)`` by ``perm``.""" + src, new_off = _ragged_arange_src(offsets, perm) + if src.size == 0: + return data[:0].copy(), new_off + return data[src], new_off + + +def _ragged_arange_gather_2level( + data: NDArray, + var_off: NDArray[np.integer], + str_off: NDArray[np.integer], + perm: NDArray[np.integer], +) -> tuple[NDArray, NDArray[np.int64], NDArray[np.int64]]: + """Reorder the rows of a 2-level ragged (opaque-string) array by ``perm``. + + ``var_off`` (len ``n_rows + 1``) bounds variants per row; ``str_off`` (len + ``n_variants + 1``) bounds bytes per variant. Returns + ``(new_data, new_var_off, new_str_off)``. Fully vectorized. + """ + var_off = np.asarray(var_off, np.int64) + str_off = np.asarray(str_off, np.int64) + var_lens = np.diff(var_off) + new_var_lens = var_lens[perm] + new_var_off = lengths_to_offsets(new_var_lens, np.int64) + total_vars = int(new_var_off[-1]) + if total_vars == 0: + return data[:0].copy(), new_var_off, np.zeros(1, np.int64) + within_var = np.arange(total_vars, dtype=np.int64) - np.repeat( + new_var_off[:-1], new_var_lens + ) + old_var_idx = np.repeat(var_off[perm], new_var_lens) + within_var + var_byte_len = np.diff(str_off) + new_byte_len = var_byte_len[old_var_idx] + new_str_off = lengths_to_offsets(new_byte_len, np.int64) + nbytes = int(new_str_off[-1]) + if nbytes == 0: + return data[:0].copy(), new_var_off, new_str_off + within_b = np.arange(nbytes, dtype=np.int64) - np.repeat( + new_str_off[:-1], new_byte_len + ) + src = np.repeat(str_off[old_var_idx], new_byte_len) + within_b + return data[src], new_var_off, new_str_off + + +@dataclass(slots=True) +class Svar2Haps(Haps[_H]): + """Read-bound SVAR2 reconstructor. See module docstring.""" + + # New fields must default (they follow base Haps' defaulted fields). + store: "Svar2Store | None" = None + cache: "_Svar2Cache | None" = None + store_contigs: list[str] = field(default_factory=list) + """The .svar2 store's contig names (used to open the store's ContigReaders).""" + ds_contigs: list[str] = field(default_factory=list) + """The dataset's contig names (``regions[:, 0]`` indexes into this).""" + max_jitter: int = 0 + """The dataset's write-time max_jitter. When > 0 the cache's per-query ranges + were computed over a max_jitter-padded window, which over-includes variants past + the (unpadded) read window in variants mode (the decode kernel has no right-clip); + guarded below.""" + store_fields: dict[str, "StoredField"] = field(default_factory=dict) + """The .svar2 store's INFO/FORMAT field manifest, keyed by field key. + + Populated from ``SparseVar2.available_fields``. These keys are additionally + advertised in ``available_var_fields`` so users can request them via ``var_fields``. + """ + + def __post_init__(self): + # Deliberately does NOT call Haps.__post_init__ (that reads an SVAR1 + # variants table / AF cache which svar2 has no analogue for). Set only + # the init=False fields the base machinery reads. + self.n_variants = self.genotypes.lengths + self.available_var_fields = ["alt", "ilen", "start"] + [ + k for k in self.store_fields if k not in _BUILTIN_VAR_FIELDS + ] + + # ---- construction ---- + + @classmethod + def from_path( # type: ignore[override] # separate svar2 signature; base returns Haps[RaggedVariants] + cls, + path: Path, + reference: Reference | None, + samples: list[str], + ploidy: int, + svar2_link: Svar2Link | None, + svar2_override: Path | str | None, + contigs: list[str], + kind: type = RaggedVariants, + min_af: float | None = None, + max_af: float | None = None, + max_jitter: int = 0, + var_fields: list[str] | None = None, + ) -> "Svar2Haps": + # Default var_fields for loading. var_fields=None means "use the default + # set" — mirrors Haps.from_path's resolution. + if var_fields is None: + var_fields = ["alt", "ilen", "start"] + + ranges_dir = path / "genotypes" / "svar2_ranges" + with open(ranges_dir / "svar2_meta.json") as f: + meta = json.load(f) + + def _mm(name: str, shape: list[int]) -> NDArray[np.int64]: + return np.memmap( + ranges_dir / name, dtype=np.int64, mode="r", shape=tuple(shape) + ) + + R = int(meta["dense_snp_range"]["shape"][0]) + S = int(meta["vk_snp_range"]["shape"][1]) + P = int(meta["ploidy"]) + if P != ploidy: + raise ValueError(f"svar2 cache ploidy ({P}) != dataset ploidy ({ploidy}).") + + cache = _Svar2Cache( + vk_snp_range=_mm("vk_snp_range.npy", meta["vk_snp_range"]["shape"]), + vk_indel_range=_mm("vk_indel_range.npy", meta["vk_indel_range"]["shape"]), + dense_snp_range=_mm( + "dense_snp_range.npy", meta["dense_snp_range"]["shape"] + ), + dense_indel_range=_mm( + "dense_indel_range.npy", meta["dense_indel_range"]["shape"] + ), + sample_cols=np.load(ranges_dir / "sample_cols.npy"), + ) + + svar2_path = _resolve_svar2(path, svar2_link, svar2_override) + _verify_svar2_fingerprint(svar2_path, svar2_link) + + # Open the query-only store. n_samples must be the store's FULL sample + # count (orig_samples / sample_cols index into it), not len(samples). + from genoray import SparseVar2 + + sv = SparseVar2(str(svar2_path)) + store = Svar2Store(str(svar2_path), sv.contigs, sv.n_samples, sv.ploidy) + store_fields = dict(sv.available_fields) + + allowed = {"alt", "ilen", "start"} | set(store_fields) + if missing := [f for f in var_fields if f not in allowed]: + raise ValueError(f"Missing variant fields: {missing}") + + # Minimal base-Haps fields. genotypes carries only the (R, S, P, None) + # shape (so ploidy = shape[-2] and n_variants.shape are available); its + # data is empty (svar2 has no per-region sparse genotype store). + empty_geno = Ragged.from_offsets( + np.empty(0, V_IDX_TYPE), + (R, S, P, None), + np.zeros(R * S * P + 1, np.int64), + ) + empty_alt = RaggedAlleles.from_offsets( + np.empty(0, np.uint8).view("S1"), (0, None), np.zeros(1, np.int64) + ) + dummy_variants = _Variants( + path=svar2_path, + start=np.empty(0, POS_TYPE), + ilen=np.empty(0, np.int32), + ref=None, + alt=empty_alt, + info={}, + ) + + return cls( + path=path, + reference=reference, + variants=dummy_variants, + genotypes=empty_geno, + dosages=None, + kind=cast("type[_H]", kind), + filter=None, + min_af=min_af, + max_af=max_af, + store=store, + cache=cache, + store_contigs=list(sv.contigs), + ds_contigs=list(contigs), + max_jitter=max_jitter, + store_fields=store_fields, + var_fields=var_fields, + ) + + # ---- reconstructor entry ---- + + def __call__( + self, + idx: NDArray[np.integer], + r_idx: NDArray[np.integer], + regions: NDArray[np.int32], + output_length: Literal["ragged", "variable"] | int, + jitter: int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: "SplicePlan | None" = None, + flat: bool = False, + to_rc: "NDArray[np.bool_] | None" = None, + ) -> _H: + self._guard_unsupported(splice_plan) + + if issubclass(self.kind, (RaggedVariants, _FlatVariantWindows)): + # variants AND variant-windows decode variants; the read-bound decode + # has NO right-clip, so max_jitter>0 / jitter>0 would over-include + # variants past the (unpadded) read window. Guard both modes. + if self.max_jitter > 0 or jitter > 0: + raise NotImplementedError( + "variants/variant-windows output for svar2 datasets written with" + f" max_jitter>0 (here max_jitter={self.max_jitter}) or read with" + f" jitter>0 (here jitter={jitter}) is not yet supported: the" + " read-bound decode does not right-clip to the post-jitter window." + ) + if issubclass(self.kind, _FlatVariantWindows): + return cast(_H, self._reconstruct_variant_windows(idx, regions)) + # RaggedVariants: RC is applied by the caller (_getitem_unspliced), + # so to_rc is intentionally ignored here (mirrors SVAR1 Haps). + return cast(_H, self._reconstruct_variants(idx, regions)) + + if issubclass(self.kind, RaggedAnnotatedHaps): + raise NotImplementedError( + "svar2 datasets do not support with_seqs('annotated') yet." + ) + + # Haplotypes: RC would need to be folded in-kernel; the read-bound haps + # kernel has no to_rc param, so any real RC is unsupported here. + if to_rc is not None and bool(np.asarray(to_rc).any()): + raise NotImplementedError( + "In-kernel reverse-complement is not supported for svar2 haplotypes." + ) + + haps, *_ = self.get_haps_and_shifts( + idx=idx, + regions=regions, + output_length=output_length, + rng=rng, + deterministic=deterministic, + splice_plan=splice_plan, + to_rc=to_rc, + need_hap_lengths=False, + ) + return cast(_H, haps) + + def get_haps_and_shifts( + self, + idx: NDArray[np.integer], + regions: NDArray[np.integer], + output_length: Literal["ragged", "variable"] | int, + rng: np.random.Generator, + deterministic: bool, + splice_plan: "SplicePlan | None" = None, + to_rc: "NDArray[np.bool_] | None" = None, + need_hap_lengths: bool = True, + ) -> tuple[ + Ragged[np.bytes_], + NDArray[np.intp], + NDArray[np.int32], + NDArray[np.int32], + NDArray[np.int32], + NDArray[np.bool_] | None, + NDArray[np.int64] | None, + ]: + """Reconstruct haplotypes + return the SVAR1-shaped 7-tuple. + + Track re-alignment reuses this for the shared shifts/diffs/ + hap_lengths; ``geno_offset_idx`` is a placeholder for svar2 (the cache is + re-sliced from ``idx`` there), and ``keep``/``keep_offsets`` are None. + """ + self._guard_unsupported(splice_plan) + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + lengths = (regions[:, 2] - regions[:, 1]).astype(np.int64) + + groups = self._contig_groups(contig_ids) + + # diffs are needed pre-reconstruct ONLY to (a) bound randomized jitter + # shifts, or (b) return hap_lengths/diffs to a caller that uses them + # (the tracks path). A deterministic/ragged haplotypes read needs + # neither: reconstruct sizes itself internally. Avoid the redundant + # gather+split+diffs in that (common warm-read) case. + randomized = not (deterministic or isinstance(output_length, str)) + need_diffs = randomized or need_hap_lengths + + # --- diffs (per contig group, stitched back to (b, P) query order) --- + if need_diffs: + diffs = np.empty((b, P), np.int32) + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + d = hap_diffs_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + P, + ) + diffs[qsel] = np.asarray(d, np.int32).reshape(len(qsel), P) + + hap_lengths = (lengths[:, None] + diffs).astype(np.int32) + else: + diffs = np.zeros( + (b, P), np.int32 + ) # diffs discarded by __call__ (only caller reaching this branch) + # hap_lengths below still feeds g_total -> should_parallelize (a perf + # heuristic only; never affects output bytes), so the ref-length + # placeholder is byte-identical-safe. + hap_lengths = np.broadcast_to( + lengths[:, None].astype(np.int32), (b, P) + ).copy() + + # --- shifts (single rng draw; mirrors Haps._prepare_request) --- + if randomized: + max_shift = diffs.clip(min=0) + max_shift = max_shift + (lengths - output_length).clip(min=0)[:, None] + shifts = rng.integers(0, max_shift + 1, dtype=np.int32) + else: + shifts = np.zeros((b, P), np.int32) + + ffi_out_len = ( + np.int64(-1) if isinstance(output_length, str) else np.int64(output_length) + ) + + # --- reconstruct (per contig group), collect in grouped query order --- + cat_data: list[NDArray[np.uint8]] = [] + cat_hap_lens: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + ref_, ref_offsets = self._ref_for_contig(ci) + g_shifts = np.ascontiguousarray(shifts[qsel], np.int32) + if isinstance(output_length, int): + g_total = int(output_length) * len(qsel) * P + else: + g_total = int(hap_lengths[qsel].sum()) + g_data, g_off = reconstruct_haplotypes_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + g_shifts, + ref_, + ref_offsets, + np.uint8(self.reference.pad_char), # type: ignore[union-attr] # reference guaranteed for haplotypes + ffi_out_len, + should_parallelize(g_total), + ) + cat_data.append(np.asarray(g_data, np.uint8)) + cat_hap_lens.append(np.diff(np.asarray(g_off, np.int64))) + cat_query_order.append(qsel) + + out = self._assemble_haps(cat_data, cat_hap_lens, cat_query_order, b, P) + + geno_offset_idx = np.repeat( + np.asarray(idx, np.intp)[:, None], P, axis=1 + ) # svar2 placeholder; realign_track_block re-slices the cache from idx. + return out, geno_offset_idx, shifts, diffs, hap_lengths, None, None + + # ---- tracks ---- + + def realign_track_block( + self, + idx: NDArray[np.integer], + o_idx: NDArray[np.integer], + regions: NDArray[np.integer], + shifts: NDArray[np.int32], + track_lengths: NDArray[np.integer], + intervals, + params: NDArray[np.float64], + strategy_id: int, + base_seed: int, + ) -> tuple[NDArray[np.float32], NDArray[np.int64]]: + """Haplotype-realign ONE track for a query block, returning a flat f32 + buffer + offsets in global ``(b, P)`` C-order (row = ``q * P + p``). + + The two-step SVAR2 track path (there is no fused interval→realign kernel): + + 1. **Materialize** the per-query reference-space track window from + ``intervals`` via the standalone :func:`intervals_to_tracks` FFI. Each + window starts at ``regions[q, 1]`` and spans ``track_lengths[q]`` ref + bases (= region length + max deletion across ploidy), exactly the + ``tracks``/``track_offsets`` input the realign kernel expects (one + window per query; all P haps of a query share it). + 2. **Realign** to haplotype coordinates via + :func:`shift_and_realign_tracks_from_svar2_readbound`, cache-sliced per + contig group EXACTLY like :meth:`get_haps_and_shifts` slices for haps. + + ``o_idx`` selects the interval row per query (``idx`` for SAMPLE tracks, + ``r_idx`` otherwise). Per-hap output length is ``ref_len + diff`` (the + kernel's native sizing), so the stitched offsets equal the haps' + ``hap_lengths`` in the same order. + """ + assert self.store is not None + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + params_c = np.ascontiguousarray(params, np.float64) + o_idx = np.asarray(o_idx) + track_lengths = np.asarray(track_lengths, np.int64) + + cat_data: list[NDArray[np.float32]] = [] + cat_lens: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + + # (1) materialize ref-space track windows for this group's queries. + tl_g = track_lengths[qsel] + track_ofsts_g = lengths_to_offsets(tl_g, np.int64) + tracks_buf = np.empty(int(track_ofsts_g[-1]), np.float32) + intervals_to_tracks( + offset_idxs=o_idx[qsel], + starts=regions[qsel, 1], + itv_starts=intervals.starts.data, + itv_ends=intervals.ends.data, + itv_values=intervals.values.data, + itv_offsets=intervals.starts.offsets, + out=tracks_buf, + out_offsets=track_ofsts_g, + ) + + # (2) realign to haplotype coordinates (cache-sliced, per contig). + g_shifts = np.ascontiguousarray(shifts[qsel], np.int32) + g_total = int(track_ofsts_g[-1]) + out_data, out_off = shift_and_realign_tracks_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + gi[6], + g_shifts, + np.ascontiguousarray(tracks_buf, np.float32), + track_ofsts_g, + params_c, + np.int64(strategy_id), + np.uint64(base_seed), + # GLOBAL batch row per local query: this group's queries land at + # positions `qsel` in the full (b, P) batch. The kernel seeds the + # FlankSample fill with this (not the contig-local `k / ploidy`), + # so the per-contig-group split matches the single fused SVAR1 + # call byte-for-byte (issue #267). + np.ascontiguousarray(qsel, np.int64), + should_parallelize(g_total * 4), + ) + cat_data.append(np.asarray(out_data, np.float32)) + cat_lens.append(np.diff(np.asarray(out_off, np.int64))) + cat_query_order.append(qsel) + + # Single contig group: grouped order already equals global (b, P) order, + # so the reorder is the identity — return the sole group's buffer directly. + if len(cat_data) == 1: + return cat_data[0], lengths_to_offsets(cat_lens[0], np.int64) + data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.float32) + lens = np.concatenate(cat_lens) if cat_lens else np.zeros(0, np.int64) + grouped_off = lengths_to_offsets(lens, np.int64) + perm = self._inverse_row_perm(cat_query_order, b, P) + return _ragged_arange_gather(data, grouped_off, perm) + + # ---- variants ---- + + def _requested_store_fields( + self, + ) -> tuple[list[str], list[tuple[str, str, str]], list[np.dtype]]: + """The store INFO/FORMAT fields requested via ``var_fields``. + + Returns ``(keys, specs, dtypes)`` where ``specs`` is what the decode kernel + expects and ``keys``/``dtypes`` are positionally parallel to the field + buffers it returns. Builtin names (alt/start/ilen/ref/dosage) always mean the + builtin, even if the store happens to carry a field of the same name. + """ + keys = [ + f + for f in self.var_fields + if f not in _BUILTIN_VAR_FIELDS and f in self.store_fields + ] + specs = [_field_spec(self.store_fields[k]) for k in keys] + dtypes = [self.store_fields[k].dtype for k in keys] + return keys, specs, dtypes + + def _type_field_bufs( + self, + bufs: list[NDArray], + isizes: list[int], + keys: list[str], + dtypes: list[np.dtype], + ) -> list[NDArray]: + """View each raw ``uint8`` field buffer the decode kernel returned as its + store dtype. + + Guards that the kernel's reported itemsize agrees with the store + manifest -- a store/kernel disagreement, not an internal invariant, so + this raises ``ValueError`` (not an assertion). + """ + typed = [] + for j, dt in enumerate(dtypes): + if isizes[j] != dt.itemsize: + raise ValueError( + f"field {keys[j]!r}: kernel itemsize {isizes[j]} != " + f"store dtype {dt} itemsize {dt.itemsize}" + ) + typed.append(np.asarray(bufs[j], np.uint8).view(dt)) + return typed + + def _reconstruct_variants( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] + ) -> RaggedVariants: + """Decode the per-query variant records (SoA) for a query block. + + Parameters + ---------- + idx + Flat ``(region, sample)`` query indices for the block. + regions + ``(n_regions, 3)`` array of ``(contig_id, start, end)``. + + Returns + ------- + RaggedVariants + Per-query ragged variant records (position, indel length, ALT + bytes, and any requested INFO/FORMAT fields). + """ + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + p_eff = 1 if self.unphased_union else P + + req_keys, field_specs, field_dtypes = self._requested_store_fields() + + cat_var_lens: list[NDArray[np.int64]] = [] + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_alt: list[NDArray[np.uint8]] = [] + cat_var_bytelen: list[NDArray[np.int64]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + cat_fields: list[list[NDArray]] = [] + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + field_specs, + ) + ) + var_off = np.asarray(var_off, np.int64) + if self.unphased_union: + var_off = np.ascontiguousarray(var_off[::P]) + str_off = np.asarray(str_off, np.int64) + cat_var_lens.append(np.diff(var_off)) + cat_pos.append(np.asarray(pos, np.int32)) + cat_ilen.append(np.asarray(ilen, np.int32)) + cat_alt.append(np.asarray(alt_bytes, np.uint8)) + cat_var_bytelen.append(np.diff(str_off)) + cat_query_order.append(qsel) + + cat_fields.append( + self._type_field_bufs(field_bufs, field_isizes, req_keys, field_dtypes) + ) + + # Single contig group: grouped order already equals global (b, P) order, + # so the reorder is the identity and every concatenate is a 1-element no-op. + # Skip both (the numpy reorder otherwise dominates single-contig reads). + if len(cat_pos) == 1: + shape = (b, p_eff, None) + var_off_g = lengths_to_offsets(cat_var_lens[0], np.int64) + str_off_g = lengths_to_offsets(cat_var_bytelen[0], np.int64) + extra = { + k: Ragged.from_offsets(cat_fields[0][j], shape, var_off_g) + for j, k in enumerate(req_keys) + } + return RaggedVariants( + alt=Ragged.from_offsets( + cat_alt[0].view("S1"), shape, var_off_g, str_offsets=str_off_g + ), + start=Ragged.from_offsets(cat_pos[0], shape, var_off_g), + ilen=Ragged.from_offsets(cat_ilen[0], shape, var_off_g), + **extra, + ) + + # Concatenate grouped outputs, then permute hap-rows back to global order. + var_lens = ( + np.concatenate(cat_var_lens) if cat_var_lens else np.zeros(0, np.int64) + ) + grouped_var_off = lengths_to_offsets(var_lens, np.int64) + pos_c = np.concatenate(cat_pos) if cat_pos else np.zeros(0, np.int32) + ilen_c = np.concatenate(cat_ilen) if cat_ilen else np.zeros(0, np.int32) + alt_c = np.concatenate(cat_alt) if cat_alt else np.zeros(0, np.uint8) + var_bytelen = ( + np.concatenate(cat_var_bytelen) + if cat_var_bytelen + else np.zeros(0, np.int64) + ) + grouped_str_off = lengths_to_offsets(var_bytelen, np.int64) + + perm = self._inverse_row_perm(cat_query_order, b, p_eff) + + src, var_off_g = _ragged_arange_src(grouped_var_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] + alt_g, alt_var_off_g, alt_str_off_g = _ragged_arange_gather_2level( + alt_c, grouped_var_off, grouped_str_off, perm + ) + + shape = (b, p_eff, None) + pos_r = Ragged.from_offsets(pos_g, shape, var_off_g) + ilen_r = Ragged.from_offsets(ilen_g, shape, var_off_g) + alt_r = Ragged.from_offsets( + alt_g.view("S1"), shape, alt_var_off_g, str_offsets=alt_str_off_g + ) + extra = {} + for j, k in enumerate(req_keys): + fc = ( + np.concatenate([g[j] for g in cat_fields]) + if cat_fields + else np.zeros(0, field_dtypes[j]) + ) + fg = fc[:0].copy() if src.size == 0 else fc[src] + extra[k] = Ragged.from_offsets(fg, shape, var_off_g) + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r, **extra) + + def _reconstruct_variant_windows( + self, idx: NDArray[np.integer], regions: NDArray[np.integer] + ) -> _FlatVariantWindows: + """Variant-windows for svar2: decode variants per contig group, then run the + shared ``assemble_variant_buffers`` window kernel over the decoded arrays via + an identity gather. ``ref="allele"`` is blocked upstream, so ref is always a + reference-read window; ``alt`` follows ``window_opt.alt``. + """ + assert self.window_opt is not None and self.token_lut is not None + assert self.reference is not None + + opt = self.window_opt + L = opt.flank_length + ref_mode = 1 # ref="window" (ref="allele" rejected in with_seqs) + alt_mode = 1 if opt.alt == "window" else 2 + include_ilen = "ilen" in self.var_fields + + regions = np.asarray(regions, np.int32) + P = int(self.genotypes.shape[-2]) + b = len(idx) + R_all, S_all = int(self.genotypes.shape[0]), int(self.genotypes.shape[1]) + r_q, si_q = np.unravel_index(np.asarray(idx), (R_all, S_all)) + contig_ids = regions[:, 0].astype(np.int64) + groups = self._contig_groups(contig_ids) + + p_eff = 1 if self.unphased_union else P + + req_keys, field_specs, field_dtypes = self._requested_store_fields() + + cat_row_off: list[NDArray[np.int64]] = [] # per-group var boundaries + cat_pos: list[NDArray[np.int32]] = [] + cat_ilen: list[NDArray[np.int32]] = [] + cat_query_order: list[NDArray[np.intp]] = [] + cat_fields: list[list[NDArray]] = [] + # name -> per-group (token_data, per-variant seq offsets) + win_data: dict[str, list[NDArray]] = {} + win_seq_off: dict[str, list[NDArray[np.int64]]] = {} + + for ci, qsel in groups: + gi = self._gather_inputs(r_q[qsel], si_q[qsel], regions[qsel], P) + pos, ilen, alt_bytes, str_off, var_off, field_bufs, field_isizes = ( + decode_variants_from_svar2_readbound( + self.store, + self.ds_contigs[ci], + gi[0], + gi[1], + gi[2], + gi[3], + gi[4], + gi[5], + P, + field_specs, + ) + ) + pos = np.asarray(pos, np.int32) + ilen = np.asarray(ilen, np.int32) + alt_bytes = np.asarray(alt_bytes, np.uint8) + str_off = np.asarray(str_off, np.int64) + var_off = np.asarray(var_off, np.int64) + + row_off = ( + np.ascontiguousarray(var_off[::P]) if self.unphased_union else var_off + ) + n_var = int(len(pos)) + ref_, ref_offsets = self._ref_for_contig(ci) + bufs = _assemble_variant_buffers_rust( + 1, # windows mode + np.arange(n_var, dtype=np.int32), # identity v_idxs (data pre-gathered) + row_off, + alt_bytes, # alt_global + str_off, # alt_off_global + None, # ref_global (ref="window") + None, # ref_off_global + False, # want_ref_bytes + False, # want_flank + ref_mode, + alt_mode, + L, + self.token_lut, + np.zeros(n_var, np.int32), # v_contigs (single-contig ref slice) + pos, # v_starts + ilen, # ilens + ref_, + ref_offsets, + self.reference.pad_char, + ) + + cat_row_off.append(row_off) + cat_pos.append(pos) + cat_ilen.append(ilen) + cat_query_order.append(qsel) + for name, (data, seq_off) in bufs.items(): + win_data.setdefault(name, []).append(np.asarray(data)) + win_seq_off.setdefault(name, []).append(np.asarray(seq_off, np.int64)) + + cat_fields.append( + self._type_field_bufs(field_bufs, field_isizes, req_keys, field_dtypes) + ) + + shape: tuple[int | None, ...] = (b, p_eff, None) + wshape: tuple[int | None, ...] = (b, p_eff, None, None) + + # Single contig group: grouped order already equals global (b, p_eff) order. + if len(cat_pos) == 1: + row_off = cat_row_off[0] + fields: dict[str, Any] = { + "start": _Flat.from_offsets(cat_pos[0], shape, row_off) + } + if include_ilen: + fields["ilen"] = _Flat.from_offsets(cat_ilen[0], shape, row_off) + for j, k in enumerate(req_keys): + fields[k] = _Flat.from_offsets(cat_fields[0][j], shape, row_off) + win = _FlatVariantWindows(fields) + for name in win_data: + setattr( + win, + name, + _FlatWindow( + win_data[name][0], win_seq_off[name][0], row_off, wshape + ), + ) + else: + perm = self._inverse_row_perm(cat_query_order, b, p_eff) + grouped_row_off = lengths_to_offsets( + np.concatenate([np.diff(r) for r in cat_row_off]), np.int64 + ) + pos_c = np.concatenate(cat_pos) + ilen_c = np.concatenate(cat_ilen) + src, row_off_g = _ragged_arange_src(grouped_row_off, perm) + if src.size == 0: + pos_g = pos_c[:0].copy() + ilen_g = ilen_c[:0].copy() + else: + pos_g = pos_c[src] + ilen_g = ilen_c[src] + fields = {"start": _Flat.from_offsets(pos_g, shape, row_off_g)} + if include_ilen: + fields["ilen"] = _Flat.from_offsets(ilen_g, shape, row_off_g) + for j, k in enumerate(req_keys): + fc = ( + np.concatenate([g[j] for g in cat_fields]) + if cat_fields + else np.zeros(0, field_dtypes[j]) + ) + fg = fc[:0].copy() if src.size == 0 else fc[src] + fields[k] = _Flat.from_offsets(fg, shape, row_off_g) + win = _FlatVariantWindows(fields) + for name in win_data: + data_c = np.concatenate(win_data[name]) + grouped_seq_off = lengths_to_offsets( + np.concatenate([np.diff(s) for s in win_seq_off[name]]), np.int64 + ) + nd, nvar, nseq = _ragged_arange_gather_2level( + data_c, grouped_row_off, grouped_seq_off, perm + ) + setattr(win, name, _FlatWindow(nd, nseq, nvar, wshape)) + + if self.dummy_variant is not None: + win = win.fill_empty_groups( + self.dummy_variant, unk=self.unknown_token, flank_length=L + ) + return win + + # ---- helpers ---- + + def _guard_unsupported(self, splice_plan: "SplicePlan | None") -> None: + if splice_plan is not None: + raise NotImplementedError( + "Spliced output is not supported for svar2 datasets yet." + ) + if self.filter == "exonic": + raise NotImplementedError( + "var_filter='exonic' (keep mask) is not supported for svar2 yet." + ) + if self.min_af is not None or self.max_af is not None: + raise NotImplementedError( + "min_af/max_af filtering is not supported for svar2 datasets yet." + ) + # No unphased_union guard: variants/variant-windows honor it via the + # ploidy-1 fold in their reconstructors; haplotypes/annotated + union is + # blocked upstream in _impl.py, and the haps/track spine ignores union + # (same as SVAR1), so no path reaches here needing a union guard. + + def _contig_groups( + self, contig_ids: NDArray[np.int64] + ) -> list[tuple[int, NDArray[np.intp]]]: + """Group query positions by contig id (store readers are per-contig). + + Preserves original order within each contig group. + """ + groups: list[tuple[int, NDArray[np.intp]]] = [] + for ci in np.unique(contig_ids): + qsel = np.nonzero(contig_ids == ci)[0].astype(np.intp) + groups.append((int(ci), qsel)) + return groups + + def _gather_inputs( + self, + r_q: NDArray[np.integer], + si_q: NDArray[np.integer], + regions_grp: NDArray[np.int32], + P: int, + ) -> tuple[ + NDArray[np.uint32], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int64], + NDArray[np.int32], + ]: + """Cache-slice a per-contig query block into the read-bound FFI inputs. + + Fancy-indexes the memmapped cache (sub-linear; no per-read search). The + vk_* rows come out ``(n, P, 2)`` -> reshaped ``(n*P, 2)`` in row = q*P+p + order, which is exactly what the kernel expects. + """ + assert self.cache is not None + c = self.cache + region_starts = np.ascontiguousarray(regions_grp[:, 1], np.uint32) + orig_samples = np.ascontiguousarray(c.sample_cols[si_q], np.int64) + vk_snp = np.ascontiguousarray( + np.asarray(c.vk_snp_range[r_q, si_q]).reshape(-1, 2), np.int64 + ) + vk_indel = np.ascontiguousarray( + np.asarray(c.vk_indel_range[r_q, si_q]).reshape(-1, 2), np.int64 + ) + dense_snp = np.ascontiguousarray(np.asarray(c.dense_snp_range[r_q]), np.int64) + dense_indel = np.ascontiguousarray( + np.asarray(c.dense_indel_range[r_q]), np.int64 + ) + region_bounds = np.ascontiguousarray(regions_grp[:, 1:3], np.int32) + return ( + region_starts, + orig_samples, + vk_snp, + vk_indel, + dense_snp, + dense_indel, + region_bounds, + ) + + def _ref_for_contig( + self, contig_idx: int + ) -> tuple[NDArray[np.uint8], NDArray[np.int64]]: + """The single-contig reference slice + ``[0, len]`` offsets the kernel wants. + + ``reference.offsets`` is built in ``ds_contigs`` order (Reference.from_path + was called with the dataset's contigs), so ``contig_idx`` indexes it + directly. + """ + ref = self.reference + assert ref is not None + o_s = int(ref.offsets[contig_idx]) + o_e = int(ref.offsets[contig_idx + 1]) + ref_ = np.ascontiguousarray(ref.reference[o_s:o_e], np.uint8) + ref_offsets = np.array([0, o_e - o_s], np.int64) + return ref_, ref_offsets + + @staticmethod + def _inverse_row_perm( + cat_query_order: list[NDArray[np.intp]], b: int, P: int + ) -> NDArray[np.intp]: + """Permutation mapping grouped hap-row order back to global (b, P) order. + + ``final_row[k] == grouped_row[perm[k]]``. + """ + if cat_query_order: + grouped_queries = np.concatenate(cat_query_order) + else: + grouped_queries = np.zeros(0, np.intp) + grouped_rows = ( + grouped_queries[:, None] * P + np.arange(P, dtype=np.intp) + ).ravel() + perm = np.empty(b * P, np.intp) + perm[grouped_rows] = np.arange(b * P, dtype=np.intp) + return perm + + def _assemble_haps( + self, + cat_data: list[NDArray[np.uint8]], + cat_hap_lens: list[NDArray[np.int64]], + cat_query_order: list[NDArray[np.intp]], + b: int, + P: int, + ) -> Ragged[np.bytes_]: + # Single contig group: grouped hap-row order already equals global (b, P) + # order (the sole group's qsel is [0..b-1]), so the reorder is the identity + # and the concatenate is a 1-element no-op. Skip both — this is the common + # single-contig read, where the O(total_bytes) numpy reorder otherwise + # dominates (~96% of the call). Byte-identical to the general path. + if len(cat_data) == 1: + out_data = cat_data[0] + out_off = lengths_to_offsets(cat_hap_lens[0], np.int64) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, (b, P, None), out_off).view("S1"), + ) + data = np.concatenate(cat_data) if cat_data else np.zeros(0, np.uint8) + hap_lens = ( + np.concatenate(cat_hap_lens) if cat_hap_lens else np.zeros(0, np.int64) + ) + grouped_off = lengths_to_offsets(hap_lens, np.int64) + perm = self._inverse_row_perm(cat_query_order, b, P) + out_data, out_off = _ragged_arange_gather(data, grouped_off, perm) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, (b, P, None), out_off).view("S1"), + ) diff --git a/python/genvarloader/_dataset/_svar2_link.py b/python/genvarloader/_dataset/_svar2_link.py new file mode 100644 index 00000000..653c43f5 --- /dev/null +++ b/python/genvarloader/_dataset/_svar2_link.py @@ -0,0 +1,135 @@ +"""Resolution and integrity for the GVL dataset -> .svar2 back-reference. + +Mirrors _svar_link.py; the fingerprint keys on the .svar2 store's stable +identity (file count + summed byte size of its data files) rather than +SVAR1's variant_idxs.npy / index.arrow, neither of which .svar2 has. +SparseVar2 exposes no cheap variant-count accessor, so a semantic +n_variants field is deliberately not part of this fingerprint -- deriving +one would require contig lengths plus a full-span decode, which is +over-engineering for an integrity check. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from pydantic import BaseModel + + +class Svar2Fingerprint(BaseModel): + n_files: int + store_bytes: int + + +class Svar2Link(BaseModel): + relative_path: str + absolute_path: str + fingerprint: Svar2Fingerprint + + +def _svar2_store_fingerprint(svar2_path: Path) -> tuple[int, int]: + """Deterministic (file count, total bytes) over the .svar2 store's data files. + + Walks the store for ``.bin``/``.npy`` files (dense + var_key + long-allele + payloads across all contigs). Changes iff the store's data files change -- + that is this fingerprint's only contract. + """ + files = sorted( + p for p in svar2_path.rglob("*") if p.is_file() and p.suffix in {".bin", ".npy"} + ) + return len(files), sum(p.stat().st_size for p in files) + + +def _resolve_svar2( + gvl_path: Path, + link: Svar2Link | None, + override: Path | str | None, +) -> Path: + """Resolve the .svar2 directory referenced by a GVL dataset. + + Order: override -> link.relative_path -> link.absolute_path -> sibling *.svar2. + Raises FileNotFoundError if none resolve to a directory. + """ + if override is not None: + p = Path(override) + if not p.is_dir(): + raise FileNotFoundError( + f"svar2 override path does not exist or is not a directory: {p}" + ) + return p + + if link is not None: + rel = (gvl_path / link.relative_path).resolve() + if rel.is_dir(): + return rel + absp = Path(link.absolute_path) + if absp.is_dir(): + return absp + + siblings = sorted(gvl_path.parent.glob("*.svar2")) + if len(siblings) == 1: + return siblings[0] + + expected = Path(link.absolute_path).name if link is not None else ".svar2" + raise FileNotFoundError( + f"Could not locate svar2 '{expected}' for GVL dataset at {gvl_path}. " + f"Tried: stored relative path, stored absolute path, sibling *.svar2. " + f"Pass `svar2=` to `Dataset.open(...)` to override." + ) + + +def _verify_svar2_fingerprint(svar2_path: Path, link: Svar2Link | None) -> None: + """Compare the recorded fingerprint against the resolved svar2 store. + + No-op when ``link`` is None (legacy dataset, or one without a svar2 link). + Raises ValueError on mismatch. + """ + if link is None: + return + + n_files_observed, bytes_observed = _svar2_store_fingerprint(svar2_path) + + exp = link.fingerprint + mismatches: list[str] = [] + if n_files_observed != exp.n_files: + mismatches.append( + f"n_files: expected {exp.n_files}, observed {n_files_observed}" + ) + if bytes_observed != exp.store_bytes: + mismatches.append( + f"store_bytes: expected {exp.store_bytes}, observed {bytes_observed}" + ) + if mismatches: + raise ValueError( + f"svar2 fingerprint mismatch at {svar2_path}: " + "; ".join(mismatches) + ) + + +def make_svar2_link(gvl_path: Path, svar2_path: Path) -> Svar2Link: + """Build a :class:`Svar2Link` recording the on-disk relationship between + a gvl dataset and the ``.svar2`` store it reads from. + + Parameters + ---------- + gvl_path + Path to the gvl dataset directory. + svar2_path + Path to the ``.svar2`` store the dataset links to. + + Returns + ------- + Svar2Link + Relative/absolute paths to the store plus its fingerprint + (file count and total byte size), used to detect a moved or + modified store on open. + """ + svar2_resolved = svar2_path.resolve() + n_files, store_bytes = _svar2_store_fingerprint(svar2_resolved) + return Svar2Link( + relative_path=os.path.relpath(svar2_resolved, start=gvl_path).replace( + os.sep, "/" + ), + absolute_path=str(svar2_resolved), + fingerprint=Svar2Fingerprint(n_files=n_files, store_bytes=store_bytes), + ) diff --git a/python/genvarloader/_dataset/_utils.py b/python/genvarloader/_dataset/_utils.py index 8913c539..4fc9884b 100644 --- a/python/genvarloader/_dataset/_utils.py +++ b/python/genvarloader/_dataset/_utils.py @@ -2,7 +2,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from numpy.typing import ArrayLike, NDArray from .._types import DTYPE diff --git a/python/genvarloader/_dataset/_write.py b/python/genvarloader/_dataset/_write.py index f3587430..035eb827 100644 --- a/python/genvarloader/_dataset/_write.py +++ b/python/genvarloader/_dataset/_write.py @@ -1,5 +1,7 @@ +import errno import gc import json +import shutil import warnings from collections.abc import Callable, Iterator, Sequence from importlib.metadata import version @@ -16,15 +18,15 @@ import numpy as np import polars as pl import seqpro as sp -from genoray import PGEN, VCF, Reader, SparseVar +from genoray import PGEN, VCF, SparseVar, SparseVar2 from genoray import exprs as _gexprs from genoray._svar import dense2sparse -from genoray._svar import _dense2sparse_with_length # type: ignore[missing-module-attribute] +from genoray._svar._convert import _dense2sparse_with_length from genoray._types import V_IDX_TYPE -from genoray._utils import ContigNormalizer, format_memory, parse_memory +from genoray._contigs import ContigNormalizer +from genoray._utils import format_memory, parse_memory from joblib import Parallel, delayed from loguru import logger -from more_itertools import mark_ends from natsort import natsorted from numpy.typing import NDArray from pydantic import BaseModel @@ -34,9 +36,10 @@ from tqdm.auto import tqdm from .._atomic import atomic_dir -from .._ragged import INTERVAL_DTYPE # noqa: F401 # Task 3 migration reader imports this +from .._ragged import INTERVAL_DTYPE # noqa: F401 # kept for the migration reader import from .._utils import lengths_to_offsets, normalize_contig_name from .._variants._utils import path_is_pgen, path_is_vcf +from ._svar2_link import Svar2Link from ._svar_link import SvarLink from ._utils import bed_to_regions, regions_to_bed @@ -92,6 +95,7 @@ class Metadata(BaseModel, arbitrary_types_allowed=True): version: SemanticVersion | None = None format_version: SemanticVersion | None = None svar_link: SvarLink | None = None + svar2_link: Svar2Link | None = None @property def n_samples(self) -> int: @@ -101,7 +105,7 @@ def n_samples(self) -> int: def write( path: str | Path, bed: str | Path | pl.DataFrame, - variants: str | Path | Reader | None = None, + variants: str | Path | VCF | PGEN | SparseVar | SparseVar2 | None = None, tracks: "IntervalTrack | Sequence[IntervalTrack] | None" = None, annot_tracks: "dict[str, str | Path | pl.DataFrame | pl.LazyFrame] | None" = None, samples: list[str] | None = None, @@ -224,6 +228,8 @@ def write( variants = VCF(variants) elif variants.is_dir() and variants.suffix == ".svar": variants = SparseVar(variants) + elif variants.is_dir() and variants.suffix == ".svar2": + variants = SparseVar2(variants) else: raise ValueError( f"File {variants} has an unrecognized file extension. Please provide either a VCF or PGEN file.`" @@ -327,6 +333,11 @@ def write( path, gvl_bed, variants, samples, extend_to_length ) metadata["svar_link"] = _svar_link + elif isinstance(variants, SparseVar2): + gvl_bed, _svar2_link = _write_from_svar2( + path, gvl_bed, variants, samples, extend_to_length + ) + metadata["svar2_link"] = _svar2_link metadata["ploidy"] = variants.ploidy # free memory del variants @@ -617,6 +628,23 @@ def _reject_unsupported_variants(index: pl.DataFrame, source: str) -> None: ) +def _link_or_copy(src: Path, dst: Path) -> None: + """Hardlink ``src`` → ``dst`` to avoid duplicating the (possibly large) variant + index, falling back to a copy when the two live on different filesystems. + + ``Path.hardlink_to`` raises ``OSError(EXDEV)`` across filesystem boundaries + (e.g. writing a dataset under ``/tmp`` from a source on a network mount). The + copy produces byte-identical content, so same-filesystem writes keep the + zero-copy hardlink and cross-device writes still succeed. + """ + try: + dst.hardlink_to(src) + except OSError as e: + if e.errno != errno.EXDEV: + raise + shutil.copyfile(src, dst) + + def _write_from_vcf( path: Path, bed: pl.DataFrame, vcf: VCF, max_mem: int, extend_to_length: bool ): @@ -629,7 +657,7 @@ def _write_from_vcf( out_dir = path / "genotypes" out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "variants.arrow").hardlink_to(vcf._index_path()) + _link_or_copy(vcf._index_path(), out_dir / "variants.arrow") return _write_phased_chunked( out_dir, bed, _vcf_region_chunks(bed, vcf, max_mem, extend_to_length) @@ -710,18 +738,18 @@ def _vcf_region_chunks( contig = cast(str, contig) starts = df["chromStart"].to_numpy() ends = df["chromEnd"].to_numpy() - # unextended in-range variant indices, split per region - v_idx, v_offsets = vcf._var_idxs(contig, starts, ends) - unextended_idxs = np.array_split(v_idx.astype(V_IDX_TYPE), v_offsets[1:-1]) contig_desc = f"Processing genotypes for {df.height} regions on contig {contig}" first_in_contig = True + unextended_idxs: list[NDArray] = [] if extend_to_length: region_iter = vcf._chunk_ranges_with_length( contig, starts, ends, max_mem, VCF.Genos8 ) else: + v_idx, v_offsets = vcf._var_idxs(contig, starts, ends) + unextended_idxs = np.array_split(v_idx.astype(V_IDX_TYPE), v_offsets[1:-1]) # one generator per region; VCF.chunk takes a single range region_iter = ( vcf.chunk(contig, s, e, max_mem, VCF.Genos8) @@ -731,33 +759,25 @@ def _vcf_region_chunks( for ri, range_ in enumerate(region_iter): q_start = int(starts[ri]) q_end = int(ends[ri]) - reg_unext = unextended_idxs[ri] desc = contig_desc if first_in_contig else None first_in_contig = False if extend_to_length: - # assemble the full window across memory-chunks - chunk_genos_list: list[NDArray] = [] - n_ext_total = 0 - for _, is_last, (chunk_genos, _chunk_end, n_ext) in mark_ends(range_): - chunk_genos_list.append(chunk_genos) - if is_last: - n_ext_total = n_ext - genos = np.concatenate(chunk_genos_list, axis=-1) - - if reg_unext.size == 0 and n_ext_total == 0: - # empty region: no variants for any sample - yield [dense2sparse(genos, reg_unext)], q_end, desc - continue + genos_list: list[NDArray] = [] + idx_list: list[NDArray] = [] + for chunk_genos, _chunk_end, chunk_idxs in range_: + genos_list.append(chunk_genos) + idx_list.append(chunk_idxs.astype(V_IDX_TYPE)) + genos = np.concatenate(genos_list, axis=-1) + var_idxs = ( + np.concatenate(idx_list) + if idx_list + else np.empty(0, dtype=V_IDX_TYPE) + ) - if n_ext_total > 0: - ext_start = int(reg_unext[-1]) + 1 - ext_idxs = np.arange( - ext_start, ext_start + n_ext_total, dtype=V_IDX_TYPE - ) - var_idxs = np.concatenate([reg_unext, ext_idxs]) - else: - var_idxs = reg_unext + if var_idxs.size == 0: + yield [dense2sparse(genos, var_idxs)], q_end, desc + continue v_starts = (pos[var_idxs] - 1).astype(np.int32) ilens = ilen_all[var_idxs].astype(np.int32) @@ -767,6 +787,7 @@ def _vcf_region_chunks( region_end = _region_end(rag, v_ends, q_end) yield [rag], region_end, desc else: + reg_unext = unextended_idxs[ri] # no extension: convert each chunk independently with plain # dense2sparse; var_idxs are exactly the unextended in-range ones ls_sparse: list[Ragged] = [] @@ -808,7 +829,7 @@ def _write_from_pgen( out_dir = path / "genotypes" out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "variants.arrow").hardlink_to(pgen._index_path()) + _link_or_copy(pgen._index_path(), out_dir / "variants.arrow") return _write_phased_chunked( out_dir, bed, _pgen_region_chunks(bed, pgen, max_mem, extend_to_length) @@ -1068,6 +1089,154 @@ def _write_from_svar( ), svar_link +def _svar2_region_max_ends( + svar2: SparseVar2, + contig: str, + starts: NDArray[np.integer], + ends: NDArray[np.integer], + samples: list[str], +) -> NDArray[np.int32]: + """SVAR1 parity: per region, the end (``pos - min(ilen, 0)``) of the + highest-position variant over the SELECTED samples' haplotypes. Regions with + no variants keep their original ``chromEnd``. + + ``SparseVar2.decode`` reports 0-based ``pos`` (unlike ``SparseVar.index``'s + 1-based VCF ``POS``, which SVAR1's ``v_ends`` formula is written against), so + ``pos`` is converted to 1-based here before applying the same formula -- + otherwise every extension would be off by one (masked in most regions + because the un-extended ``chromEnd`` already dominates the max). + + Vectorized as a per-region scatter-max over a ``(pos << 21) | end`` composite + key, which reproduces the pos-then-end tie-break exactly (a single haplotype + never carries two variants at the same position, so a global per-region max + over the selected samples' variants equals the original per-hap-argmax loop). + """ + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = np.asarray([svar2.available_samples.index(s) for s in samples], np.int64) + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = np.asarray(dec.data["pos"], np.int64) + ilen_arr = np.asarray(dec.data["ilen"], np.int64) + off = np.asarray(dec.offsets, np.int64) # length R*S_all*P + 1 + out = np.asarray(ends, np.int64).copy() # default = chromEnd + if pos_arr.size: + n_hap = R * S_all * P + counts = np.diff(off) # variants per hap + hap_of_var = np.repeat(np.arange(n_hap), counts) # region-major hap per variant + s_of_hap = (np.arange(n_hap) // P) % S_all + keep = np.isin(s_of_hap[hap_of_var], sel) # only selected samples + region_of_var = hap_of_var // (S_all * P) + # end = pos + ext, where ext = 1 - min(ilen, 0) (1-based bump plus the + # deletion extension: SNP/INS -> 1, DEL -> 1 + |ilen|). Pack the BOUNDED + # `ext` into the low bits, NOT the absolute `end` (which is ~pos-sized and + # would overflow past ~2 Mb into any contig), so the composite key orders + # by pos then by end; recover end = pos + ext on unpack. + ext_var = 1 - np.minimum(ilen_arr, 0) # small: 1 + deletion length + SHIFT = 21 + # raise (not assert) so it still fails fast under `python -O`: a pathological + # >~2 Mb deletion footprint would otherwise silently corrupt the packed key. + if int(ext_var.max(initial=0)) >= (1 << SHIFT): + raise ValueError("variant footprint exceeds tie-break packing width") + key = (pos_arr << SHIFT) | ext_var + key_k = key[keep] + region_k = region_of_var[keep] + if key_k.size: + best = np.full(R, -1, np.int64) + np.maximum.at(best, region_k, key_k) # per-region max composite key + has = best >= 0 + # end = pos + ext = (key >> SHIFT) + (key & mask) + out[has] = (best[has] >> SHIFT) + (best[has] & ((1 << SHIFT) - 1)) + return out.astype(np.int32) + + +def _write_from_svar2( + path: Path, + bed: pl.DataFrame, + svar2: SparseVar2, + samples: list[str], + extend_to_length: bool, +) -> tuple[pl.DataFrame, Svar2Link]: + # symbolic/breakend variants are rejected upstream at .svar2 conversion; the + # store cannot represent them, and SparseVar2 exposes no index to re-check. + + if not extend_to_length: + raise NotImplementedError( + "extend_to_length=False is not supported for a .svar2 variant source: " + "the read-bound kernel always sizes haplotype output at read time and " + "the write-time ranges cache is built for the extended chromEnd. Use a " + ".svar/VCF/PGEN source if you need un-extended haplotypes." + ) + + out_dir = path / "genotypes" / "svar2_ranges" + out_dir.mkdir(parents=True, exist_ok=True) + + R, S, P = bed.height, len(samples), svar2.ploidy + vk_snp = np.memmap(out_dir / "vk_snp_range.npy", np.int64, "w+", shape=(R, S, P, 2)) + vk_indel = np.memmap( + out_dir / "vk_indel_range.npy", np.int64, "w+", shape=(R, S, P, 2) + ) + dense_snp = np.memmap(out_dir / "dense_snp_range.npy", np.int64, "w+", shape=(R, 2)) + dense_indel = np.memmap( + out_dir / "dense_indel_range.npy", np.int64, "w+", shape=(R, 2) + ) + # sample_cols: selected slot -> original sample index (same for every contig). + sample_cols = np.asarray( + [svar2.available_samples.index(s) for s in samples], np.int64 + ) + np.save(out_dir / "sample_cols.npy", sample_cols) + + with open(out_dir / "svar2_meta.json", "w") as f: + json.dump( + { + "vk_snp_range": {"shape": [R, S, P, 2], "dtype": " NDArray[RDTYPE]: leading = rag.shape[:rag_dim] if leading: - out = out.reshape((*leading, out_len)) # pyrefly: ignore[no-matching-overload] + out = out.reshape((*leading, out_len)) return out diff --git a/python/genvarloader/_table.py b/python/genvarloader/_table.py index 4614ded7..d5bebcf6 100644 --- a/python/genvarloader/_table.py +++ b/python/genvarloader/_table.py @@ -13,7 +13,7 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer if TYPE_CHECKING: from numpy.typing import ArrayLike, NDArray diff --git a/python/genvarloader/_variants/_sitesonly.py b/python/genvarloader/_variants/_sitesonly.py index 9803b9f3..0611a3ba 100644 --- a/python/genvarloader/_variants/_sitesonly.py +++ b/python/genvarloader/_variants/_sitesonly.py @@ -9,7 +9,7 @@ import polars as pl import seqpro as sp from genoray import VCF -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from numpy.typing import NDArray from .._dataset._impl import SEQ, ArrayDataset, MaybeTRK diff --git a/skills/genvarloader/SKILL.md b/skills/genvarloader/SKILL.md index ea0da1d5..34be2f5c 100644 --- a/skills/genvarloader/SKILL.md +++ b/skills/genvarloader/SKILL.md @@ -63,7 +63,7 @@ See `docs/source/write.md` for the canonical recipe and BED/BigWig table layouts `.svar` is a sparse columnar variant archive (from `genoray`). Pass it to `gvl.write(variants="x.svar")` exactly like a BCF or PGEN — the resulting dataset stores a back-reference instead of duplicating per-variant arrays. Use SVAR when: -- You need **allele-frequency filtering at read time** (`Dataset.open(min_af=..., max_af=...)` requires SVAR-backed genotypes — will raise otherwise). +- You need **allele-frequency filtering at read time** (`Dataset.open(min_af=..., max_af=...)` is supported for `.svar` only — a `.svar2`-backed dataset raises `NotImplementedError`). - Many datasets share the same variant source — SVAR avoids duplicating `variant_idxs.npy`/`dosages.npy`/`variants.arrow` into each `.gvl` directory. - You're working at population scale and want compact on-disk variant storage. @@ -72,14 +72,38 @@ Use BCF/PGEN directly when you have a one-off dataset and don't need AF filterin Create an SVAR from a normalized VCF/PGEN with `genoray`: ```python -from genoray._svar import dense2sparse -from genoray import VCF +from genoray import VCF, SparseVar -dense2sparse(VCF("normed.bcf"), "normed.svar") # writes a .svar/ directory +SparseVar.from_vcf("normed.svar", VCF("normed.bcf"), max_mem="4g") # writes a .svar/ directory ``` SVARs are resolved at `Dataset.open` time via `metadata.json` → caller `svar=` arg → recorded relative path → recorded absolute path → sibling `*.svar`. See `docs/source/format.md` ("SVAR resolution at open time") and `_dataset/_svar_link.py`. Legacy symlink-based SVAR layouts: run `gvl.migrate_svar_link(path)` once to upgrade. +## `.svar2` — the read-bound sparse variant format + +`.svar2` is genoray's newer sparse columnar variant store. Pass it to `gvl.write` exactly like a `.svar`, BCF, or PGEN — `gvl.write(path, bed, variants="cohort.svar2")` or `variants=SparseVar2("cohort.svar2")`. Like `.svar`, the dataset stores a back-reference (`metadata.json` → `svar2_link`) instead of duplicating per-variant arrays, so the `.svar2` store must remain accessible at read time. + +Unlike `.svar` (whose read path builds an interval search tree + a per-read dense-union over the queried window), a `.svar2`-backed dataset reconstructs via a **read-bound** path: `gvl.write` caches small per-`(region, sample, ploid)` variant-key ranges under `/genotypes/svar2_ranges/` (sized to the dataset's *selected* samples, not the full `.svar2` cohort), and at read time gvl gathers directly off that cache and calls all-Rust kernels — **no interval-search-tree build and no dense-union rebuild per read**. `.svar2` stores are also typically smaller on disk than `.svar`, especially for large cohorts. See `docs/source/faq.md`. + +`.svar2` is resolved at `Dataset.open` time in the same order as `.svar`: caller `svar2=` arg → recorded relative path → recorded absolute path → sibling `*.svar2`. `Dataset.open(path, svar2=)` mirrors `svar=`. See `docs/source/format.md` ("`.svar2` resolution at open time"). + +**Phase-1 scope — unsupported combinations raise `NotImplementedError`.** `.svar2`-backed datasets support all four output modes (`haplotypes`, `variants`, `variant-windows`, and haplotype-realigned `tracks`) byte-identical to the `.svar`/union-oracle backend (except pure-deletion ALT bytes — see below), and `with_seqs("variant-windows")` (`ref="window"`, `alt ∈ {"window", "allele"}`), `unphased_union` (for both `"variants"` and `"variant-windows"` output), and `var_fields`-selected store INFO/FORMAT fields (also for both `"variants"` and `"variant-windows"`; see `var_fields` under `Dataset.open` below) are all fully wired for `.svar2`. The following are still not yet wired and raise a clear error instead of silently mis-computing: +- Spliced output. +- The `var_filter="exonic"` (keep-mask) variant filter. +- `min_af` / `max_af` filtering. +- `annotated` haplotypes (`with_seqs("annotated")`). +- `VarWindowOpt(ref="allele")` (bare-allele REF mode; blocked upstream of `.svar2` too — REF alleles aren't stored). +- In-kernel reverse-complement (`to_rc`). +- Fixed-length (integer `output_length`) haplotype-realigned **track** output (plain haplotype output at a fixed length is fine — only the track kernel is guarded). +- `variants` / `variant-windows` output on a dataset written with `max_jitter>0` or read with `jitter>0` (the read-bound decode does not right-clip to the post-jitter window; haplotypes and tracks are unaffected and support jitter fully). +- `gvl.write(..., extend_to_length=False)` for a `.svar2` variant source (write-time; raises `NotImplementedError` — `.svar2` sources must use the default `extend_to_length=True`). + +(`FlankSample` insertion-fill for tracks spanning multiple contigs in one query is now supported and byte-identical to the `.svar` backend — issue #267.) + +**`variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions (format convention, not a bug).** For a pure deletion (e.g. VCF `GTA>G`), `with_seqs("variants")` on a `.svar` dataset yields the VCF anchor base as ALT (`b"G"`), while a `.svar2` dataset yields the atomized empty ALT (`b""`) — this is how genoray's `.svar2` format represents pure deletions. The same convention carries into `with_seqs("variant-windows")`: `ref_window` is byte-identical between `.svar`/`.svar2`, but `alt`/`alt_window` differ for pure-deletion records for the same reason. Reconstructed **haplotypes are byte-identical** between the two backends (both consume the ALT identically when building sequence); only the raw allele/window bytes differ for pure-deletion records. See `docs/source/faq.md`. + +Symbolic/breakend variants are rejected the same as `.svar`, but for `.svar2` the rejection happens **upstream, at `.svar2` conversion time** (the store format cannot represent them) — a `.svar2` must be built from an already-filtered source; gvl cannot re-filter a materialized `.svar2` any more than it can a materialized `.svar`. + ## `gvl.write` — key arguments ```python @@ -102,7 +126,7 @@ Notable: - `tracks`: a `gvl.BigWigs` (or a list of them), or a `gvl.Table`. Each must have a unique `.name`. BigWigs need a sample→path mapping (dict or table with `sample`, `path` columns; see `BigWigs.from_table`). `gvl.Table` is a core interval-track source backed by a Rust COITrees overlap engine (zero-based half-open coordinates); pass it directly as a `tracks=` or `annot_tracks=` source in `gvl.write`. - `annot_tracks`: `dict[str, str | Path | pl.DataFrame | pl.LazyFrame] | None` — sample-independent annotation tracks, written to `/annot_intervals//`. Each value is either a path to an interval table/bigWig file, or a polars DataFrame/LazyFrame with BED-like columns (`chrom`, `chromStart`, `chromEnd`, `score`). Annotation tracks are sample-independent and can be read without a per-sample variant source. - `max_jitter`: max read-time jitter; pads stored data on both sides of every region by this many bases so `Dataset.with_settings(jitter=j)` works for any `j <= max_jitter`. -- `extend_to_length=True` keeps reading past the BED end until every haplotype is ≥ the region length (matters when deletions would shorten output); set `False` for faster writes if shorter haps are acceptable. +- `extend_to_length=True` keeps reading past the BED end until every haplotype is ≥ the region length (matters when deletions would shorten output); set `False` for faster writes if shorter haps are acceptable. **Not supported for a `.svar2` variant source** — `extend_to_length=False` raises `NotImplementedError` there; only BCF/PGEN/`.svar` sources may disable it. - Inner-joins samples across `variants` and all `tracks`. **Parallelism:** `gvl.write` now parallelizes over write categories. Variants are processed first (serially). Then per-sample `tracks` and `annot_tracks` run concurrently (joblib loky backend). The `max_mem` budget is divided across the concurrently-running categories. @@ -144,16 +168,16 @@ Source: `python/genvarloader/_dataset/_write.py`. gvl.Dataset.open( path, reference=None, jitter=0, rng=None, deterministic=True, rc_neg=True, - min_af=None, max_af=None, # SVAR only + min_af=None, max_af=None, # .svar only — raises NotImplementedError on .svar2 region_names=None, splice_info=None, # see "Spliced haplotypes" var_filter=None, # None | "exonic" var_fields=None, # list[str] | None — see below - *, svar=None, + *, svar=None, svar2=None, ) ``` -Without `reference=`, a genotypes-only dataset opens with the **`"variants"`** view by default (yielding `RaggedVariants`) — `Dataset.open(path)` just works, no `with_seqs` needed. The `"haplotypes"`, `"annotated"`, and `"reference"` views all require a reference; requesting one via `with_seqs` on a reference-less dataset raises a clear `ValueError`. `svar=` overrides the recorded SVAR location. +Without `reference=`, a genotypes-only dataset opens with the **`"variants"`** view by default (yielding `RaggedVariants`) — `Dataset.open(path)` just works, no `with_seqs` needed. The `"haplotypes"`, `"annotated"`, and `"reference"` views all require a reference; requesting one via `with_seqs` on a reference-less dataset raises a clear `ValueError`. `svar=` overrides the recorded SVAR location; `svar2=` mirrors it for a `.svar2`-backed dataset. **`with_settings(dummy_variant=...)`** — inserts a `gvl.DummyVariant` into every **empty** `(region, sample, ploid)` variant group so that every group has at least one variant. Only fills groups that are empty; non-empty groups are unchanged. Valid for both `"variants"` and `"variant-windows"` output kinds; indexing raises `ValueError` if `dummy_variant` is set and the output kind is any other kind (`"haplotypes"`, `"annotated"`, `"reference"`, or no seqs) — the check is order-independent with `with_seqs`. `False` disables dummy padding; `None` (default) leaves the current setting unchanged. Setting `dummy_variant=False` when the output is an unsupported kind is a harmless no-op. @@ -169,6 +193,11 @@ Scalar fields (`start`/`ilen`/`dosage`/`info[...]`) are still filled from `Dummy - **`var_fields: list[str] | None`** — Variant fields to include on `RaggedVariants` output. Defaults to the minimum useful set `["alt", "ilen", "start"]`. Pass additional names (e.g. `"ref"`, `"dosage"`, or any numeric info column in the source variants table) to load them eagerly at open time. Must be a subset of `Dataset.available_var_fields`. Can be reconfigured later via `Dataset.with_settings(var_fields=...)`, which lazily loads any newly-requested columns. `"dosage"` must be requested explicitly — it is *not* added automatically even when `dosages.npy` exists on disk. Beyond the built-ins (`alt`, `start`, `ref`, `ilen`, `dosage`) and per-variant INFO columns, a genoray `.svar` may register arbitrary per-call (`Number=G`) FORMAT fields in `/metadata.json["fields"]`; these appear in `Dataset.available_var_fields` and can be requested via `Dataset.open(..., var_fields=[...])` or `with_settings(var_fields=[...])`. Each surfaces in `variants`, `variant-windows`, and `flat` outputs as a per-call ragged field aligned with the genotypes. A FORMAT field shadows a same-named INFO column. + **On `.svar2`**, `available_var_fields` is `["alt", "ilen", "start"]` plus whatever store fields the + `.svar2` was written with — **`"ref"` and `"dosage"` are not available fields for a `.svar2` source and + requesting either raises** (they are only valid for BCF/PGEN/`.svar`). Beyond that, `var_fields` + additionally exposes the store's own scalar-numeric INFO/FORMAT fields — whichever ones the `.svar2` was written with via `genoray.SparseVar2.from_vcf(info_fields=[...], format_fields=[...])` (bare `str` names also work there). Only scalar-numeric fields can exist in a `.svar2` store at all — INFO/FORMAT `Type=Integer`/`Float` with `Number=1` or `Number=A`, plus INFO `Type=Flag` (stored as bool); anything else is rejected by genoray at write time and never reaches gvl. `gvl` only *reads* whatever the store already carries — it cannot add fields, and re-requesting a field the store doesn't have raises (it isn't in `available_var_fields`). `Dataset.available_var_fields` advertises each store field's key, sourced from `genoray.SparseVar2.available_fields`: the bare field name when it's unique across the store's INFO/FORMAT namespace, else `"INFO/"` / `"FORMAT/"`. A builtin name (`alt`/`start`/`ref`/`ilen`/`dosage`) always wins — a store field that happens to be named e.g. `alt` is never advertised and cannot shadow the builtin. Values keep the store's dtype exactly, with no widening (an `i32` field decodes `int32`, an `f32` field `float32`), and a VCF-missing entry carries the store's stored default verbatim (`NaN` for a float field declared with no default). Both entry points route to the same svar2 reconstructor: `gvl.Dataset.open(path, reference=..., var_fields=[...])` and `ds.with_seqs("variants")`/`.with_settings(var_fields=[...])`. Supported on both output modes: `"variants"` (the field appears on the returned `RaggedVariants`, e.g. `rv["AF"]`, sharing `alt`/`start`/`ilen`'s variant offsets) and `"variant-windows"` (the field appears in `win.fields["AF"]` alongside `start`/`ilen`). A FORMAT field's value is the value for the sample that row belongs to (not sample 0). Empty `(region, sample, ploid)` groups fill each store field via the same `DummyVariant.info[]` mechanism as any other scalar field (see `with_settings(dummy_variant=...)` above): the user-supplied value if given, else `NaN` for a float column or `0` for an integer column. + ## Output modes — `with_seqs` × `with_tracks` `with_seqs(kind)` selects the sequence output channel: @@ -367,6 +396,7 @@ ds.gvl/ ├── input_regions.arrow # BED + region index map ├── genotypes/ # variant_idxs.npy, dosages.npy, variants.arrow │ # (absent when sourced from .svar; see svar_link) +│ # svar2_ranges/ present iff sourced from .svar2 (see svar2_link) ├── intervals// # per-sample track data (BigWigs / Table) └── annot_intervals// # sample-independent annotation track data ``` @@ -391,6 +421,7 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai | Track re-alignment internals | `python/genvarloader/_dataset/_tracks.py`, `_reconstruct.py` | | Insertion fill internals | `python/genvarloader/_dataset/_insertion_fill.py` | | SVAR back-reference / migration | `python/genvarloader/_dataset/_svar_link.py` | +| `.svar2` back-reference / read-bound wiring | `python/genvarloader/_dataset/_svar2_link.py`, `_svar2_haps.py` | | Format 1.x → 2.0 migration internals | `python/genvarloader/_dataset/_migrate.py` | | Flat-buffer ragged containers | `python/genvarloader/_flat.py` | | Flat variants + alleles types | `python/genvarloader/_dataset/_flat_variants.py` | @@ -402,15 +433,17 @@ See `docs/source/format.md` for the full schema, versioning, and SVAR-link detai - **`gvl.update` does not hot-reload open datasets.** A `Dataset` instance that was opened before `gvl.update` ran will not see the new track; reopen the dataset to pick it up. The update itself is safe to run while readers are active — each track is published atomically so a reader never sees a half-written track. - **`Dataset.write_annot_tracks` has been removed.** Use `gvl.update(dataset, annot_tracks={"name": source})` instead, or pass `annot_tracks=` to `gvl.write` at creation time. - **`gvl.Table` is a core public API.** No extra install required. It uses a Rust COITrees overlap engine and is CI-covered. Import it as `gvl.Table` (re-exported from the top-level package). -- **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. +- **Symbolic / breakend variants are rejected, not skipped.** Remove them before `gvl.write` — e.g. `bcftools view -e 'ALT~"<" || ALT~"\["'` (drop SVs and breakends), or construct the genoray reader with `filter=genoray.exprs.is_biallelic & ~genoray.exprs.is_symbolic & ~genoray.exprs.is_breakend`. SVAR inputs must be built from an already-filtered source, since gvl validates but cannot re-filter a materialized `.svar`. For `.svar2` the same variants are rejected **upstream at `.svar2` conversion time** (genoray), not at `gvl.write` time — the store format cannot represent them at all. +- **`.svar2` has a Phase-1 unsupported-combination matrix.** Spliced output, `var_filter="exonic"`, `min_af`/`max_af`, `annotated` haplotypes, `VarWindowOpt(ref="allele")`, in-kernel `to_rc`, fixed-length haplotype-realigned tracks, `variants`/`variant-windows` output with jitter (`max_jitter>0` at write or `jitter>0` at read), and `extend_to_length=False` at write time all raise `NotImplementedError` on a `.svar2`-backed dataset instead of silently mis-computing. `with_seqs("variant-windows")`, `unphased_union`, `var_fields`-selected store INFO/FORMAT fields (on both `"variants"` and `"variant-windows"`), and multi-contig `FlankSample` track fills (issue #267) are now supported for `.svar2`. See "`.svar2` — the read-bound sparse variant format" above. +- **`.svar2` `variants`/`variant-windows` ALT bytes differ from `.svar` for pure deletions.** `.svar` keeps the VCF anchor base (`b"G"` for `GTA>G`); `.svar2` decodes the atomized empty ALT (`b""`). Reconstructed haplotypes are byte-identical either way; `ref_window` is also byte-identical — only raw ALT/`alt_window` bytes differ for pure-deletion records. - Opening a genotypes-only dataset without a `reference=` defaults to the `"variants"` view (`RaggedVariants`), not `"haplotypes"`. Only `"variants"` is available without a reference; `with_seqs("haplotypes" | "annotated" | "reference")` raises `ValueError` if no reference was provided. - `with_insertion_fill` raises unless the dataset has both haplotypes AND tracks active. -- `min_af` / `max_af` raise unless the dataset is SVAR-backed. +- `min_af` / `max_af` are supported for `.svar` only. A non-SVAR-backed dataset (plain BCF/PGEN) raises `RuntimeError`; a `.svar2`-backed dataset raises `NotImplementedError` specifically. - `with_len(L)` requires `L + 2·jitter ≤ min(region_length) + 2·max_jitter` — set `max_jitter` accordingly at `write` time. - Tracks must have unique `.name`; the on-disk layout is `intervals//`. - BED `strand` of `.` is treated as `+`. Reverse-complement happens automatically when `rc_neg=True` (default) and `strand == "-"`. - Splicing is a read-time setting on a *flat* BED of exons — do not pre-concatenate exons before `gvl.write`. -- `extend_to_length=False` at write time will produce haplotypes shorter than the BED region when deletions are present; downstream code must tolerate `<` region length. +- `extend_to_length=False` at write time will produce haplotypes shorter than the BED region when deletions are present; downstream code must tolerate `<` region length. Not an option for a `.svar2` variant source — `gvl.write(..., variants=<.svar2>, extend_to_length=False)` raises `NotImplementedError`. - Missing a `dosage` field on a `RaggedVariants` output you expected? Check `var_fields` — `dosage` must be requested explicitly even if `dosages.npy` exists on disk. - `FlatRagged` / `FlatVariants` offsets are **int64**. PyTorch nested tensors require int32 offsets — cast with `.astype(np.int32)` or `tensor.to(torch.int32)` before passing to `torch.nested.narrow`. - `kind="intervals"` cannot be re-aligned: combining it with a variant-aware seq mode (`haplotypes`/`annotated`/`variants`/`variant-windows`) raises unless `with_settings(realign_tracks=False)`. (Breaking change: `haplotypes`+`intervals` previously returned un-realigned intervals silently under the default.) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 913af48b..9331e5c0 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -1,8 +1,11 @@ //! PyO3 boundary for migrated core kernels. The ONLY place new kernels touch Python. -use ndarray::Array1; -use numpy::{IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2, PyReadwriteArray1}; +use ndarray::{Array1, Array2}; +use numpy::{ + IntoPyArray, PyArray1, PyArray2, PyReadonlyArray1, PyReadonlyArray2, PyReadwriteArray1, +}; use pyo3::prelude::*; use pyo3::types::PyDict; +use std::ops::Range; use crate::variants::windows::{assemble_variants_mode, assemble_windows_mode, VariantBufs}; @@ -31,6 +34,34 @@ fn uninit_output(len: usize) -> Array1 { Array1::from_vec(v) } +/// Marshal a `(n, 2)` int64 array of `[start, end)` pairs into `Range`s. +fn arr2_to_ranges(a: numpy::ndarray::ArrayView2) -> Vec> { + a.rows() + .into_iter() + .map(|r| (r[0] as usize)..(r[1] as usize)) + .collect() +} + +/// Validate that a Python-supplied 1-D array is C-contiguous, or return a Python +/// `ValueError`. Some SVAR2 readbound kernels below slice these arrays and then +/// call `.as_slice()`/`.as_slice_mut()` on the result; a non-contiguous view (e.g. +/// a strided `a[::2]` slice) makes that call panic — a Rust panic surfaces to +/// Python as an uncatchable `pyo3_runtime.PanicException`, not a normal exception +/// — instead of raising. Validate at the FFI boundary so a bad input surfaces as a +/// clean `ValueError`. Only apply this to arrays actually consumed via +/// `.as_slice()` on a stride-preserving view downstream; arrays consumed via +/// `.as_array()` + direct ndarray indexing, `arr2_to_ranges`, or `.to_vec()` are +/// already stride-safe and must NOT be gated here (doing so would reject +/// currently-valid strided inputs). +fn require_contiguous_1d( + arr: &PyReadonlyArray1, + name: &str, +) -> PyResult<()> { + arr.as_slice().map(|_| ()).map_err(|_| { + pyo3::exceptions::PyValueError::new_err(format!("`{name}` must be C-contiguous")) + }) +} + /// Per-(query, hap) reference-length diffs (see `genotypes::get_diffs_sparse`). /// `geno_offsets` is the normalized (2, n) int64 starts/stops array. #[pyfunction] @@ -204,11 +235,8 @@ pub fn compact_keep_i32<'py>( row_offsets: PyReadonlyArray1, keep: PyReadonlyArray1, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::compact_keep_i32( - values.as_array(), - row_offsets.as_array(), - keep.as_array(), - ); + let (v, off) = + variants::compact_keep_i32(values.as_array(), row_offsets.as_array(), keep.as_array()); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -221,11 +249,8 @@ pub fn compact_keep_f32<'py>( row_offsets: PyReadonlyArray1, keep: PyReadonlyArray1, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::compact_keep_f32( - values.as_array(), - row_offsets.as_array(), - keep.as_array(), - ); + let (v, off) = + variants::compact_keep_f32(values.as_array(), row_offsets.as_array(), keep.as_array()); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -238,11 +263,7 @@ pub fn fill_empty_scalar_i32<'py>( offsets: PyReadonlyArray1, fill: i32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_scalar_i32( - data.as_array(), - offsets.as_array(), - fill, - ); + let (v, off) = variants::fill_empty_scalar_i32(data.as_array(), offsets.as_array(), fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -255,11 +276,7 @@ pub fn fill_empty_scalar_f32<'py>( offsets: PyReadonlyArray1, fill: f32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_scalar_f32( - data.as_array(), - offsets.as_array(), - fill, - ); + let (v, off) = variants::fill_empty_scalar_f32(data.as_array(), offsets.as_array(), fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -273,12 +290,7 @@ pub fn fill_empty_fixed_i32<'py>( inner: i64, fill: i32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_fixed_i32( - data.as_array(), - offsets.as_array(), - inner, - fill, - ); + let (v, off) = variants::fill_empty_fixed_i32(data.as_array(), offsets.as_array(), inner, fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -292,12 +304,7 @@ pub fn fill_empty_fixed_f32<'py>( inner: i64, fill: f32, ) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { - let (v, off) = variants::fill_empty_fixed_f32( - data.as_array(), - offsets.as_array(), - inner, - fill, - ); + let (v, off) = variants::fill_empty_fixed_f32(data.as_array(), offsets.as_array(), inner, fill); (v.into_pyarray(py), off.into_pyarray(py)) } @@ -322,7 +329,11 @@ pub fn fill_empty_seq_u8<'py>( seq_offsets.as_array(), dummy.as_array(), ); - (nd.into_pyarray(py), nvar.into_pyarray(py), nseq.into_pyarray(py)) + ( + nd.into_pyarray(py), + nvar.into_pyarray(py), + nseq.into_pyarray(py), + ) } /// Two-level dummy-fill for token windows (int32). @@ -346,7 +357,11 @@ pub fn fill_empty_seq_i32<'py>( seq_offsets.as_array(), dummy.as_array(), ); - (nd.into_pyarray(py), nvar.into_pyarray(py), nseq.into_pyarray(py)) + ( + nd.into_pyarray(py), + nvar.into_pyarray(py), + nseq.into_pyarray(py), + ) } /// Build the `{name: (data, seq_offsets)}` dict from assembled buffers. @@ -461,9 +476,26 @@ pub fn assemble_variant_buffers_u8<'py>( pad_char: u8, ) -> Bound<'py, PyDict> { assemble_variant_buffers_impl::( - py, mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, - ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, - lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char, + py, + mode, + v_idxs, + row_offsets, + alt_global, + alt_off_global, + ref_global, + ref_off_global, + want_ref_bytes, + want_flank, + ref_mode, + alt_mode, + flank_len, + lut, + v_contigs, + v_starts, + ilens, + reference, + ref_offsets, + pad_char, ) } @@ -493,9 +525,26 @@ pub fn assemble_variant_buffers_i32<'py>( pad_char: u8, ) -> Bound<'py, PyDict> { assemble_variant_buffers_impl::( - py, mode, v_idxs, row_offsets, alt_global, alt_off_global, ref_global, - ref_off_global, want_ref_bytes, want_flank, ref_mode, alt_mode, flank_len, - lut, v_contigs, v_starts, ilens, reference, ref_offsets, pad_char, + py, + mode, + v_idxs, + row_offsets, + alt_global, + alt_off_global, + ref_global, + ref_off_global, + want_ref_bytes, + want_flank, + ref_mode, + alt_mode, + flank_len, + lut, + v_contigs, + v_starts, + ilens, + reference, + ref_offsets, + pad_char, ) } @@ -574,7 +623,7 @@ pub fn reconstruct_haplotypes_from_sparse( }); } -/// Fused haplotypes __getitem__ kernel (Task 13). +/// Fused haplotypes __getitem__ kernel. /// /// Collapses two FFI crossings into one: /// 1. Compute per-haplotype length diffs (``get_diffs_sparse`` logic). @@ -591,7 +640,8 @@ pub fn reconstruct_haplotypes_from_sparse( /// layout as the existing ``reconstruct_haplotypes_from_sparse`` FFI entry). /// /// Annotation buffers are not supported in the fused entry (annotated path -/// remains on the unfused dispatch wrappers — see Task 13 report for rationale). +/// remains on the unfused dispatch wrappers, which carry the extra +/// `annot_*` output buffers this fused entry does not allocate). /// `parallel` enables rayon batch parallelism (caller computes `should_parallelize`). #[pyfunction] #[allow(clippy::too_many_arguments)] @@ -733,6 +783,863 @@ pub fn reconstruct_haplotypes_fused<'py>( (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) } +/// Fused SVAR2 two-source haplotype reconstruction: merge each hap's `var_key` ⋈ +/// `dense` channels and decode via `svar2-codec` inline (no materialized global +/// variant table), sizing and allocating the output buffer in Rust — one FFI +/// crossing, mirrors `reconstruct_haplotypes_fused` above. +/// +/// `output_length`: +/// - ``-1`` → ragged mode (each haplotype gets its natural length = ref_len + diff). +/// - ``>= 0`` → fixed-length mode (every haplotype is padded/truncated to this length). +/// +/// No annotation, no to_rc; mirrors the plain fused path. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2<'py>( + py: Python<'py>, + regions: PyReadonlyArray2, + shifts: PyReadonlyArray2, + vk_pos: PyReadonlyArray1, + vk_key: PyReadonlyArray1, + vk_off: PyReadonlyArray1, + dense_pos: PyReadonlyArray1, + dense_key: PyReadonlyArray1, + dense_range: PyReadonlyArray2, + dense_present: PyReadonlyArray1, + dense_present_off: PyReadonlyArray1, + lut_bytes: PyReadonlyArray1, + lut_off: PyReadonlyArray1, + ref_: PyReadonlyArray1, + ref_offsets: PyReadonlyArray1, + pad_char: u8, + output_length: i64, + parallel: bool, +) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { + use crate::reconstruct; + use crate::svar2; + + let regions_a = regions.as_array(); + let shifts_a = shifts.as_array(); + let vk_pos_a = vk_pos.as_array(); + let vk_key_a = vk_key.as_array(); + let vk_off_a = vk_off.as_array(); + let dense_pos_a = dense_pos.as_array(); + let dense_key_a = dense_key.as_array(); + let dense_range_a = dense_range.as_array(); + let dense_present_a = dense_present.as_array(); + let dense_present_off_a = dense_present_off.as_array(); + let lut_bytes_a = lut_bytes.as_array(); + let lut_off_a = lut_off.as_array(); + let ref_a = ref_.as_array(); + let ref_offsets_a = ref_offsets.as_array(); + + let ploidy = shifts_a.ncols(); + let n_q = regions_a.nrows(); + let n_work = n_q * ploidy; + + let (out_data, out_offsets_vec) = py.detach(move || { + // Step 1: compute per-haplotype length diffs via the two-source diff core. + let vk_pos_s: &[i32] = vk_pos_a.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key_a.as_slice().unwrap(); + let vk_off_s: &[i64] = vk_off_a.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos_a.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key_a.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present_a.as_slice().unwrap(); + let dense_present_off_s: &[i64] = dense_present_off_a.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes_a.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off_a.as_slice().unwrap(); + + let diffs = svar2::hap_diffs_svar2( + regions_a, + ploidy, + vk_pos_s, + vk_key_s, + vk_off_s, + dense_pos_s, + dense_key_s, + dense_range_a, + dense_present_s, + dense_present_off_s, + lut_bytes_s, + lut_off_s, + ); + + // Step 2: compute per-haplotype output lengths and prefix-sum offsets. + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let len: i64 = if output_length >= 0 { + output_length + } else { + let ref_len = (regions_a[[query, 2]] - regions_a[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + (ref_len + diff).max(0) + }; + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust — Python never calls np.empty. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = uninit_output(total); + + // Step 4: reconstruct all haplotypes into the owned buffer. + reconstruct::reconstruct_haplotypes_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions_a, + shifts_a, + vk_pos_a, + vk_key_a, + vk_off_a, + dense_pos_a, + dense_key_a, + dense_range_a, + dense_present_a, + dense_present_off_a, + lut_bytes_a, + lut_off_a, + ref_a, + ref_offsets_a, + pad_char, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) +} + +/// Read-bound SVAR2 haplotype reconstruction: gather off a query-only genoray +/// `Svar2Store` reader with NO interval-search-tree rebuild and NO dense-union +/// rebuild (`genoray_core::query::gather_haps_readbound`), marshal the split +/// result into the flat layout via [`crate::svar2::split_to_flat`], then reuse +/// the byte-validated [`reconstruct_haplotypes_from_svar2`] kernel unchanged — +/// one FFI crossing, byte-identical to the union-path oracle. +/// +/// `region_starts`/`orig_samples`/`vk_snp_range`/`vk_indel_range`/ +/// `dense_snp_range`/`dense_indel_range` are the per-query outputs of +/// `SparseVar2.find_ranges` (flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_haps`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + shifts: PyReadonlyArray2, + ref_: PyReadonlyArray1, + ref_offsets: PyReadonlyArray1, + pad_char: u8, + output_length: i64, + parallel: bool, +) -> PyResult<(Bound<'py, PyArray1>, Bound<'py, PyArray1>)> { + use crate::reconstruct; + use crate::svar2; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let shifts_a = shifts.as_array(); + let ploidy = shifts_a.ncols(); + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end) — `ref_` is the + // single contig slice the caller passed in (ref_offsets = [0, len]). + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + + // `ref_` is sliced (`ref_.slice(s![c_s..c_e])`) and then `.as_slice().unwrap()`'d + // inside `reconstruct::reconstruct_haplotypes_from_svar2` (src/reconstruct/mod.rs) — + // a non-contiguous `ref_` (e.g. `ref_[::2]`) panics there. `ref_offsets` is only + // ever indexed directly (`ref_offsets[c_idx]`), which is stride-safe, so it is not + // gated here. + require_contiguous_1d(&ref_, "ref_")?; + + // NOTE: the pure-DEL anchor-base read inside + // `reconstruct::reconstruct_haplotypes_from_svar2` + // (`&contig_ref_s[pos as usize..pos as usize + 1]`, src/reconstruct/mod.rs) is + // in-bounds for all valid input: gathered variants come from within-contig records + // so `pos < contig_ref_len` always holds. It is NOT bounded here by the query + // window: gvl regions legitimately extend past the contig end (jitter / max_jitter + // padding, right-padded with `pad_char`), so `region_bounds[q, 1] > contig_ref_len` + // is a normal, valid read — rejecting it would break SVAR1 parity. The only way to + // reach the anchor OOB is a corrupt store (a variant `pos >= contig_len`); that is + // caught by a `debug_assert!` at the read site (fires in test/debug builds). + + let ref_a = ref_.as_array(); + let ref_offsets_a = ref_offsets.as_array(); + + let (out_data, out_offsets_vec) = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + // Step 1: size via the same two-source diff core the union path uses. + let diffs = svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ); + + // Step 2: per-haplotype output lengths and prefix-sum offsets. + let n_work = n_q * ploidy; + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let len: i64 = if output_length >= 0 { + output_length + } else { + let ref_len = (regions[[query, 2]] - regions[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + (ref_len + diff).max(0) + }; + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = uninit_output(total); + + // Step 4: reconstruct — reuse the byte-validated union-path kernel + // unchanged, now fed the read-bound gather's flat channels. + reconstruct::reconstruct_haplotypes_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions.view(), + shifts_a, + numpy::ndarray::ArrayView1::from(flat.vk_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_key.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_off.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_key.as_slice()), + dense_range_a, + numpy::ndarray::ArrayView1::from(flat.dense_present.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_present_off.as_slice()), + numpy::ndarray::ArrayView1::from(lut_bytes.as_slice()), + numpy::ndarray::ArrayView1::from(lut_off.as_slice()), + ref_a, + ref_offsets_a, + pad_char, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) +} + +/// Read-bound SVAR2 per-hap ilen diffs: the same gather +/// (`genoray_core::query::gather_haps_readbound` + [`crate::svar2::split_to_flat`]) as +/// [`reconstruct_haplotypes_from_svar2_readbound`], but stops after +/// [`crate::svar2::hap_diffs_svar2`] and returns just the `(n_q, ploidy)` diffs — +/// no reconstruct sizing/allocation/kernel pass. Used by the dataset read path to +/// compute random jitter shifts from diffs BEFORE reconstructing (mirrors how the +/// SVAR1 path derives shifts from diffs in `_prepare_request`). +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range`/`region_bounds` +/// argument semantics (the per-query outputs of `SparseVar2.find_ranges`, +/// flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_diffs`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn hap_diffs_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + ploidy: usize, +) -> PyResult>> { + use crate::svar2; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end) — matches the + // reconstruct-readbound FFI's convention. + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + + let diffs = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ) + }); + + Ok(diffs.into_pyarray(py)) +} + +/// Read-bound SVAR2 track re-alignment: gather off a query-only genoray +/// `Svar2Store` reader with NO interval-search-tree rebuild and NO dense-union +/// rebuild (`genoray_core::query::gather_haps_readbound`), marshal the split +/// result into the flat layout via [`crate::svar2::split_to_flat`], then reuse +/// the byte-validated [`shift_and_realign_tracks_from_svar2`] kernel unchanged — +/// one FFI crossing, byte-identical to the union-path oracle. +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range`/`region_bounds` +/// argument semantics (the per-query outputs of `SparseVar2.find_ranges`, +/// flattened region-major, sample-minor); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_tracks`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + region_bounds: PyReadonlyArray2, + shifts: PyReadonlyArray2, + tracks: PyReadonlyArray1, + track_offsets: PyReadonlyArray1, + params: PyReadonlyArray1, + strategy_id: i64, + base_seed: u64, + global_query: PyReadonlyArray1, + parallel: bool, +) -> PyResult<(Bound<'py, PyArray1>, Bound<'py, PyArray1>)> { + use crate::svar2; + use crate::tracks; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let shifts_a = shifts.as_array(); + let ploidy = shifts_a.ncols(); + let region_bounds_a = region_bounds.as_array(); + let n_q = region_bounds_a.nrows(); + + // Build `regions` (n_q, 3) as [contig_idx=0, start, end). + let mut regions = Array2::::zeros((n_q, 3)); + for q in 0..n_q { + regions[[q, 1]] = region_bounds_a[[q, 0]]; + regions[[q, 2]] = region_bounds_a[[q, 1]]; + } + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + + // `tracks` is `.as_slice().expect("tracks must be contiguous (C-order)")`'d inside + // `tracks::shift_and_realign_tracks_from_svar2` (src/tracks/mod.rs) — a + // non-contiguous `tracks` (e.g. `tracks[::2]`) panics there. `track_offsets` and + // `params` are only ever indexed directly (`track_offsets[query]`, `params[0]`), + // which is stride-safe, so neither is gated here. + require_contiguous_1d(&tracks, "tracks")?; + + let tracks_a = tracks.as_array(); + let track_offsets_a = track_offsets.as_array(); + let params_a = params.as_array(); + // (n_q,) LOCAL query -> GLOBAL batch row map for the FlankSample fill seed; + // the read-bound path calls this kernel once per contig group, so the local + // `k / ploidy` index would otherwise diverge from the single fused SVAR1 call + // (issue #267). + let global_query_a = global_query.as_array(); + + let (out_data, out_offsets_vec) = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + let br = genoray_core::query::gather_haps_readbound(reader, &rb); + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + let flat = svar2::split_to_flat(&br); + let dense_range_a = + numpy::ndarray::ArrayView2::from_shape((n_q, 2), &flat.dense_range).unwrap(); + + // Step 1: size via the same two-source diff core the union path uses. + let diffs = svar2::hap_diffs_svar2( + regions.view(), + ploidy, + &flat.vk_pos, + &flat.vk_key, + &flat.vk_off, + &flat.dense_pos, + &flat.dense_key, + dense_range_a, + &flat.dense_present, + &flat.dense_present_off, + &lut_bytes, + &lut_off, + ); + + // Step 2: per-haplotype output lengths and prefix-sum offsets — tracks + // always size to ref_len + diff (no `output_length` override). + let n_work = n_q * ploidy; + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let ref_len = (regions[[query, 2]] - regions[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + let len: i64 = (ref_len + diff).max(0); + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = Array1::::zeros(total); + + // Step 4: realign — reuse the byte-validated union-path kernel unchanged, + // now fed the read-bound gather's flat channels. + tracks::shift_and_realign_tracks_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions.view(), + shifts_a, + numpy::ndarray::ArrayView1::from(flat.vk_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_key.as_slice()), + numpy::ndarray::ArrayView1::from(flat.vk_off.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_pos.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_key.as_slice()), + dense_range_a, + numpy::ndarray::ArrayView1::from(flat.dense_present.as_slice()), + numpy::ndarray::ArrayView1::from(flat.dense_present_off.as_slice()), + numpy::ndarray::ArrayView1::from(lut_bytes.as_slice()), + numpy::ndarray::ArrayView1::from(lut_off.as_slice()), + tracks_a, + track_offsets_a, + params_a, + strategy_id, + base_seed, + Some(global_query_a), + parallel, + ); + + (out_data, out_offsets_vec) + }); + + Ok((out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py))) +} + +/// Read-bound SVAR2 variants decode: gather off a query-only genoray `Svar2Store` +/// reader with NO interval-search-tree rebuild and NO dense-union rebuild +/// (`genoray_core::query::gather_haps_readbound`), then decode each hap's merged +/// `var_key ⋈ dense` keys via [`crate::svar2::decode_variants_from_split`] — one +/// FFI crossing, mirroring genoray's `decode_hap` (no overlap/clip filter; the +/// gather already restricts to overlapping variants). +/// +/// See [`reconstruct_haplotypes_from_svar2_readbound`] for the shared +/// `region_starts`/`orig_samples`/`vk_*_range`/`dense_*_range` argument semantics +/// (the per-query outputs of `SparseVar2.find_ranges`, flattened region-major, +/// sample-minor). `ploidy` is passed explicitly (there is no `shifts` array to +/// infer it from here). +/// +/// `fields` is a list of `(category, name, dtype_str)` triples (`category` is +/// `"info"` or `"format"`; `dtype_str` is genoray's `StorageDtype` meta string, +/// e.g. `"i32"`), and may be empty. When non-empty, this opens the four +/// `FieldSub`-keyed `FieldView`s per field and gathers their bytes alongside the +/// variant decode via the var_key-provenance-tracking +/// `genoray_core::query::gather_haps_readbound_src` (plain `gather_haps_readbound` +/// is used when `fields` is empty, since it doesn't need that provenance). +/// +/// Returns the `RaggedVariants` SoA `(pos, ilen, alt_bytes, str_off, var_off)` +/// plus, per requested field in `fields` order, a flat `u8` byte buffer and its +/// per-value itemsize (`field_bufs`, `field_itemsizes`); see +/// `python/genvarloader/_dataset/_svar2_store_py.py::build_readbound_variants`. +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn decode_variants_from_svar2_readbound<'py>( + py: Python<'py>, + store: PyRef<'py, crate::svar2::store::Svar2Store>, + contig: &str, + region_starts: PyReadonlyArray1, + orig_samples: PyReadonlyArray1, + vk_snp_range: PyReadonlyArray2, + vk_indel_range: PyReadonlyArray2, + dense_snp_range: PyReadonlyArray2, + dense_indel_range: PyReadonlyArray2, + ploidy: usize, + fields: Vec<(String, String, String)>, +) -> PyResult<( + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + Vec>>, + Vec, +)> { + use crate::svar2; + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + use genoray_core::query::FieldView; + + let reader = store.reader(contig).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("contig {contig} not in store")) + })?; + + let region_starts_v: Vec = region_starts.as_array().to_vec(); + let orig_samples_v: Vec = orig_samples + .as_array() + .iter() + .map(|&x| x as usize) + .collect(); + let vk_snp_range_v = arr2_to_ranges(vk_snp_range.as_array()); + let vk_indel_range_v = arr2_to_ranges(vk_indel_range.as_array()); + let dense_snp_range_v = arr2_to_ranges(dense_snp_range.as_array()); + let dense_indel_range_v = arr2_to_ranges(dense_indel_range.as_array()); + + let n_samples = reader.n_samples(); + let paths = ContigPaths::new(store.store_path(), contig); + + let gathers: Vec = fields + .iter() + .map(|(cat, name, dtype_str)| { + let dtype = StorageDtype::from_meta_str(dtype_str).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!( + "field {name}: unknown storage dtype {dtype_str:?}" + )) + })?; + let width = dtype.width_bytes().ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!( + "field {name}: unresolved dtype" + )) + })?; + // views MUST be in FieldSub::all() order — FieldGather indexes them by sub_ix. + let mut views = Vec::with_capacity(4); + for sub in FieldSub::all() { + views.push( + FieldView::open(&paths, cat, name, sub, dtype, n_samples).map_err(|e| { + pyo3::exceptions::PyIOError::new_err(format!("open field {name}: {e}")) + })?, + ); + } + let views: [FieldView; 4] = views + .try_into() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("expected 4 field views"))?; + Ok(svar2::FieldGather { + views, + is_format: cat == "format", + width, + cohort_n_samples: n_samples, + }) + }) + .collect::>>()?; + + let itemsizes: Vec = gathers.iter().map(|g| g.width).collect(); + let has_fields = !gathers.is_empty(); + + let (soa, field_bufs) = py.detach(move || { + let rb = genoray_core::query::HapRanges::new( + ®ion_starts_v, + &orig_samples_v, + &vk_snp_range_v, + &vk_indel_range_v, + &dense_snp_range_v, + &dense_indel_range_v, + ploidy, + ); + // Field gather needs var_key provenance (vk_src), which ONLY the _src + // variant populates. + let br = if has_fields { + genoray_core::query::gather_haps_readbound_src(reader, &rb) + } else { + genoray_core::query::gather_haps_readbound(reader, &rb) + }; + + let (lut_bytes, lut_off_u64) = reader.lut_arrays(); + let lut_off: Vec = lut_off_u64.iter().map(|&x| x as i64).collect(); + + svar2::decode_variants_from_split( + &br, + &lut_bytes, + &lut_off, + &gathers, + // ON-DISK dense windows (from find_ranges / HapRanges), NOT br's output windows. + &dense_snp_range_v, + &dense_indel_range_v, + &orig_samples_v, + ) + }); + + let field_out: Vec>> = field_bufs + .into_iter() + .map(|b| Array1::from_vec(b).into_pyarray(py)) + .collect(); + + Ok(( + Array1::from_vec(soa.pos).into_pyarray(py), + Array1::from_vec(soa.ilen).into_pyarray(py), + Array1::from_vec(soa.alt_bytes).into_pyarray(py), + Array1::from_vec(soa.str_off).into_pyarray(py), + Array1::from_vec(soa.var_off).into_pyarray(py), + field_out, + itemsizes, + )) +} + +/// Fused SVAR2 two-source track shift+realign: merge each hap's `var_key` ⋈ `dense` +/// channels and decode via `svar2-codec` inline, sizing and allocating the output +/// buffer in Rust — one FFI crossing, mirrors `reconstruct_haplotypes_from_svar2` +/// above but for f32 tracks (see `tracks::shift_and_realign_tracks_from_svar2`). +#[pyfunction] +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2<'py>( + py: Python<'py>, + regions: PyReadonlyArray2, + shifts: PyReadonlyArray2, + vk_pos: PyReadonlyArray1, + vk_key: PyReadonlyArray1, + vk_off: PyReadonlyArray1, + dense_pos: PyReadonlyArray1, + dense_key: PyReadonlyArray1, + dense_range: PyReadonlyArray2, + dense_present: PyReadonlyArray1, + dense_present_off: PyReadonlyArray1, + lut_bytes: PyReadonlyArray1, + lut_off: PyReadonlyArray1, + tracks: PyReadonlyArray1, + track_offsets: PyReadonlyArray1, + params: PyReadonlyArray1, + strategy_id: i64, + base_seed: u64, + parallel: bool, +) -> (Bound<'py, PyArray1>, Bound<'py, PyArray1>) { + use crate::svar2; + use crate::tracks; + + let regions_a = regions.as_array(); + let shifts_a = shifts.as_array(); + let vk_pos_a = vk_pos.as_array(); + let vk_key_a = vk_key.as_array(); + let vk_off_a = vk_off.as_array(); + let dense_pos_a = dense_pos.as_array(); + let dense_key_a = dense_key.as_array(); + let dense_range_a = dense_range.as_array(); + let dense_present_a = dense_present.as_array(); + let dense_present_off_a = dense_present_off.as_array(); + let lut_bytes_a = lut_bytes.as_array(); + let lut_off_a = lut_off.as_array(); + let tracks_a = tracks.as_array(); + let track_offsets_a = track_offsets.as_array(); + let params_a = params.as_array(); + + let ploidy = shifts_a.ncols(); + let n_q = regions_a.nrows(); + let n_work = n_q * ploidy; + + let (out_data, out_offsets_vec) = py.detach(move || { + // Step 1: compute per-haplotype length diffs via the two-source diff core + // (a realigned track has haplotype length = ref_len + diff, same as reconstruct). + let vk_pos_s: &[i32] = vk_pos_a.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key_a.as_slice().unwrap(); + let vk_off_s: &[i64] = vk_off_a.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos_a.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key_a.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present_a.as_slice().unwrap(); + let dense_present_off_s: &[i64] = dense_present_off_a.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes_a.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off_a.as_slice().unwrap(); + + let diffs = svar2::hap_diffs_svar2( + regions_a, + ploidy, + vk_pos_s, + vk_key_s, + vk_off_s, + dense_pos_s, + dense_key_s, + dense_range_a, + dense_present_s, + dense_present_off_s, + lut_bytes_s, + lut_off_s, + ); + + // Step 2: compute per-haplotype output lengths and prefix-sum offsets. + let mut out_offsets_vec: Array1 = Array1::zeros(n_work + 1); + { + let mut acc: i64 = 0; + out_offsets_vec[0] = 0; + for k in 0..n_work { + let query = k / ploidy; + let hap = k % ploidy; + let ref_len = (regions_a[[query, 2]] - regions_a[[query, 1]]) as i64; + let diff = diffs[[query, hap]] as i64; + let len: i64 = (ref_len + diff).max(0); + acc += len; + out_offsets_vec[k + 1] = acc; + } + } + + // Step 3: allocate the output buffer in Rust — Python never calls np.empty. + // f32 track fill writes every position it needs; zeros is a safe default. + let total = out_offsets_vec[n_work] as usize; + let mut out_data: Array1 = Array1::::zeros(total); + + // Step 4: realign all tracks into the owned buffer. + tracks::shift_and_realign_tracks_from_svar2( + out_data.view_mut(), + out_offsets_vec.view(), + regions_a, + shifts_a, + vk_pos_a, + vk_key_a, + vk_off_a, + dense_pos_a, + dense_key_a, + dense_range_a, + dense_present_a, + dense_present_off_a, + lut_bytes_a, + lut_off_a, + tracks_a, + track_offsets_a, + params_a, + strategy_id, + base_seed, + // Single fused call over the whole batch: `k / ploidy` IS the global + // row, so the FlankSample seed needs no remap (issue #267). + None, + parallel, + ); + + (out_data, out_offsets_vec) + }); + + (out_data.into_pyarray(py), out_offsets_vec.into_pyarray(py)) +} + /// Fused spliced-haplotype reconstruction: reconstruct in one FFI crossing using /// precomputed output offsets. /// @@ -940,7 +1847,7 @@ pub fn reconstruct_annotated_haplotypes_spliced_fused<'py>( pad_char, keep_a, keep_offsets_a, - Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide + Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide Some(annot_pos.view_mut()), // annot_ref_pos — reference coordinate per nucleotide parallel, ); @@ -956,9 +1863,21 @@ pub fn reconstruct_annotated_haplotypes_spliced_fused<'py>( out_offsets_a.len() - 1, "to_rc mask length must equal number of output rows (offsets.len() - 1)" ); - crate::reverse::rc_flat_rows_inplace(out_data.as_slice_mut().unwrap(), out_offsets_a, m); - crate::reverse::reverse_flat_rows_inplace(annot_v.as_slice_mut().unwrap(), out_offsets_a, m); - crate::reverse::reverse_flat_rows_inplace(annot_pos.as_slice_mut().unwrap(), out_offsets_a, m); + crate::reverse::rc_flat_rows_inplace( + out_data.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_v.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_pos.as_slice_mut().unwrap(), + out_offsets_a, + m, + ); } (out_data, annot_v, annot_pos) @@ -1116,7 +2035,7 @@ pub fn reconstruct_annotated_haplotypes_fused<'py>( pad_char, keep_a, keep_offsets_a, - Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide + Some(annot_v.view_mut()), // annot_v_idxs — variant index per nucleotide Some(annot_pos.view_mut()), // annot_ref_pos — reference coordinate per nucleotide parallel, ); @@ -1128,9 +2047,21 @@ pub fn reconstruct_annotated_haplotypes_fused<'py>( out_offsets_vec.len() - 1, "to_rc mask length must equal number of output rows (offsets.len() - 1)" ); - crate::reverse::rc_flat_rows_inplace(out_data.as_slice_mut().unwrap(), out_offsets_vec.view(), m); - crate::reverse::reverse_flat_rows_inplace(annot_v.as_slice_mut().unwrap(), out_offsets_vec.view(), m); - crate::reverse::reverse_flat_rows_inplace(annot_pos.as_slice_mut().unwrap(), out_offsets_vec.view(), m); + crate::reverse::rc_flat_rows_inplace( + out_data.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_v.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); + crate::reverse::reverse_flat_rows_inplace( + annot_pos.as_slice_mut().unwrap(), + out_offsets_vec.view(), + m, + ); } (out_data, annot_v, annot_pos, out_offsets_vec) @@ -1278,7 +2209,7 @@ pub fn tracks_to_intervals<'py>( ) } -/// Fused per-track __getitem__ kernel (Task 14). +/// Fused per-track __getitem__ kernel. /// /// Collapses two FFI crossings into one per track: /// 1. ``intervals_to_tracks`` core: fills a Rust-side scratch buffer from @@ -1301,22 +2232,22 @@ pub fn tracks_to_intervals<'py>( #[allow(clippy::too_many_arguments)] pub fn intervals_and_realign_track_fused( py: Python<'_>, - mut out: PyReadwriteArray1, // (b*p*l) — caller's per-track slice - out_offsets: PyReadonlyArray1, // (b*p + 1) - regions: PyReadonlyArray2, // (b, 3) - shifts: PyReadonlyArray2, // (b, p) - geno_offset_idx: PyReadonlyArray2, // (b, p) - geno_v_idxs: PyReadonlyArray1, // (r*s*p*v) - geno_offsets: PyReadonlyArray2, // (2, r*s*p) - v_starts: PyReadonlyArray1, // (tot_v) - ilens: PyReadonlyArray1, // (tot_v) + mut out: PyReadwriteArray1, // (b*p*l) — caller's per-track slice + out_offsets: PyReadonlyArray1, // (b*p + 1) + regions: PyReadonlyArray2, // (b, 3) + shifts: PyReadonlyArray2, // (b, p) + geno_offset_idx: PyReadonlyArray2, // (b, p) + geno_v_idxs: PyReadonlyArray1, // (r*s*p*v) + geno_offsets: PyReadonlyArray2, // (2, r*s*p) + v_starts: PyReadonlyArray1, // (tot_v) + ilens: PyReadonlyArray1, // (tot_v) // intervals (reference-coordinate, for this track) - offset_idxs: PyReadonlyArray1, // (b) — per-query index into itv_offsets - itv_starts: PyReadonlyArray1, // (n_intervals) - itv_ends: PyReadonlyArray1, // (n_intervals) - itv_values: PyReadonlyArray1, // (n_intervals) - itv_offsets: PyReadonlyArray1, // (n_samples*n_regions + 1) - track_offsets: PyReadonlyArray1, // (b+1) — out_offsets for scratch buffer + offset_idxs: PyReadonlyArray1, // (b) — per-query index into itv_offsets + itv_starts: PyReadonlyArray1, // (n_intervals) + itv_ends: PyReadonlyArray1, // (n_intervals) + itv_values: PyReadonlyArray1, // (n_intervals) + itv_offsets: PyReadonlyArray1, // (n_samples*n_regions + 1) + track_offsets: PyReadonlyArray1, // (b+1) — out_offsets for scratch buffer // insertion-fill strategy params: PyReadonlyArray1, strategy_id: i64, @@ -1420,9 +2351,9 @@ pub fn intervals_and_realign_track_fused( Ok(()) } -// ── Task 3: guard test — drives rc_flat_rows_inplace on a synthetic hap buffer ─ -// ── Task 4: guard test — drives reverse_flat_rows_inplace:: (reverse only) ─ -// ── Task 6: guard test — proves per-element masking over permuted offsets ──────── +// ── guard test — drives rc_flat_rows_inplace on a synthetic hap buffer ─ +// ── guard test — drives reverse_flat_rows_inplace:: (reverse only) ─ +// ── guard test — proves per-element masking over permuted offsets ──────── #[cfg(test)] mod tests { #[test] @@ -1456,9 +2387,9 @@ mod tests { #[test] fn annotated_rc_complements_bytes_reverses_indices() { - let mut bytes = b"ACG".to_vec(); // revcomp -> "CGT" - let mut vidx = vec![5i32, 6, 7]; // reverse -> [7,6,5] - let mut rpos = vec![100i32, 101, 102]; // reverse -> [102,101,100] + let mut bytes = b"ACG".to_vec(); // revcomp -> "CGT" + let mut vidx = vec![5i32, 6, 7]; // reverse -> [7,6,5] + let mut rpos = vec![100i32, 101, 102]; // reverse -> [102,101,100] let offsets = ndarray::array![0i64, 3]; let m = ndarray::array![true]; crate::reverse::rc_flat_rows_inplace(&mut bytes, offsets.view(), m.view()); @@ -1470,9 +2401,9 @@ mod tests { } } -// ── DEBUG exports for PRNG parity tests (Task 7) ───────────────────────────── +// ── DEBUG exports for PRNG parity tests ───────────────────────────── // These thin wrappers exist solely to make the Rust PRNG functions callable from -// Python tests. Decision (final-review, Task 15): KEEP permanently as the direct +// Python tests. Decision: KEEP permanently as the direct // PRNG parity guard. The njit-internal xorshift64/hash4 leaves have no other // Python entry point, so these are the only way to assert byte-identity of the // PRNG core from test_prng_parity.py. Do NOT remove. diff --git a/src/lib.rs b/src/lib.rs index 096545ef..eed4c43e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod ragged; pub mod reconstruct; pub mod reference; pub mod reverse; +pub mod svar2; pub mod tables; pub mod tracks; pub mod variants; @@ -19,6 +20,7 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(bigwig_intervals, m)?)?; m.add_function(wrap_pyfunction!(bigwig_write_track, m)?)?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(ragged::ragged_to_padded, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_to_tracks, m)?)?; m.add_function(wrap_pyfunction!(ffi::get_diffs_sparse, m)?)?; @@ -38,15 +40,45 @@ fn genvarloader(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(ffi::assemble_variant_buffers_i32, m)?)?; m.add_function(wrap_pyfunction!(ffi::rc_alleles, m)?)?; m.add_function(wrap_pyfunction!(ffi::get_reference, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_from_sparse, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_from_sparse, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_annotated_haplotypes_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_spliced_fused, m)?)?; - m.add_function(wrap_pyfunction!(ffi::reconstruct_annotated_haplotypes_spliced_fused, m)?)?; + m.add_function(wrap_pyfunction!(ffi::reconstruct_haplotypes_from_svar2, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_from_svar2_readbound, + m + )?)?; + m.add_function(wrap_pyfunction!(ffi::hap_diffs_from_svar2_readbound, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_annotated_haplotypes_fused, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_haplotypes_spliced_fused, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::reconstruct_annotated_haplotypes_spliced_fused, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::shift_and_realign_tracks_sparse, m)?)?; + m.add_function(wrap_pyfunction!( + ffi::shift_and_realign_tracks_from_svar2, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::shift_and_realign_tracks_from_svar2_readbound, + m + )?)?; + m.add_function(wrap_pyfunction!( + ffi::decode_variants_from_svar2_readbound, + m + )?)?; m.add_function(wrap_pyfunction!(ffi::tracks_to_intervals, m)?)?; m.add_function(wrap_pyfunction!(ffi::intervals_and_realign_track_fused, m)?)?; - // DEBUG: PRNG parity exports (Task 7) — keep or remove after Task 8/9 review + // DEBUG: PRNG parity exports used by the Python parity tests. m.add_function(wrap_pyfunction!(ffi::_debug_xorshift64, m)?)?; m.add_function(wrap_pyfunction!(ffi::_debug_hash4, m)?)?; Ok(()) diff --git a/src/reconstruct/mod.rs b/src/reconstruct/mod.rs index 4b77ea77..26d4db37 100644 --- a/src/reconstruct/mod.rs +++ b/src/reconstruct/mod.rs @@ -5,34 +5,41 @@ use ndarray::{s, ArrayView1, ArrayView2, ArrayViewMut1}; use rayon::prelude::*; -/// Reconstruct a single haplotype from reference sequence and variants. +/// Single-haplotype inner kernel, generic over the variant source. /// -/// Single-haplotype inner kernel. Mirror of numba -/// `reconstruct_haplotype_from_sparse` (`_genotypes.py:277-465`). +/// Mirror of numba `reconstruct_haplotype_from_sparse` (`_genotypes.py:277-465`), with +/// the per-variant reads factored out behind the `provide` closure so the same loop +/// body can be driven by either the SVAR 1.0 global variant table +/// (`reconstruct_haplotype_from_sparse`) or the SVAR2 two-channel (var_key ⋈ dense) +/// source (`reconstruct_haplotypes_from_svar2`). **Do not change the reconstruction +/// math here** — only the data source differs between call sites. /// /// # Parameters -/// - `v_idxs` – indices into the full variant table for this haplotype (i32) -/// - `v_starts` – genomic start position of each variant (i32, indexed by variant) -/// - `ilens` – insertion-length (ilen = alt_len − ref_len + 1) per variant (i32) +/// - `n_variants` – number of variants this haplotype sees, i.e. `0..n_variants` are +/// valid indices into `provide` +/// - `provide` – `Fn(v) -> (v_pos, v_diff, allele, annot_id)` for `v in 0..n_variants`: +/// `v_pos` genomic start, `v_diff` ilen (`alt_len - ref_len` for +/// atomized variants), `allele` the full ALT allele bytes (borrowed or +/// owned via `Cow`), `annot_id` the value written into `annot_v_idxs` +/// when the variant is applied. The named lifetime `'a` ties the +/// returned `Cow`'s borrow to the driver-call scope (`alt_flat` for +/// SVAR1; `lut_bytes` for SVAR2) — using `Cow<'_, ...>` here instead +/// would force a higher-ranked bound that fails to unify across the +/// two call sites. /// - `shift` – total amount to shift by (i64) -/// - `alt_alleles` – packed ALT allele bytes for all variants (u8) -/// - `alt_offsets` – byte offsets into `alt_alleles`; length = total_variants + 1 (i64) /// - `ref_` – reference contig bytes (u8) /// - `ref_start` – start position into the reference; may be negative (i64) /// - `out` – output buffer to fill (u8, length = desired haplotype length) /// - `pad_char` – byte used for padding where reference is unavailable /// - `keep` – optional per-haplotype-variant mask; `None` means use all -/// - `annot_v_idxs` – optional annotation: variant index per output position (i32; -1 = ref/pad) +/// - `annot_v_idxs` – optional annotation: `annot_id` per output position (i32; -1 = ref/pad) /// - `annot_ref_pos` – optional annotation: reference position per output position (i32; /// -1 = leading pad, i32::MAX = trailing pad) #[allow(clippy::too_many_arguments)] -pub fn reconstruct_haplotype_from_sparse( - v_idxs: ArrayView1, - v_starts: ArrayView1, - ilens: ArrayView1, +fn reconstruct_haplotype_core<'a>( + n_variants: usize, + provide: impl Fn(usize) -> (i64, i64, std::borrow::Cow<'a, [u8]>, i32), shift: i64, - alt_alleles: ArrayView1, - alt_offsets: ArrayView1, ref_: ArrayView1, ref_start: i64, mut out: ArrayViewMut1, @@ -42,13 +49,11 @@ pub fn reconstruct_haplotype_from_sparse( mut annot_ref_pos: Option>, ) { let length = out.len() as i64; - let n_variants = v_idxs.len(); // Hoist contiguous-slice pointers once so the hot loops use direct byte ops // (fill/copy_from_slice) instead of ndarray's stride/do_slice dispatch path. let out_flat: &mut [u8] = out.as_slice_mut().unwrap(); let ref_flat: &[u8] = ref_.as_slice().unwrap(); - let alt_flat: &[u8] = alt_alleles.as_slice().unwrap(); let mut av_flat: Option<&mut [i32]> = annot_v_idxs.as_mut().and_then(|a| a.as_slice_mut()); let mut ap_flat: Option<&mut [i32]> = annot_ref_pos.as_mut().and_then(|a| a.as_slice_mut()); @@ -84,13 +89,8 @@ pub fn reconstruct_haplotype_from_sparse( } } - let variant = v_idxs[v] as usize; - let v_pos = v_starts[variant] as i64; - let v_diff = ilens[variant] as i64; - let ao_s = alt_offsets[variant] as usize; - let ao_e = alt_offsets[variant + 1] as usize; - // full allele slice; may be sub-sliced below for shift consumption - let allele_full = &alt_flat[ao_s..ao_e]; + let (v_pos, v_diff, allele_cow, annot_id) = provide(v); + let allele_full: &[u8] = allele_cow.as_ref(); let v_len_full = allele_full.len() as i64; // +1 assumes atomized variants, exactly 1 nt shared between REF and ALT let v_ref_end: i64 = v_pos - 0i64.min(v_diff) + 1; @@ -181,7 +181,7 @@ pub fn reconstruct_haplotype_from_sparse( let oe = (out_idx + writable_length) as usize; out_flat[os..oe].copy_from_slice(&allele[..writable_length as usize]); if let Some(av) = av_flat.as_deref_mut() { - av[os..oe].fill(variant as i32); + av[os..oe].fill(annot_id); } if let Some(ap) = ap_flat.as_deref_mut() { ap[os..oe].fill(v_pos as i32); @@ -255,6 +255,69 @@ pub fn reconstruct_haplotype_from_sparse( } } +/// Reconstruct a single haplotype from reference sequence and variants (SVAR 1.0 +/// global variant table). Thin wrapper over [`reconstruct_haplotype_core`]: builds a +/// closure that indexes the global table and delegates. **Behavior is unchanged** from +/// before the closure-source refactor — this only changes where the per-variant data +/// comes from. +/// +/// # Parameters +/// - `v_idxs` – indices into the full variant table for this haplotype (i32) +/// - `v_starts` – genomic start position of each variant (i32, indexed by variant) +/// - `ilens` – insertion-length (ilen = alt_len − ref_len + 1) per variant (i32) +/// - `shift` – total amount to shift by (i64) +/// - `alt_alleles` – packed ALT allele bytes for all variants (u8) +/// - `alt_offsets` – byte offsets into `alt_alleles`; length = total_variants + 1 (i64) +/// - `ref_` – reference contig bytes (u8) +/// - `ref_start` – start position into the reference; may be negative (i64) +/// - `out` – output buffer to fill (u8, length = desired haplotype length) +/// - `pad_char` – byte used for padding where reference is unavailable +/// - `keep` – optional per-haplotype-variant mask; `None` means use all +/// - `annot_v_idxs` – optional annotation: variant index per output position (i32; -1 = ref/pad) +/// - `annot_ref_pos` – optional annotation: reference position per output position (i32; +/// -1 = leading pad, i32::MAX = trailing pad) +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotype_from_sparse( + v_idxs: ArrayView1, + v_starts: ArrayView1, + ilens: ArrayView1, + shift: i64, + alt_alleles: ArrayView1, + alt_offsets: ArrayView1, + ref_: ArrayView1, + ref_start: i64, + out: ArrayViewMut1, + pad_char: u8, + keep: Option>, + annot_v_idxs: Option>, + annot_ref_pos: Option>, +) { + let alt_flat: &[u8] = alt_alleles.as_slice().unwrap(); + let provide = |v: usize| { + let variant = v_idxs[v] as usize; + let ao_s = alt_offsets[variant] as usize; + let ao_e = alt_offsets[variant + 1] as usize; + ( + v_starts[variant] as i64, + ilens[variant] as i64, + std::borrow::Cow::Borrowed(&alt_flat[ao_s..ao_e]), + variant as i32, + ) + }; + reconstruct_haplotype_core( + v_idxs.len(), + provide, + shift, + ref_, + ref_start, + out, + pad_char, + keep, + annot_v_idxs, + annot_ref_pos, + ); +} + /// Batch driver: reconstruct haplotypes for all (query, hap) pairs. /// /// Mirrors `reconstruct_haplotypes_from_sparse` (plural) in @@ -484,6 +547,10 @@ pub fn reconstruct_haplotypes_from_sparse( for k in 0..n_work { let out_s = out_offsets[k] as usize; let out_e = out_offsets[k + 1] as usize; + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); // SAFETY: `out_offsets` is required by the calling contract to be monotonically // non-decreasing, so consecutive (out_s, out_e) pairs are strictly non-overlapping @@ -498,9 +565,7 @@ pub fn reconstruct_haplotypes_from_sparse( // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent // aliasing. let av_view: Option> = av_raw.map(|p| { - let chunk = unsafe { - std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) - }; + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; ArrayViewMut1::from(chunk) }); @@ -508,9 +573,7 @@ pub fn reconstruct_haplotypes_from_sparse( // each [out_s..out_e] is a disjoint sub-range; serial loop prevents concurrent // aliasing. let ap_view: Option> = ap_raw.map(|p| { - let chunk = unsafe { - std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) - }; + let chunk = unsafe { std::slice::from_raw_parts_mut(p.add(out_s), out_e - out_s) }; ArrayViewMut1::from(chunk) }); @@ -519,6 +582,214 @@ pub fn reconstruct_haplotypes_from_sparse( } } +/// Batch driver: reconstruct haplotypes for all (query, hap) pairs from the SVAR2 +/// two-channel (var_key ⋈ dense) source. Mirrors +/// [`reconstruct_haplotypes_from_sparse`]'s parallel/serial disjoint-slice carving +/// exactly — only the per-work-item variant source and decode differ. Additive: SVAR +/// 1.0's `reconstruct_haplotype[s]_from_sparse` are untouched by this function. +/// +/// # Parameters +/// - `out` – flat output buffer, length = out_offsets[-1] (u8); written in place +/// - `out_offsets` – shape (batch*ploidy + 1,) offsets into `out` +/// - `regions` – shape (batch, 3) as (contig_idx, start, end) i32 +/// - `shifts` – shape (batch, ploidy) i32 +/// - `vk_pos` / `vk_key` – this hap's `var_key` channel: position + uniform key (i32) +/// - `vk_off` – shape (n_work + 1) CSR offsets into `vk_pos`/`vk_key`, indexed by flat +/// work index `k = query * ploidy + hap` +/// - `dense_pos` / `dense_key` – the shared `dense` channel: position + uniform key (i32) +/// - `dense_range` – shape (batch, 2) as `[ds, de)`, the window into `dense_pos`/`dense_key` +/// for each query row +/// - `dense_present` – packed presence bits over the dense window, LSB-first per byte (u8) +/// - `dense_present_off` – shape (n_work + 1) BIT offsets into `dense_present`, indexed by +/// flat work index `k` +/// - `lut_bytes` / `lut_off` – long-allele-bank bytes + CSR offsets, for `Lookup` keys +/// - `ref_` – packed reference bytes u8 +/// - `ref_offsets` – per-contig offsets into ref_ i64 +/// - `pad_char` – padding byte u8 +/// - `parallel` – if true, use rayon to process work items concurrently +/// +/// SVAR2 read-bound haplotypes are never annotated (annotated `.svar2` output is +/// `NotImplementedError`-guarded), so this kernel produces only un-annotated +/// sequence; the shared `reconstruct_haplotype_core` still carries the optional +/// annotation outputs for the SVAR1 path. +#[allow(clippy::too_many_arguments)] +pub fn reconstruct_haplotypes_from_svar2( + mut out: ArrayViewMut1, + out_offsets: ArrayView1, + regions: ArrayView2, + shifts: ArrayView2, + vk_pos: ArrayView1, + vk_key: ArrayView1, + vk_off: ArrayView1, + dense_pos: ArrayView1, + dense_key: ArrayView1, + dense_range: ArrayView2, + dense_present: ArrayView1, + dense_present_off: ArrayView1, + lut_bytes: ArrayView1, + lut_off: ArrayView1, + ref_: ArrayView1, + ref_offsets: ArrayView1, + pad_char: u8, + parallel: bool, +) { + let batch_size = regions.nrows(); + let ploidy = shifts.ncols(); + let n_work = batch_size * ploidy; + + // Hoist contiguous-slice pointers once, exactly as SVAR1's do_work captures + // read-only views — these are Send+Sync so the rayon parallel path is unchanged. + let vk_pos_s: &[i32] = vk_pos.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off.as_slice().unwrap(); + + // Per-k inner work: merge this hap's var_key ⋈ dense entries, then reconstruct via + // the shared core with a decode closure. All read-only ArrayViews/slices are + // Send+Sync so the closure can borrow them freely. + let do_work = |k: usize, out_view: ArrayViewMut1| { + let query = k / ploidy; + let hap = k % ploidy; + + // region/ref + let c_idx = regions[[query, 0]] as usize; + let c_s = ref_offsets[c_idx] as usize; + let c_e = ref_offsets[c_idx + 1] as usize; + let contig_ref = ref_.slice(s![c_s..c_e]); + let ref_start = regions[[query, 1]] as i64; + let shift = shifts[[query, hap]] as i64; + + // var_key window for this hap + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + + // dense window for this query + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + + // presence bits for this hap start at bit `dense_present_off[k]` + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| crate::svar2::present_bit(dense_present_s, base_bit, j); + + let merged = crate::svar2::merge_hap( + vk_pos_s, + vk_key_s, + vk_lo, + vk_hi, + dense_pos_s, + dense_key_s, + ds, + de, + present_bit, + ); + + let contig_ref_s: &[u8] = contig_ref.as_slice().unwrap(); + let provide = |v: usize| { + let (pos, key) = merged[v]; + let (v_diff, allele) = crate::svar2::decode_alt(key, lut_bytes_s, lut_off_s); + // A pure DEL decodes to an empty ALT: genoray stores no anchor base in the key + // (recovered from the reference downstream). But the reconstruction kernel writes + // `allele_full` as the replacement and advances ref by |v_diff|+1, so it needs the + // anchor base = ref[pos] present; an empty allele would delete the anchor too + // (off-by-one: -(|v_diff|+1) instead of -|v_diff|). SNP/INS/Lookup alleles are + // already non-empty and pass through unchanged. + let allele = if allele.is_empty() { + // The anchor base is `ref[pos]`. Valid gathered variants are within-contig + // records (`pos < contig_ref_s.len()`); a `pos` past the contig end can only + // come from a corrupt store. Assert it in debug/test builds rather than + // silently reading OOB (release keeps the raw slice for speed). + debug_assert!( + (pos as usize) < contig_ref_s.len(), + "pure-DEL anchor position {pos} is beyond contig ref length {} (corrupt store?)", + contig_ref_s.len() + ); + std::borrow::Cow::Borrowed(&contig_ref_s[pos as usize..pos as usize + 1]) + } else { + allele + }; + (pos as i64, v_diff, allele, v as i32) + }; + + reconstruct_haplotype_core( + merged.len(), + provide, + shift, + contig_ref, + ref_start, + out_view, + pad_char, + None, // keep: SVAR2 has no per-haplotype keep mask + None, // annot_v_idxs: SVAR2 read-bound haps are never annotated + None, // annot_ref_pos: SVAR2 read-bound haps are never annotated + ); + }; + + if parallel { + // Build disjoint per-k mutable slices for all active buffers using the + // proven split_at_mut chain idiom (mirrors get_reference in reference/mod.rs). + // &mut [_] slices are Send, unlike raw *mut pointers — safe for rayon closures. + let bounds: Vec<(usize, usize)> = (0..n_work) + .map(|k| (out_offsets[k] as usize, out_offsets[k + 1] as usize)) + .collect(); + + let out_slice = out.as_slice_mut().unwrap(); + let mut out_chunks: Vec<&mut [u8]> = Vec::with_capacity(n_work); + { + let mut rest = &mut out_slice[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + // Contract: `out_offsets` is monotonically non-decreasing, so each + // work item's range starts at or after the previous one's end. This + // guarantees `s - cursor` does not underflow and the carved slices + // are disjoint. The same `bounds` drives the annotation carves below. + debug_assert!( + s >= cursor && e >= s, + "out_offsets must be monotonically non-decreasing (got s={s}, e={e}, cursor={cursor})" + ); + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + out_chunks.push(mid); + rest = tail2; + cursor = e; + } + } + + // SVAR2 read-bound haps are never annotated, so dispatch the un-annotated + // work items straight across rayon. + out_chunks + .into_par_iter() + .enumerate() + .for_each(|(k, out_chunk)| { + do_work(k, ArrayViewMut1::from(out_chunk)); + }); + } else { + // Serial path: use raw pointers for disjoint sub-range access, exactly as before. + // The serial loop prevents concurrent aliasing. + let out_raw: *mut u8 = out.as_mut_ptr(); + + for k in 0..n_work { + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; + debug_assert!( + out_e >= out_s, + "out_offsets must be monotonically non-decreasing (got out_s={out_s}, out_e={out_e})" + ); + + // SAFETY: `out_offsets` is required by the calling contract to be monotonically + // non-decreasing, so consecutive (out_s, out_e) pairs are strictly non-overlapping + // address ranges within the same allocation. Because the loop is serial there are + // no concurrent borrows, so constructing a `&mut [u8]` from each disjoint sub-range + // is free of aliasing UB. + let out_chunk = + unsafe { std::slice::from_raw_parts_mut(out_raw.add(out_s), out_e - out_s) }; + do_work(k, ArrayViewMut1::from(out_chunk)); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -595,9 +866,9 @@ mod tests { &[], // alt_alleles &[0i64], // alt_offsets (1 sentinel for 0 variants) &[10, 20, 30, 40, 50], - 1, // ref_start - 3, // out_len - 0, // pad_char + 1, // ref_start + 3, // out_len + 0, // pad_char None, false, ); @@ -627,7 +898,11 @@ mod tests { ); assert_eq!(out, vec![9, 9, 1, 2, 3]); assert_eq!(&av[..2], &[-1i32, -1]); - assert_eq!(&ap[..2], &[-1i32, -1], "leading pad annot_ref_pos must be -1"); + assert_eq!( + &ap[..2], + &[-1i32, -1], + "leading pad annot_ref_pos must be -1" + ); assert_eq!(&ap[2..], &[0i32, 1, 2]); } @@ -644,14 +919,14 @@ mod tests { // variant at pos=2 (G→T), ilen=0 → v_ref_end = 2 - 0 + 1 = 3 // out: A C [T] T A let (out, av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[2], // v_starts: variant 0 is at pos 2 - &[0], // ilens: SNP, no length change - 0, // shift - &[84u8], // alt_alleles: T - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[2], // v_starts: variant 0 is at pos 2 + &[0], // ilens: SNP, no length change + 0, // shift + &[84u8], // alt_alleles: T + &[0i64, 1], // alt_offsets &[65, 67, 71, 84, 65], // A C G T A - 0, // ref_start + 0, // ref_start 5, 0, None, @@ -676,8 +951,8 @@ mod tests { fn two_bp_insertion() { let (out, _av, _ap) = run( &[0], - &[2], // variant 0 at pos 2 - &[2], // ilen=+2 + &[2], // variant 0 at pos 2 + &[2], // ilen=+2 0, &[10u8, 11, 12], &[0i64, 3], @@ -705,10 +980,10 @@ mod tests { fn deletion() { let (out, _av, _ap) = run( &[0], - &[2], // variant 0 at pos 2 - &[-2], // ilen=-2 + &[2], // variant 0 at pos 2 + &[-2], // ilen=-2 0, - &[30u8], // anchor allele byte + &[30u8], // anchor allele byte &[0i64, 1], &[1, 2, 3, 4, 5, 6, 7], 0, @@ -735,13 +1010,13 @@ mod tests { fn del_spanning_ref_start() { let (out, _av, ap) = run( &[0], - &[1], // v_pos=1 - &[-3], // ilen=-3 + &[1], // v_pos=1 + &[-3], // ilen=-3 0, &[99u8], &[0i64, 1], &[1, 2, 3, 4, 5, 6, 7], - 3, // ref_start=3 + 3, // ref_start=3 5, 0, None, @@ -767,10 +1042,10 @@ mod tests { fn overshoot_ref_past_contig() { let (out, _av, _ap) = run( &[0], - &[2], // v_pos=2 - &[-5], // ilen=-5 (deletion past contig end) - 0, // shift - &[50u8], // anchor allele + &[2], // v_pos=2 + &[-5], // ilen=-5 (deletion past contig end) + 0, // shift + &[50u8], // anchor allele &[0i64, 1], &[1, 2, 3, 4], // ref, len 4 0, // ref_start @@ -793,9 +1068,9 @@ mod tests { #[test] fn overlapping_alts_first_applied() { let (out, _av, _ap) = run( - &[0, 1], // v_idxs: variants 0 then 1 - &[2, 2], // both at pos=2 - &[0, 0], // both SNPs + &[0, 1], // v_idxs: variants 0 then 1 + &[2, 2], // both at pos=2 + &[0, 0], // both SNPs 0, &[20u8, 30], // alleles: 20 and 30 &[0i64, 1, 2], @@ -844,9 +1119,9 @@ mod tests { // out_len=5: [3, 99, 88, 5, 6] let (out, _av, _ap) = run( &[0], - &[3], // v_pos=3 - &[1], // ilen=+1 - 2, // shift=2 + &[3], // v_pos=3 + &[1], // ilen=+1 + 2, // shift=2 &[99u8, 88], &[0i64, 2], &[1, 2, 3, 4, 5, 6], @@ -878,8 +1153,8 @@ mod tests { let (out, _av, _ap) = run( &[0], &[3], - &[1], // ilen=+1, allele 2 bytes - 4, // shift=4 + &[1], // ilen=+1, allele 2 bytes + 4, // shift=4 &[99u8, 88], &[0i64, 2], &[1, 2, 3, 4, 5, 6, 7, 8], @@ -955,16 +1230,16 @@ mod tests { #[test] fn allele_start_idx_eq_v_len_continue() { let (out, _av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[3], // v_starts: variant 0 at pos 3 - &[0], // ilens: SNP, ilen=0 - 4, // shift=4 - &[88u8], // alt_allele - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[3], // v_starts: variant 0 at pos 3 + &[0], // ilens: SNP, ilen=0 + 4, // shift=4 + &[88u8], // alt_allele + &[0i64, 1], // alt_offsets &[1, 2, 3, 4, 5, 6, 7, 8], - 0, // ref_start - 4, // out_len - 0, // pad_char + 0, // ref_start + 4, // out_len + 0, // pad_char None, false, ); @@ -996,16 +1271,16 @@ mod tests { fn skip_variant_not_enough_distance() { let ref_: Vec = (1u8..=15).collect(); let (out, _av, _ap) = run( - &[0], // v_idxs: only variant 0 - &[3], // v_starts: variant 0 at pos 3 - &[0], // ilens: SNP, ilen=0 - 10, // shift=10 - &[77u8], // alt_allele (never used) - &[0i64, 1], // alt_offsets + &[0], // v_idxs: only variant 0 + &[3], // v_starts: variant 0 at pos 3 + &[0], // ilens: SNP, ilen=0 + 10, // shift=10 + &[77u8], // alt_allele (never used) + &[0i64, 1], // alt_offsets &ref_, - 0, // ref_start - 3, // out_len - 0, // pad_char + 0, // ref_start + 3, // out_len + 0, // pad_char None, false, ); @@ -1037,18 +1312,18 @@ mod tests { #[test] fn keep_mask_excludes_variant() { let (out, av, _ap) = run( - &[0, 1], // v_idxs: variants 0 and 1 - &[1, 3], // v_starts: variant 0 at pos 1, variant 1 at pos 3 - &[0, 0], // ilens: both SNPs - 0, // shift=0 - &[55u8, 99], // alleles: 55 for v0, 99 for v1 - &[0i64, 1, 2], // alt_offsets + &[0, 1], // v_idxs: variants 0 and 1 + &[1, 3], // v_starts: variant 0 at pos 1, variant 1 at pos 3 + &[0, 0], // ilens: both SNPs + 0, // shift=0 + &[55u8, 99], // alleles: 55 for v0, 99 for v1 + &[0i64, 1, 2], // alt_offsets &[1, 2, 3, 4, 5], - 0, // ref_start - 5, // out_len - 0, // pad_char + 0, // ref_start + 5, // out_len + 0, // pad_char Some(&[false, true]), // keep mask: skip v0, apply v1 - true, // annotate + true, // annotate ); // variant 0 (pos=1, allele=55) excluded by keep mask: ref[1] NOT replaced // variant 1 (pos=3, allele=99) applied: ref[3] replaced by 99 @@ -1064,28 +1339,29 @@ mod tests { #[test] fn annotated_vs_non_annotated_identical_out() { let params = ( - &[0i32][..], // v_idxs - &[2i32][..], // v_starts - &[0i32][..], // ilens - 0i64, // shift - &[77u8][..], // alt_alleles - &[0i64, 1][..],// alt_offsets - &[1u8,2,3,4,5][..], // ref_ - 0i64, // ref_start - 5usize, // out_len - 0u8, // pad_char + &[0i32][..], // v_idxs + &[2i32][..], // v_starts + &[0i32][..], // ilens + 0i64, // shift + &[77u8][..], // alt_alleles + &[0i64, 1][..], // alt_offsets + &[1u8, 2, 3, 4, 5][..], // ref_ + 0i64, // ref_start + 5usize, // out_len + 0u8, // pad_char ); let (out_annot, _, _) = run( - params.0, params.1, params.2, params.3, - params.4, params.5, params.6, params.7, + params.0, params.1, params.2, params.3, params.4, params.5, params.6, params.7, params.8, params.9, None, true, ); let (out_plain, _, _) = run( - params.0, params.1, params.2, params.3, - params.4, params.5, params.6, params.7, + params.0, params.1, params.2, params.3, params.4, params.5, params.6, params.7, params.8, params.9, None, false, ); - assert_eq!(out_annot, out_plain, "annotated and non-annotated must produce identical out bytes"); + assert_eq!( + out_annot, out_plain, + "annotated and non-annotated must produce identical out bytes" + ); } #[test] @@ -1202,7 +1478,156 @@ mod tests { false, ); - assert_eq!(&out.as_slice().unwrap()[0..4], b"ATGT", "region 0 with SNP applied"); - assert_eq!(&out.as_slice().unwrap()[4..8], b"ACGT", "region 1 reference-only"); + assert_eq!( + &out.as_slice().unwrap()[0..4], + b"ATGT", + "region 0 with SNP applied" + ); + assert_eq!( + &out.as_slice().unwrap()[4..8], + b"ACGT", + "region 1 reference-only" + ); + } + + // ------------------------------------------------------------------------- + // SVAR2 driver: one region (R=1), one sample-slot (S=1), one ploid (P=1) → + // n_work = 1, k = query = hap = 0. Hand-built two-channel input: a SNP in the + // var_key channel and a pure DEL in the dense channel (with its presence bit set), + // exercising both the merge and the decode paths in a single call. + // + // ref = "ACGTACGT" (8 bp, positions 0..8: A C G T A C G T) + // var_key: SNP at pos=1 (ref 'C' -> 'T'), Inline key, decode_alt -> (v_diff=0, "T") + // dense: pure DEL at pos=4, ilen=-1, decode_alt -> (v_diff=-1, "" empty). The driver's + // `provide` closure substitutes the anchor base ref[pos]='A' for the empty ALT + // (genoray stores no anchor in the key; the kernel needs it, else the anchor is + // deleted too). So the DEL contributes allele="A" and deletes the following 'C'. + // merge_hap orders by position: v=0 is the SNP (pos=1), v=1 is the DEL (pos=4). + // + // Hand derivation of reconstruct_haplotype_core(n_variants=2, ..., shift=0): + // start: ref_idx=0, out_idx=0 (ref_start=0, not negative: no leading pad) + // v=0 (SNP, pos=1, v_diff=0, allele="T", v_len_full=1): + // v_ref_end = 1 - min(0,0) + 1 = 2 + // not a DEL-spanning-start case; pos(1) >= ref_idx(0); shift=0 so no shift branch + // ref_len = 1-0 = 1 -> out[0..1] = ref[0..1] = "A"; out_idx=1 + // writable_length = min(1, 8-1)=1 -> out[1..2] = "T"; out_idx=2 + // ref_idx = v_ref_end = 2 + // v=1 (DEL, pos=4, v_diff=-1, allele="A" (anchor = ref[4])): + // v_ref_end = 4 - min(0,-1) + 1 = 4+1+1 = 6 + // pos(4) >= ref_idx(2); shift=0 so no shift branch + // ref_len = 4-2 = 2 -> out[2..4] = ref[2..4] = "GT"; out_idx=4 + // writable_length = min(1, 8-4) = 1 -> out[4..5] = "A"; out_idx=5 + // ref_idx = v_ref_end = 6 (ref 'C' at pos 5 is skipped = deleted) + // tail: unfilled_length = 8-5 = 3; writable_ref = min(3, 8-6) = 2 + // -> out[5..7] = ref[6..8] = "GT"; out_end_idx=7 + // right-pad out[7..8] with pad_char 'N' + // out = "A" + "T" + "GT" + "A" + "GT" + "N" = "ATGTAGTN" + // (haplotype = ref with SNP C->T at pos1 and the 'C' at pos5 deleted) + #[test] + fn svar2_reconstruct_snp_and_del() { + let reference = b"ACGTACGT"; + let ref_ = arr1(reference.as_ref()); + let ref_offsets = arr1(&[0i64, 8]); + + // One region on contig 0, one sample-slot, one ploid -> n_work = 1. + let regions = ndarray::arr2(&[[0i32, 0, 8]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // var_key channel: one SNP at pos=1 (Inline key, 1-base ALT "T"). + let snp_key = svar2_codec::encode_alt_inline(b"T", 0) as i32; + let vk_pos = arr1(&[1i32]); + let vk_key = arr1(&[snp_key]); + let vk_off = arr1(&[0i64, 1]); + + // dense channel: one pure DEL (1bp) at pos=4, present for this hap. + let del_key = svar2_codec::encode_pure_del(-1) as i32; + let dense_pos = arr1(&[4i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + // Presence bits, LSB-first: bit 0 (this hap's only dense entry) is set. + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + + // No long-allele-bank lookups exercised in this test. + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + + let out_offsets = arr1(&[0i64, 8]); + let pad_char = b'N'; + let mut out = Array1::::from_elem(8, pad_char); + + super::reconstruct_haplotypes_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + ref_.view(), + ref_offsets.view(), + pad_char, + false, // serial + ); + + assert_eq!(out.as_slice().unwrap(), b"ATGTAGTN"); + } + + // Regression guard for the pure-DEL anchor fix: a lone 1bp DEL must delete exactly + // one base (the one AFTER the anchor), not two. ref="ACGT", DEL at pos=1 (ilen=-1): + // keep anchor 'C' at pos1, delete 'G' at pos2 -> "ACT" + right-pad 'N' = "ACTN". + #[test] + fn svar2_pure_del_keeps_anchor() { + let reference = b"ACGT"; + let ref_ = arr1(reference.as_ref()); + let ref_offsets = arr1(&[0i64, 4]); + let regions = ndarray::arr2(&[[0i32, 0, 4]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // No var_key entries; a single dense pure DEL at pos=1. + let vk_pos = arr1::(&[]); + let vk_key = arr1::(&[]); + let vk_off = arr1(&[0i64, 0]); + let del_key = svar2_codec::encode_pure_del(-1) as i32; + let dense_pos = arr1(&[1i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + let out_offsets = arr1(&[0i64, 4]); + let pad_char = b'N'; + let mut out = Array1::::from_elem(4, pad_char); + + super::reconstruct_haplotypes_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + ref_.view(), + ref_offsets.view(), + pad_char, + false, + ); + + assert_eq!(out.as_slice().unwrap(), b"ACTN"); } } diff --git a/src/svar2/mod.rs b/src/svar2/mod.rs new file mode 100644 index 00000000..8fa46d01 --- /dev/null +++ b/src/svar2/mod.rs @@ -0,0 +1,1071 @@ +//! SVAR2 two-source variant provider: merge a hap's var_key ⋈ dense channels and +//! decode keys via svar2-codec, feeding the reconstruction kernel with no +//! intermediate variant table. Additive to the SVAR 1.0 global-table path. + +use std::borrow::Cow; +use std::ops::Range; + +use genoray_core::query::{dense_abs_row, unpack_vk_src, BatchResultSplit, FieldView}; +use ndarray::{Array2, ArrayView2}; +use svar2_codec::{decode_key, DecodedKey}; + +pub mod store; + +/// Decode one uniform key into `(v_diff, allele)`, resolving long-INS via the LUT +/// arrays. Mirrors genoray's `decode_keyref`. +pub fn decode_alt<'a>(key: u32, lut_bytes: &'a [u8], lut_off: &[i64]) -> (i64, Cow<'a, [u8]>) { + match decode_key(key) { + DecodedKey::Inline { alt } => (alt.len() as i64 - 1, Cow::Owned(alt)), + DecodedKey::PureDel { ilen } => (ilen as i64, Cow::Borrowed(&[][..])), + DecodedKey::Lookup { row } => { + let s = lut_off[row as usize] as usize; + let e = lut_off[row as usize + 1] as usize; + let alt = &lut_bytes[s..e]; + (alt.len() as i64 - 1, Cow::Borrowed(alt)) + } + } +} + +/// LSB-first presence-bit lookup into a packed presence bitmap: returns whether bit +/// `base_bit + j` of `dense_present` is set, indexed from the least-significant bit +/// within each byte. Shared by the haplotype (`reconstruct`) and track (`tracks`) +/// per-hap `merge_hap` calls, which both read a hap's presence window this way. +#[inline] +pub fn present_bit(dense_present: &[u8], base_bit: usize, j: usize) -> bool { + let bit = base_bit + j; + (dense_present[bit / 8] >> (bit % 8)) & 1 == 1 +} + +/// Merge one hap's `var_key` slice with its carried `dense` set-bits into a single +/// position-sorted `(pos, key)` list (stable: var_key before dense on ties, matching +/// genoray's merge). `dense` is region `r`'s `[ds, de)` window; `present` are this hap's +/// LSB-first presence bits over that window. +#[allow(clippy::too_many_arguments)] +pub fn merge_hap( + vk_pos: &[i32], + vk_key: &[i32], + vk_lo: usize, + vk_hi: usize, + dense_pos: &[i32], + dense_key: &[i32], + ds: usize, + de: usize, + present_bit: impl Fn(usize) -> bool, // present_bit(k) for k in 0..(de-ds) +) -> Vec<(u32, u32)> { + let mut a: Vec<(u32, u32)> = (vk_lo..vk_hi) + .map(|i| (vk_pos[i] as u32, vk_key[i] as u32)) + .collect(); + for (k, j) in (ds..de).enumerate() { + if present_bit(k) { + a.push((dense_pos[j] as u32, dense_key[j] as u32)); + } + } + a.sort_by_key(|&(p, _)| p); // stable; var_key pushed first keeps it ahead on ties + a +} + +/// Per-hap applied-ilen diff for the two-source path, mirroring +/// `genotypes::get_diffs_sparse`'s q_start/q_end-clipped branch. Used to size the fused +/// SVAR2 reconstruct/track outputs. Serial (n is tiny; the fused callers already parallelize +/// the heavy reconstruct pass). +#[allow(clippy::too_many_arguments)] +pub fn hap_diffs_svar2( + regions: ArrayView2, // (n_q, 3) + ploidy: usize, + vk_pos: &[i32], + vk_key: &[i32], + vk_off: &[i64], // (n_work+1) + dense_pos: &[i32], + dense_key: &[i32], + dense_range: ArrayView2, // (n_q, 2) + dense_present: &[u8], + dense_present_off: &[i64], // (n_work+1) BIT offsets + lut_bytes: &[u8], + lut_off: &[i64], +) -> Array2 { + let n_q = regions.nrows(); + let mut diffs = Array2::::zeros((n_q, ploidy)); + for k in 0..(n_q * ploidy) { + let query = k / ploidy; + let hap = k % ploidy; + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| -> bool { + let bit = base_bit + j; + (dense_present[bit / 8] >> (bit % 8)) & 1 == 1 + }; + let merged = merge_hap( + vk_pos, + vk_key, + vk_lo, + vk_hi, + dense_pos, + dense_key, + ds, + de, + present_bit, + ); + if merged.is_empty() { + continue; + } + let q_start = regions[[query, 1]] as i64; + let q_end = regions[[query, 2]] as i64; + let mut ref_idx = q_start; + let mut acc: i64 = 0; + for &(pos, key) in &merged { + let v_start = pos as i64; + let (mut v_ilen, _allele) = decode_alt(key, lut_bytes, lut_off); + let v_end = v_start - v_ilen.min(0) + 1; + if v_end <= q_start { + continue; + } + if v_start >= q_end { + break; + } + if v_start >= q_start && v_start < ref_idx { + continue; + } + ref_idx = ref_idx.max(v_end); + if v_ilen < 0 { + v_ilen += (q_start - v_start - 1).max(0); + } + v_ilen += (v_end - q_end).max(0); + acc += v_ilen; + } + diffs[[query, hap]] = acc as i32; + } + diffs +} + +/// The flat, single-dense-channel layout consumed by [`hap_diffs_svar2`] / +/// `reconstruct::reconstruct_haplotypes_from_svar2` — see [`split_to_flat`]. +pub struct FlatChannels { + pub vk_pos: Vec, + pub vk_key: Vec, + pub vk_off: Vec, + pub dense_pos: Vec, + pub dense_key: Vec, + /// Flat, length `n_q*2`; view as `(n_q, 2)` at the call site. + pub dense_range: Vec, + pub dense_present: Vec, + pub dense_present_off: Vec, +} + +/// Marshal a read-bound [`BatchResultSplit`] (genoray's per-class-split dense +/// channels) into the flat single-dense-channel layout that the already-validated +/// [`hap_diffs_svar2`] / `reconstruct::reconstruct_haplotypes_from_svar2` kernels +/// consume — so read-bound gather can reuse those kernels unchanged instead of +/// duplicating the merge/decode logic for a three-source layout. +/// +/// Per query `q` the combined dense window is `dense_snp[q] ++ dense_indel[q]` +/// (SNP entries first — matches genoray's `merge_keys(vec![vk, dense_snp, +/// dense_indel])` tie order, where the dense side is `dense_snp` before +/// `dense_indel`). Per hap the combined presence bits are `snp_bits ++ +/// indel_bits` over that combined window, LSB-first packed into one bitstream, +/// so `dense_present_off`/`dense_range` line up with `dense_pos`/`dense_key` +/// exactly like the union path's single dense channel. +pub fn split_to_flat(br: &BatchResultSplit) -> FlatChannels { + let ploidy = br.ploidy; + let n_q = br.n_regions; // n_samples == 1 for read-bound gather + let h_count = n_q * ploidy; + + let vk_pos: Vec = br.vk.iter().map(|k| k.position as i32).collect(); + let vk_key: Vec = br.vk.iter().map(|k| k.key as i32).collect(); + let vk_off: Vec = br.vk_off.iter().map(|&o| o as i64).collect(); + + let dense_total: usize = (0..n_q) + .map(|q| { + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); + (se - ss) + (ie - is_) + }) + .sum(); + let mut dense_pos: Vec = Vec::with_capacity(dense_total); + let mut dense_key: Vec = Vec::with_capacity(dense_total); + let mut dense_range: Vec = Vec::with_capacity(n_q * 2); + for q in 0..n_q { + let base = dense_pos.len() as i32; + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + for j in ss..se { + dense_pos.push(br.dense_snp[j].position as i32); + dense_key.push(br.dense_snp[j].key as i32); + } + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); + for j in is_..ie { + dense_pos.push(br.dense_indel[j].position as i32); + dense_key.push(br.dense_indel[j].key as i32); + } + dense_range.push(base); + dense_range.push(dense_pos.len() as i32); + } + + // Per query `q`, every one of its `ploidy` haps has the exact same dense + // window width `(se - ss) + (ie - is_)` — so summing per-h widths over + // `0..h_count` (which recomputes `q = h / ploidy` via a runtime-value + // hardware division on *every* h) is equivalent to summing the per-q + // width once and multiplying by `ploidy`. Same total, no per-h division. + let total_bits: usize = (0..n_q) + .map(|q| { + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); + ((se - ss) + (ie - is_)) * ploidy + }) + .sum(); + let mut dense_present: Vec = vec![0u8; total_bits.div_ceil(8)]; + let mut dense_present_off: Vec = Vec::with_capacity(h_count + 1); + let mut bit_acc: usize = 0; + dense_present_off.push(0); + // Same hoist as `decode_variants_from_split`: `q = h / ploidy` is + // loop-invariant across each hap's ploidy-many iterations, so iterate `q` + // in the outer loop and track `h` with a plain running counter instead of + // recomputing `h / ploidy` (a runtime division) on every hap. Visits the + // exact same `(h, q)` pairs in the exact same order (h = 0, 1, ..., + // h_count-1, q = h/ploidy for each) — byte-identical — and also hoists the + // per-query `(ss, se)`/`(is_, ie)` window lookup out of the + // ploidy-many-times-redundant per-hap load. + let mut h = 0usize; + for q in 0..n_q { + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); + for _hap in 0..ploidy { + let snp_base = br.dense_snp_present_off[h]; + for k in 0..(se - ss) { + if genoray_core::bits_get_bit(&br.dense_snp_present, snp_base + k) { + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + } + bit_acc += 1; + } + let indel_base = br.dense_indel_present_off[h]; + for k in 0..(ie - is_) { + if genoray_core::bits_get_bit(&br.dense_indel_present, indel_base + k) { + dense_present[bit_acc / 8] |= 1 << (bit_acc % 8); + } + bit_acc += 1; + } + dense_present_off.push(bit_acc as i64); + h += 1; + } + } + // `dense_present` was pre-sized to `total_bits.div_ceil(8)` above, and the + // fill loop increments `bit_acc` exactly `total_bits` times, so this resize + // is a defensive no-op (kept to document the ceil-byte invariant the reused + // kernels rely on: they read `dense_present[bit/8]` for every window entry). + dense_present.resize(bit_acc.div_ceil(8), 0); + + FlatChannels { + vk_pos, + vk_key, + vk_off, + dense_pos, + dense_key, + dense_range, + dense_present, + dense_present_off, + } +} + +/// One INFO/FORMAT field's on-disk sidecars, opened for gather. +/// +/// Contracts the caller MUST uphold when constructing this: +/// * `views` must be opened in [`genoray_core::layout::FieldSub::all`] order +/// (VkSnp, VkIndel, DenseSnp, DenseIndel) — `decode_variants_from_split` +/// indexes into it positionally, not by name. `FieldView` owns its mmap, +/// so this needs no lifetime. +/// * `cohort_n_samples` MUST equal the `n_samples` passed to +/// `FieldView::open` for every view in `views`. If they diverge, the dense +/// FORMAT stride computed here (`row * cohort_n_samples + orig_sample`) +/// silently disagrees with how the on-disk store was laid out — no panic, +/// just wrong values. +/// * `FieldView::bytes_at` panics on an empty sub-stream. This is only safe +/// because an empty sub-stream implies no emitted record ever resolves to +/// it (e.g. a store with zero indel calls never routes a variant to +/// `VkIndel`/`DenseIndel`). A future constructor that opens a placeholder +/// view for a sub-stream that CAN be referenced must not rely on this. +pub struct FieldGather { + pub views: [FieldView; 4], + pub is_format: bool, + /// `dtype.width_bytes()`; consumed by the FFI caller, not here. + pub width: usize, + pub cohort_n_samples: usize, +} + +/// Per-hap decoded variant SoA, matching genoray's `decode_hap` output layout — +/// see [`decode_variants_from_split`]. +pub struct VariantsSoa { + /// Per-variant (flat, hap-major) 0-based start position. + pub pos: Vec, + /// Per-variant (flat) `ilen` (ALT len - 1 for inline/lookup keys, negative + /// deletion length for pure-deletion keys). + pub ilen: Vec, + /// Concatenated ALT bytes for all variants (pure-deletion ALT is empty). + pub alt_bytes: Vec, + /// Per-variant byte offsets into `alt_bytes`, len = `total_variants + 1`. + pub str_off: Vec, + /// Per-hap offsets into `pos`/`ilen`/`str_off`'s variant axis, len = `H + 1`. + pub var_off: Vec, +} + +/// Per-hap decode of a read-bound split into the [`VariantsSoa`], mirroring +/// genoray's `decode_hap`: merge each hap's `vk` with its present-dense entries +/// (position-sorted, `vk` before dense, snp before indel — see [`merge_hap`] via +/// [`split_to_flat`]), then decode each merged key via [`decode_alt`]. There is +/// NO overlap/clip filter here — the gather already restricts to overlapping +/// variants, unlike [`hap_diffs_svar2`]/reconstruct's ref_idx-consumed clipping, +/// which only matters for sizing a fixed-length output buffer. +#[allow(clippy::too_many_arguments)] +pub fn decode_variants_from_split( + br: &BatchResultSplit, + lut_bytes: &[u8], + lut_off: &[i64], + fields: &[FieldGather], + on_disk_snp: &[Range], + on_disk_indel: &[Range], + orig_samples: &[usize], +) -> (VariantsSoa, Vec>) { + // Fused decode straight from the split gather result: NO `split_to_flat` + // marshaling copy, and NO per-hap `merge_hap` Vec+sort. The three per-hap + // runs (var_key, present dense-snp, present dense-indel) are each already + // position-sorted, so we stream a 3-way merge, decoding each key as we go. + // + // NOTE: dense keys are hap-independent, so pre-decoding them once per query + // and copying across haps was tried (to kill the apparent per-(hap,variant) + // re-decode). It REGRESSED — dense-SNP `decode_alt` is a 2-bit->1-byte no-op, + // cheaper than the pre-decoded slice-copy + its allocation, so inline decode + // wins. The real variant cost is the gather bit-extraction + the unavoidable + // per-hap alt-byte emit, not the decode. (Measured 2026-07-06.) + use genoray_core::bits_get_bit; + let ploidy = br.ploidy; + let n_q = br.n_regions; + let h_count = n_q * ploidy; + + // Upper bound on total merged variants: every vk entry plus every dense + // present bit (over-reserving is harmless). + let vk_total = br.vk_off[h_count]; + let dense_bits = br.dense_snp_present_off[h_count] + br.dense_indel_present_off[h_count]; + let cap = vk_total + dense_bits; + let mut pos: Vec = Vec::with_capacity(cap); + let mut ilen: Vec = Vec::with_capacity(cap); + let mut alt_bytes: Vec = Vec::with_capacity(cap); + let mut str_off: Vec = Vec::with_capacity(cap + 1); + str_off.push(0); + let mut var_off: Vec = Vec::with_capacity(h_count + 1); + var_off.push(0); + let mut field_bufs: Vec> = fields + .iter() + .map(|f| Vec::with_capacity(cap * f.width)) + .collect(); + + // A `BatchResultSplit` without provenance (`gather_haps_readbound`, not + // `_src`) leaves `vk_src` empty; indexing it below would panic with an + // opaque out-of-bounds error instead of naming the real cause. Check once, + // O(1), instead of a per-variant `debug_assert` in the hot loop — this is + // an unconditional `assert_eq!` (matches genoray's own contract check in + // `BatchResultSplit`) so it fires in release builds too, before any wrong + // provenance can attach a field value to the wrong variant. + if !fields.is_empty() { + assert_eq!( + br.vk_src.len(), + br.vk.len(), + "fields require a BatchResultSplit from gather_haps_readbound_src \ + (vk_src must be populated 1:1 with vk)" + ); + } + + let mut h = 0usize; + for q in 0..n_q { + let Range { start: ss, end: se } = br.dense_snp_range[q].clone(); + let Range { start: is_, end: ie } = br.dense_indel_range[q].clone(); + for _hap in 0..ploidy { + let vk_lo = br.vk_off[h]; + let vk_hi = br.vk_off[h + 1]; + let snp_base = br.dense_snp_present_off[h]; + let indel_base = br.dense_indel_present_off[h]; + + // 3-way merge, position-sorted. On equal positions the priority is + // var_key < dense-snp < dense-indel, exactly the stable-sort tie + // order of the previous collect-then-sort `merge_hap` (var_key + // pushed first, then dense-snp, then dense-indel). Byte-identical. + let mut i_vk = vk_lo; + let mut i_sn = ss; + let mut i_in = is_; + loop { + // Advance dense pointers to the next PRESENT entry. Pointers are + // monotonic, so total skip work is O(window) per hap, not O(n^2). + while i_sn < se && !bits_get_bit(&br.dense_snp_present, snp_base + (i_sn - ss)) { + i_sn += 1; + } + while i_in < ie + && !bits_get_bit(&br.dense_indel_present, indel_base + (i_in - is_)) + { + i_in += 1; + } + let has_vk = i_vk < vk_hi; + let has_sn = i_sn < se; + let has_in = i_in < ie; + if !has_vk && !has_sn && !has_in { + break; + } + let p_vk = if has_vk { br.vk[i_vk].position } else { u32::MAX }; + let p_sn = if has_sn { br.dense_snp[i_sn].position } else { u32::MAX }; + let p_in = if has_in { br.dense_indel[i_in].position } else { u32::MAX }; + let (p, key, chan, cidx) = if has_vk && p_vk <= p_sn && p_vk <= p_in { + let e = &br.vk[i_vk]; + let out = (e.position, e.key, 0u8, i_vk); + i_vk += 1; + out + } else if has_sn && p_sn <= p_in { + let e = &br.dense_snp[i_sn]; + let out = (e.position, e.key, 1u8, i_sn); + i_sn += 1; + out + } else { + let e = &br.dense_indel[i_in]; + let out = (e.position, e.key, 2u8, i_in); + i_in += 1; + out + }; + let (il, alt) = decode_alt(key, lut_bytes, lut_off); + pos.push(p as i32); + ilen.push(il as i32); + alt_bytes.extend_from_slice(&alt); + str_off.push(alt_bytes.len() as i64); + + if !fields.is_empty() { + // Resolve (sub_ix, row) for this emitted variant. + let (sub_ix, row, is_dense) = match chan { + 0 => { + let (is_indel, call_idx) = unpack_vk_src(br.vk_src[cidx]); + (if is_indel { 1 } else { 0 }, call_idx, false) + } + 1 => ( + 2, + dense_abs_row(&on_disk_snp[q], &br.dense_snp_range[q], cidx), + true, + ), + _ => ( + 3, + dense_abs_row(&on_disk_indel[q], &br.dense_indel_range[q], cidx), + true, + ), + }; + for (fi, f) in fields.iter().enumerate() { + // var_key entries are already per-(variant, sample) CALLS, so a + // FORMAT value is indexed by the call index directly. Only the + // DENSE channel, which is variant-major over the whole cohort, + // needs the sample stride. + let elem = if is_dense && f.is_format { + row * f.cohort_n_samples + orig_samples[q] + } else { + row + }; + field_bufs[fi].extend_from_slice(f.views[sub_ix].bytes_at(elem)); + } + } + } + var_off.push(pos.len() as i64); + h += 1; + } + } + + ( + VariantsSoa { + pos, + ilen, + alt_bytes, + str_off, + var_off, + }, + field_bufs, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_decode_inline_del_lookup() { + // Pure DEL of length 2 → v_diff -2, empty allele. + let del = svar2_codec::encode_pure_del(-2); + let (d, a) = decode_alt(del, &[], &[0]); + assert_eq!(d, -2); + assert!(a.is_empty()); + + // Lookup row 0 → "ACGT" from the LUT (v_diff = len-1 = 3). + let lut = b"ACGT".to_vec(); + let off = vec![0i64, 4]; + let lk = svar2_codec::encode_lookup(0); + let (d, a) = decode_alt(lk, &lut, &off); + assert_eq!(d, 3); + assert_eq!(a.as_ref(), b"ACGT"); + } + + #[test] + fn test_merge_hap_position_sorted_var_key_before_dense_on_tie() { + // var_key entries for this hap (positions 10 and 20; 20 ties with a dense entry). + let vk_pos = [10i32, 20]; + let vk_key = [100i32, 200]; + + // dense channel spans multiple haps/regions; this hap's window is [ds, de). + // dense positions: 15, 20 (ties with vk_pos[1]=20), 30. + let dense_pos = [15i32, 20, 30]; + let dense_key = [150i32, 250, 300]; + let ds = 0usize; + let de = 3usize; + + // Present bits (LSB-first) over the window: all three dense entries present. + let present = [true, true, true]; + let present_bit = |k: usize| present[k]; + + let merged = merge_hap( + &vk_pos, + &vk_key, + 0, + vk_pos.len(), + &dense_pos, + &dense_key, + ds, + de, + present_bit, + ); + + // Expect position-sorted (pos, key): 10, 15, 20 (var_key first on tie), 20 (dense), 30. + assert_eq!( + merged, + vec![(10, 100), (15, 150), (20, 200), (20, 250), (30, 300)] + ); + } + + #[test] + fn test_hap_diffs_svar2_snp_and_del() { + // 1 query, 1 hap, region [0, 100). Two var_key entries, no dense entries: + // a SNP at pos 10 (ilen 0) and a single-base DEL at pos 20 (ilen -1), both + // fully inside the region. Expected diff = 0 + (-1) = -1. + let regions = ndarray::array![[0i32, 0, 100]]; + let ploidy = 1usize; + + let vk_pos = [10i32, 20]; + let vk_key = [ + svar2_codec::encode_alt_inline(b"A", 0) as i32, + svar2_codec::encode_pure_del(-1) as i32, + ]; + let vk_off: [i64; 2] = [0, 2]; + + let dense_pos: [i32; 0] = []; + let dense_key: [i32; 0] = []; + let dense_range = ndarray::array![[0i32, 0]]; + let dense_present: [u8; 0] = []; + let dense_present_off: [i64; 2] = [0, 0]; + + let lut_bytes: [u8; 0] = []; + let lut_off: [i64; 0] = []; + + let diffs = hap_diffs_svar2( + regions.view(), + ploidy, + &vk_pos, + &vk_key, + &vk_off, + &dense_pos, + &dense_key, + dense_range.view(), + &dense_present, + &dense_present_off, + &lut_bytes, + &lut_off, + ); + + assert_eq!(diffs[[0, 0]], -1); + } + + #[test] + fn test_split_to_flat_marshals_readbound_split() { + use genoray_core::query::KeyRef; + + // ploidy=1, n_regions=1: one vk key, one dense_snp entry (present for + // this hap), one dense_indel entry (absent for this hap). + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 5, + key: 100, + }], + vk_off: vec![0, 1], + dense_snp: vec![KeyRef { + position: 10, + key: 200, + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], // present + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 15, + key: 300, + }], + dense_indel_range: vec![0..1], + dense_indel_present: vec![0b0], // absent + dense_indel_present_off: vec![0, 1], + ..Default::default() + }; + + let flat = split_to_flat(&br); + + assert_eq!(flat.vk_pos, vec![5]); + assert_eq!(flat.vk_key, vec![100]); + assert_eq!(flat.vk_off, vec![0, 1]); + + // Combined dense window: snp entries first, then indel entries. + assert_eq!(flat.dense_pos, vec![10, 15]); + assert_eq!(flat.dense_key, vec![200, 300]); + assert_eq!(flat.dense_range, vec![0, 2]); + + // Combined presence bits: snp bit (present) then indel bit (absent), + // LSB-first -> byte 0 = 0b01. + assert_eq!(flat.dense_present, vec![0b01]); + assert_eq!(flat.dense_present_off, vec![0, 2]); + } + + #[test] + fn test_split_to_flat_trailing_zero_byte_is_allocated() { + use genoray_core::query::KeyRef; + + // Regression for the OOB defect: `dense_present` must always be + // ceil(total_bits/8) bytes, even when the trailing byte is entirely + // zero. 12 haps (ploidy=1, n_regions=12), each with a 1-entry dense/snp + // window and no dense/indel -> 12 combined presence bits spanning 2 + // bytes. Only haps 0 and 3 carry the entry (both in byte 0); haps 8..12 + // (byte 1) are all UNSET. The grow-on-set path alone would leave byte 1 + // unallocated, so the reused kernels' `dense_present[bit/8]` read for + // hap 8..12 would panic. + let n = 12usize; + + // genoray's own bitstream: window size 1 per hap, bits set at 0 and 3. + let mut dense_snp_present = vec![0u8; n.div_ceil(8)]; // 2 bytes + dense_snp_present[0] = 0b0000_1001; // bits 0 and 3 + let dense_snp_present_off: Vec = (0..=n).collect(); + + let br = BatchResultSplit { + n_regions: n, + n_samples: 1, + ploidy: 1, + vk: vec![], + vk_off: vec![0; n + 1], + dense_snp: vec![KeyRef { + position: 42, + key: 7, + }], + dense_snp_range: vec![0..1; n], // every query points at the lone entry + dense_snp_present, + dense_snp_present_off, + dense_indel: vec![], + dense_indel_range: vec![0..0; n], // no indel window + dense_indel_present: vec![], + dense_indel_present_off: vec![0; n + 1], + ..Default::default() + }; + + // Must not panic. + let flat = split_to_flat(&br); + + // Buffer sized to ceil(total_bits/8) = ceil(12/8) = 2, NOT just up to + // the highest set bit (byte 0). + assert_eq!(flat.dense_present.len(), 2); + assert_eq!(flat.dense_present, vec![0b0000_1001, 0b0000_0000]); + // 12 haps, each contributing exactly 1 combined presence bit. + assert_eq!(*flat.dense_present_off.last().unwrap(), n as i64); + assert_eq!( + flat.dense_present.len(), + (*flat.dense_present_off.last().unwrap() as usize).div_ceil(8) + ); + + // The set bits land exactly where expected (haps 0 and 3), and the + // trailing-byte haps are unset — proving no shift/corruption. + for h in 0..n { + let want = h == 0 || h == 3; + let got = genoray_core::bits_get_bit(&flat.dense_present, h); + assert_eq!(got, want, "hap {h} presence bit"); + } + } + + #[test] + fn test_split_to_flat_ploidy_gt1_reuses_per_query_window_across_haps() { + use genoray_core::query::KeyRef; + + // Regression for the q=h/ploidy hoist: ploidy=2, n_regions=2 (h_count=4) + // so each query's dense_snp/dense_indel window is shared by 2 haps, but + // each hap has its own presence bits at its own `*_present_off` offset. + // This exercises the outer-q/inner-hap loop restructuring (the old code + // recomputed `q = h / ploidy` per hap via a runtime division; the new + // code hoists the per-query window lookup out of the hap loop) — proving + // it doesn't mix up which hap's presence bits attach to which query's + // window. The 12 combined presence bits (3/hap * 4 haps) also cross a + // byte boundary with mixed set/unset bits, like the trailing-zero test, + // but here across queries+haps instead of a single-bit-per-hap window. + let dense_snp: Vec = (0..4) + .map(|i| KeyRef { + position: 10 + i, + key: 200 + i, + }) + .collect(); + let dense_indel: Vec = (0..2) + .map(|i| KeyRef { + position: 50 + i, + key: 500 + i, + }) + .collect(); + + let br = BatchResultSplit { + n_regions: 2, + n_samples: 1, + ploidy: 2, + vk: vec![], + vk_off: vec![0; 5], + dense_snp, + // query 0 owns snp[0..2), query 1 owns snp[2..4) — width 2/query, + // shared by both of that query's haps. + dense_snp_range: vec![0..2, 2..4], + // hap0 bits(0,1)=(1,0), hap1 bits(2,3)=(0,1), hap2 bits(4,5)=(1,1), + // hap3 bits(6,7)=(0,0) -> byte 0b0011_1001. + dense_snp_present: vec![0b0011_1001], + dense_snp_present_off: vec![0, 2, 4, 6, 8], + dense_indel, + // query 0 owns indel[0..1), query 1 owns indel[1..2) — width 1/query. + dense_indel_range: vec![0..1, 1..2], + // hap0=1, hap1=0, hap2=1, hap3=1 -> byte 0b0000_1101. + dense_indel_present: vec![0b0000_1101], + dense_indel_present_off: vec![0, 1, 2, 3, 4], + ..Default::default() + }; + + let flat = split_to_flat(&br); + + // snp-then-indel per query, queries in order. + assert_eq!(flat.dense_pos, vec![10, 11, 50, 12, 13, 51]); + assert_eq!(flat.dense_key, vec![200, 201, 500, 202, 203, 501]); + assert_eq!(flat.dense_range, vec![0, 3, 3, 6]); + + // 4 haps * 3 bits/hap = 12 bits -> 2 bytes, spanning a byte boundary. + assert_eq!(flat.dense_present_off, vec![0, 3, 6, 9, 12]); + assert_eq!(flat.dense_present, vec![0b1101_0101, 0b0000_1001]); + } + + #[test] + fn test_decode_variants_from_split_merges_and_decodes() { + use genoray_core::query::KeyRef; + + // 1 region, 1 sample (read-bound), ploidy 1 -> 1 hap. var_key: SNP at + // pos 5 (inline ALT "T"). dense_snp: one entry at pos 8, PRESENT for + // this hap (inline ALT "G"). dense_indel: one entry at pos 12, PRESENT + // for this hap (pure DEL, ilen -2, empty ALT). + let vk_key = svar2_codec::encode_alt_inline(b"T", 0); + let dense_snp_key = svar2_codec::encode_alt_inline(b"G", 0); + let dense_indel_key = svar2_codec::encode_pure_del(-2); + + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 5, + key: vk_key, + }], + vk_off: vec![0, 1], + dense_snp: vec![KeyRef { + position: 8, + key: dense_snp_key, + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 12, + key: dense_indel_key, + }], + dense_indel_range: vec![0..1], + dense_indel_present: vec![0b1], + dense_indel_present_off: vec![0, 1], + ..Default::default() + }; + + let (soa, _bufs) = decode_variants_from_split(&br, &[], &[0], &[], &[], &[], &[]); + + // Position-sorted: var_key SNP@5, dense/snp@8, dense/indel@12. + assert_eq!(soa.var_off, vec![0, 3]); + assert_eq!(soa.pos, vec![5, 8, 12]); + assert_eq!(soa.ilen, vec![0, 0, -2]); + assert_eq!(soa.alt_bytes, b"TG".to_vec()); + // Pure-del ALT is empty -> the 3rd variant's [start, end) is [2, 2). + assert_eq!(soa.str_off, vec![0, 1, 2, 2]); + } + + /// Exercises the two things the asm-inspection pass touched: (1) `q = + /// h / ploidy` computed via an incrementing counter instead of a division + /// (needs `ploidy > 1` so some consecutive haps share a `q`, which the + /// single-hap tests above never trigger), and (2) the `present_bit` + /// closure's read of `dense_present`, with a mix of + /// present/absent bits whose per-hap `base_bit` windows straddle a byte + /// boundary (hap 1's 5-bit window covers global bits 5..10, i.e. bytes 0 + /// and 1). + #[test] + fn test_decode_variants_from_split_byte_identical_presence_edge() { + use genoray_core::query::KeyRef; + + let k = |b: &[u8]| svar2_codec::encode_alt_inline(b, 0); + + // 2 regions x ploidy 2 = 4 haps, no var_key entries (isolates the + // dense/present-bit path). Region 0's dense window is 3 snp + 2 indel + // (width 5, shared by haps 0 & 1); region 1's is 3 snp + 0 indel + // (width 3, shared by haps 2 & 3). Combined presence bitstream is 16 + // bits = 2 bytes, with hap 1's window (global bits 5..10) crossing + // the byte-0/byte-1 boundary at bit 8. + let br = BatchResultSplit { + n_regions: 2, + n_samples: 1, + ploidy: 2, + vk: vec![], + vk_off: vec![0, 0, 0, 0, 0], + dense_snp: vec![ + KeyRef { position: 10, key: k(b"A") }, + KeyRef { position: 11, key: k(b"C") }, + KeyRef { position: 12, key: k(b"G") }, + KeyRef { position: 50, key: k(b"T") }, + KeyRef { position: 51, key: k(b"A") }, + KeyRef { position: 52, key: k(b"C") }, + ], + dense_snp_range: vec![0..3, 3..6], + // Per-hap snp-bit widths 3,3,3,3 -> offsets 0,3,6,9,12. Bitstream + // (idx0..11): 1,0,1, 0,1,0, 1,1,0, 0,0,1 -> byte0 = 0b1101_0101 + // (bits0-7: 1,0,1,0,1,0,1,1 -> 1+4+16+64+128=213), byte1 low + // nibble (bits8-11: 0,0,0,1 -> 8). + dense_snp_present: vec![213, 8], + dense_snp_present_off: vec![0, 3, 6, 9, 12], + dense_indel: vec![ + KeyRef { position: 13, key: svar2_codec::encode_pure_del(-2) }, + KeyRef { position: 14, key: svar2_codec::encode_pure_del(-5) }, + ], + dense_indel_range: vec![0..2, 2..2], + // Per-hap indel-bit widths 2,2,0,0 -> offsets 0,2,4,4,4. + // Bitstream (idx0..3): 1,0, 0,1 -> byte0 = 0b1001 (1+8=9). + dense_indel_present: vec![9], + dense_indel_present_off: vec![0, 2, 4, 4, 4], + ..Default::default() + }; + + let (soa, _bufs) = decode_variants_from_split(&br, &[], &[0], &[], &[], &[], &[]); + + // hap0 (q0): snp present [1,0,1] -> keeps pos10("A"),pos12("G"); + // indel present [1,0] -> keeps pos13(ilen -2). + // hap1 (q0): snp present [0,1,0] -> keeps pos11("C"); + // indel present [0,1] -> keeps pos14(ilen -5). + // hap2 (q1): snp present [1,1,0] -> keeps pos50("T"),pos51("A"). + // hap3 (q1): snp present [0,0,1] -> keeps pos52("C"). + assert_eq!(soa.pos, vec![10, 12, 13, 11, 14, 50, 51, 52]); + assert_eq!(soa.ilen, vec![0, 0, -2, 0, -5, 0, 0, 0]); + assert_eq!(soa.alt_bytes, b"AGCTAC".to_vec()); + assert_eq!(soa.str_off, vec![0, 1, 2, 2, 3, 3, 4, 5, 6]); + assert_eq!(soa.var_off, vec![0, 3, 5, 7, 8]); + } + + /// Build 4 FieldViews over a store where each sub-stream's content is + /// distinguishable: element `i` of sub-stream `sub_ix` (in + /// `FieldSub::all()` order, i.e. `[VkSnp=0, VkIndel=1, DenseSnp=2, + /// DenseIndel=3]`) has value `100 * sub_ix + i`. A test that decodes value + /// `V` therefore reveals BOTH which sub-stream (`V / 100`) and which row + /// (`V % 100`) the decoder attributed to that variant — a same-content + /// store (as an earlier version of this fixture used) cannot distinguish + /// "read the right sub-stream" from "read some sub-stream". + /// Returns the TempDir too — it must outlive the views (dropping it deletes the mmapped files). + fn make_identity_i32_fields() -> (tempfile::TempDir, Vec) { + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + + let tmp = tempfile::tempdir().unwrap(); + let base = tmp.path().to_str().unwrap(); + let paths = ContigPaths::new(base, "chr1"); + + const N: usize = 8; // enough for call idx 5 and dense row 3/4 + + let mut views = Vec::with_capacity(4); + for (sub_ix, sub) in FieldSub::all().into_iter().enumerate() { + let bytes: Vec = (0..N as i32) + .flat_map(|i| (100 * sub_ix as i32 + i).to_le_bytes()) + .collect(); + let p = paths.field_values("info", "X", sub); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, &bytes).unwrap(); + views.push(FieldView::open(&paths, "info", "X", sub, StorageDtype::I32, 1).unwrap()); + } + let views: [FieldView; 4] = views.try_into().map_err(|_| ()).unwrap(); + + ( + tmp, + vec![FieldGather { + views, + is_format: false, // INFO -> element index is the row itself + width: 4, + cohort_n_samples: 1, + }], + ) + } + + #[test] + fn test_decode_fields_provenance_identity() { + use genoray_core::query::{pack_vk_src, KeyRef}; + + // One query, ploidy 1, all four provenance channels represented: + // var_key SNP entry at pos 10, provenance = (snp, call idx 5) -> sub VkSnp=0 + // var_key indel entry at pos 15, provenance = (indel, call idx 2) -> sub VkIndel=1 + // dense-snp entry at pos 20, output window 0..1, ON-DISK window 3..4 -> abs row 3, sub DenseSnp=2 + // dense-indel entry at pos 25, output window 0..1, ON-DISK window 4..5 -> abs row 4, sub DenseIndel=3 + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![ + KeyRef { + position: 10, + key: svar2_codec::encode_pure_del(-1), + }, + KeyRef { + position: 15, + key: svar2_codec::encode_pure_del(-1), + }, + ], + vk_off: vec![0, 2], + vk_src: vec![pack_vk_src(false, 5), pack_vk_src(true, 2)], + dense_snp: vec![KeyRef { + position: 20, + key: svar2_codec::encode_pure_del(-1), + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![KeyRef { + position: 25, + key: svar2_codec::encode_pure_del(-1), + }], + dense_indel_range: vec![0..1], + dense_indel_present: vec![0b1], + dense_indel_present_off: vec![0, 1], + ..Default::default() + }; + + let (_tmp, fields) = make_identity_i32_fields(); // keep _tmp alive: it owns the tempdir + + let (soa, bufs) = decode_variants_from_split( + &br, + &[], + &[0i64], + &fields, + &[3..4], // on_disk_snp: the dense-snp window really lives at rows 3..4 on disk + &[4..5], // on_disk_indel: the dense-indel window really lives at rows 4..5 on disk + &[0], // orig_samples + ); + + // Position-sorted: var_key(10,15) before dense(20,25); snp before indel on ties (none here). + assert_eq!(soa.pos, vec![10, 15, 20, 25]); + + // value = 100*sub_ix + row, so each decoded value pins BOTH the + // sub-stream routing and the row/offset arithmetic for its channel: + // var_key snp -> sub 0, call idx 5 -> 100*0+5 = 5 + // var_key indel -> sub 1, call idx 2 -> 100*1+2 = 102 + // dense-snp -> sub 2, abs row 3 -> 100*2+3 = 203 + // dense-indel -> sub 3, abs row 4 -> 100*3+4 = 304 + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(vals, vec![5, 102, 203, 304]); + } + + #[test] + fn test_decode_fields_format_dense_stride_and_var_key_unstrided() { + use genoray_core::field::StorageDtype; + use genoray_core::layout::{ContigPaths, FieldSub}; + use genoray_core::query::{pack_vk_src, KeyRef}; + + // A FORMAT field over a 3-sample cohort. Store is plain identity + // (element i has value i) across all four subs — this test isolates + // the *stride* arithmetic, not sub-stream routing (that's pinned by + // `test_decode_fields_provenance_identity`). + let tmp = tempfile::tempdir().unwrap(); + let base = tmp.path().to_str().unwrap(); + let paths = ContigPaths::new(base, "chr1"); + + const N: usize = 16; // covers dense elem 3*3+2=11 and var_key call idx 5 + let bytes: Vec = (0..N as i32).flat_map(|i| i.to_le_bytes()).collect(); + + let mut views = Vec::with_capacity(4); + for sub in FieldSub::all() { + let p = paths.field_values("format", "DP", sub); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, &bytes).unwrap(); + // n_samples passed to `open` MUST match `cohort_n_samples` below. + views.push(FieldView::open(&paths, "format", "DP", sub, StorageDtype::I32, 3).unwrap()); + } + let views: [FieldView; 4] = views.try_into().map_err(|_| ()).unwrap(); + + let fields = vec![FieldGather { + views, + is_format: true, + width: 4, + cohort_n_samples: 3, + }]; + + // var_key SNP entry at pos 10, call idx 5; dense-snp entry at pos 20, + // output window 0..1, ON-DISK window 3..4 -> abs row 3. + let br = BatchResultSplit { + n_regions: 1, + n_samples: 1, + ploidy: 1, + vk: vec![KeyRef { + position: 10, + key: svar2_codec::encode_pure_del(-1), + }], + vk_off: vec![0, 1], + vk_src: vec![pack_vk_src(false, 5)], + dense_snp: vec![KeyRef { + position: 20, + key: svar2_codec::encode_pure_del(-1), + }], + dense_snp_range: vec![0..1], + dense_snp_present: vec![0b1], + dense_snp_present_off: vec![0, 1], + dense_indel: vec![], + dense_indel_range: vec![0..0], + dense_indel_present: vec![], + dense_indel_present_off: vec![0, 0], + ..Default::default() + }; + + let (soa, bufs) = decode_variants_from_split( + &br, + &[], + &[0i64], + &fields, + &[3..4], // on_disk_snp + &[0..0], // on_disk_indel + &[2], // orig_samples: this query's original cohort sample index is 2 + ); + + assert_eq!(soa.pos, vec![10, 20]); + + let vals: Vec = bufs[0] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + // var_key FORMAT is UNSTRIDED: a var_key entry is already a per-CALL + // value, so the element index is the call index directly (5), NOT + // `call_idx * cohort_n_samples + orig_sample` (5*3+2=17). This pins + // the `is_dense && f.is_format` gate — the part most likely to + // silently regress. + // dense FORMAT IS strided: abs row 3, orig_sample 2, cohort_n_samples 3 + // -> element 3*3+2 = 11. + assert_eq!(vals, vec![5, 11]); + } +} diff --git a/src/svar2/store.rs b/src/svar2/store.rs new file mode 100644 index 00000000..1e7f6549 --- /dev/null +++ b/src/svar2/store.rs @@ -0,0 +1,55 @@ +use std::collections::HashMap; + +use genoray_core::query::ContigReader; +use pyo3::exceptions::PyIOError; +use pyo3::prelude::*; + +/// Opened once at Dataset.open; holds one query-only ContigReader per contig for +/// the store's lifetime (SVAR2 analog of SVAR1's cached _HapsFfiStatic). +#[pyclass] +pub struct Svar2Store { + readers: HashMap, + store_path: String, +} + +impl Svar2Store { + /// Returns the cached `ContigReader` for `contig`, if one was opened. + pub fn reader(&self, contig: &str) -> Option<&ContigReader> { + self.readers.get(contig) + } + /// Returns the filesystem path the store was opened from. + pub fn store_path(&self) -> &str { + &self.store_path + } +} + +#[pymethods] +impl Svar2Store { + /// Opens one query-only `ContigReader` per contig under `store_path`. + #[new] + fn new( + store_path: &str, + contigs: Vec, + n_samples: usize, + ploidy: usize, + ) -> PyResult { + let mut readers = HashMap::with_capacity(contigs.len()); + for c in contigs { + let r = ContigReader::open(store_path, &c, n_samples, ploidy) + .map_err(|e| PyIOError::new_err(format!("open contig {c}: {e}")))?; + readers.insert(c, r); + } + let store_path = store_path.to_string(); + Ok(Self { + readers, + store_path, + }) + } + + /// Returns the sorted list of contigs with an opened reader. + fn contigs(&self) -> Vec { + let mut v: Vec = self.readers.keys().cloned().collect(); + v.sort(); + v + } +} diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index a0bfcb0c..c17c1088 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -189,10 +189,17 @@ pub fn apply_insertion_fill( } } -/// Shift and realign a single track to correspond to one haplotype. +/// Shift and realign a single track to correspond to one haplotype, generic over the +/// variant source. /// -/// Mirrors numba `shift_and_realign_track_sparse` (lines 230-401 of `_tracks.py`) -/// statement-by-statement. +/// Mirrors numba `shift_and_realign_track_sparse` (lines 230-401 of `_tracks.py`), with +/// the per-variant reads factored out behind the `provide` closure so the same loop body +/// can be driven by either the SVAR 1.0 global variant table +/// (`shift_and_realign_track_sparse`) or the SVAR2 two-channel (var_key ⋈ dense) source +/// (`shift_and_realign_tracks_from_svar2`). **Do not change the realignment math +/// here** — only the data source differs between call sites. Unlike the reconstruction +/// core, there is no allele/annotation plumbing and no anchor fix: a DEL implicitly +/// writes `track[v_rel_pos]` via the `else` branch below. /// /// Three key differences from the haplotype reconstruction kernel: /// 1. SNPs (`v_diff == 0`) are SKIPPED — tracks match reference at SNP positions. @@ -201,11 +208,10 @@ pub fn apply_insertion_fill( /// 3. Trailing fill pads with `0.0` (NOT a pad_char byte). /// /// # Parameters -/// - `offset_idx`: index into geno_o_starts/geno_o_stops for this (query, hap) pair -/// - `geno_v_idxs`: flat variant index array -/// - `geno_o_starts`, `geno_o_stops`: normalized (2, n) offsets split into two rows -/// - `v_starts`: variant start positions (absolute genomic coordinates) -/// - `ilens`: variant insertion-length differences (signed) +/// - `n_variants`: number of variants this haplotype sees, i.e. `0..n_variants` are +/// valid indices into `provide` +/// - `provide`: `Fn(v) -> (v_start_abs, v_diff)` for `v in 0..n_variants`: `v_start_abs` +/// genomic start, `v_diff` ilen (signed) /// - `shift`: total shift for this haplotype /// - `track`: reference track values for this query (f32 slice) /// - `query_start`: the genomic start of this query region @@ -215,13 +221,9 @@ pub fn apply_insertion_fill( /// - `strategy_id`: insertion-fill strategy /// - `base_seed`, `query`, `hap`: seed components for FlankSample strategy #[allow(clippy::too_many_arguments)] -pub fn shift_and_realign_track_sparse( - offset_idx: usize, - geno_v_idxs: ndarray::ArrayView1, - geno_o_starts: ndarray::ArrayView1, - geno_o_stops: ndarray::ArrayView1, - v_starts: ndarray::ArrayView1, - ilens: ndarray::ArrayView1, +fn shift_and_realign_track_core( + n_variants: usize, + provide: impl Fn(usize) -> (i64, i64), // (v_start_abs, v_diff) shift: i64, track: ndarray::ArrayView1, query_start: i64, @@ -233,14 +235,7 @@ pub fn shift_and_realign_track_sparse( query: u64, hap: u64, ) { - // Numba: o_s, o_e = geno_offsets[offset_idx], geno_offsets[offset_idx + 1] (1-D branch) - // or geno_offsets[:, offset_idx] (2-D branch — normalized form) - // We receive the pre-split (2, n) rows directly. - let o_s = geno_o_starts[offset_idx] as usize; - let o_e = geno_o_stops[offset_idx] as usize; - let variant_idxs = &geno_v_idxs.as_slice().unwrap()[o_s..o_e]; let length = out.len(); - let n_variants = variant_idxs.len(); if n_variants == 0 { // Numba: out[:] = track[:length] @@ -263,12 +258,11 @@ pub fn shift_and_realign_track_sparse( } } - let variant = variant_idxs[v] as usize; + let (v_start, v_diff) = provide(v); // Numba: v_rel_pos = v_starts[variant] - query_start - let v_rel_pos = v_starts[variant] as i64 - query_start; - // Numba: v_diff = ilens[variant] - let v_diff = ilens[variant] as i64; + let v_rel_pos = v_start - query_start; + // v_diff already i64 // Numba: v_rel_end = v_rel_pos - min(0, v_diff) + 1 let v_rel_end = v_rel_pos - v_diff.min(0) + 1; @@ -411,11 +405,76 @@ pub fn shift_and_realign_track_sparse( } } +/// Shift and realign a single track to correspond to one haplotype (SVAR 1.0 global +/// variant table). Thin wrapper over [`shift_and_realign_track_core`]: builds a closure +/// that indexes the global table and delegates. **Behavior is unchanged** from before +/// the closure-source refactor — this only changes where the per-variant data comes +/// from. +/// +/// # Parameters +/// - `offset_idx`: index into geno_o_starts/geno_o_stops for this (query, hap) pair +/// - `geno_v_idxs`: flat variant index array +/// - `geno_o_starts`, `geno_o_stops`: normalized (2, n) offsets split into two rows +/// - `v_starts`: variant start positions (absolute genomic coordinates) +/// - `ilens`: variant insertion-length differences (signed) +/// - `shift`: total shift for this haplotype +/// - `track`: reference track values for this query (f32 slice) +/// - `query_start`: the genomic start of this query region +/// - `out`: output slice to fill (length = haplotype output length) +/// - `params`: per-strategy parameter (f64) +/// - `keep`: optional boolean mask over the variant group for this (query, hap) +/// - `strategy_id`: insertion-fill strategy +/// - `base_seed`, `query`, `hap`: seed components for FlankSample strategy +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_track_sparse( + offset_idx: usize, + geno_v_idxs: ndarray::ArrayView1, + geno_o_starts: ndarray::ArrayView1, + geno_o_stops: ndarray::ArrayView1, + v_starts: ndarray::ArrayView1, + ilens: ndarray::ArrayView1, + shift: i64, + track: ndarray::ArrayView1, + query_start: i64, + out: &mut ndarray::ArrayViewMut1, + params: ndarray::ArrayView1, + keep: Option>, + strategy_id: i64, + base_seed: u64, + query: u64, + hap: u64, +) { + // Numba: o_s, o_e = geno_offsets[offset_idx], geno_offsets[offset_idx + 1] (1-D branch) + // or geno_offsets[:, offset_idx] (2-D branch — normalized form) + // We receive the pre-split (2, n) rows directly. + let o_s = geno_o_starts[offset_idx] as usize; + let o_e = geno_o_stops[offset_idx] as usize; + let variant_idxs = &geno_v_idxs.as_slice().unwrap()[o_s..o_e]; + let provide = |v: usize| { + let variant = variant_idxs[v] as usize; + (v_starts[variant] as i64, ilens[variant] as i64) + }; + shift_and_realign_track_core( + variant_idxs.len(), + provide, + shift, + track, + query_start, + out, + params, + keep, + strategy_id, + base_seed, + query, + hap, + ); +} + /// Shift and realign tracks for a batch of (query, hap) pairs in place (writes `out`). /// /// Mirrors numba `shift_and_realign_tracks_sparse` (lines 141-228 of `_tracks.py`) -/// statement-by-statement. Serial-only (rayon deferred to Phase 5, matching Task 5 -/// precedent for initial parity verification). +/// statement-by-statement. Serial-only (rayon deferred) for initial parity +/// verification. /// /// # Parameters /// - `out`: flat output buffer (f32), written in place @@ -463,11 +522,16 @@ pub fn shift_and_realign_tracks_sparse( // applied the same pattern: out.as_slice_mut().unwrap() once, then index [a..b] // directly. Here we do the same for out, tracks, and keep. // geno_v_idxs already uses .as_slice().unwrap() (inner fn line 240) — same contract. - let out_flat = out.as_slice_mut().expect("out must be contiguous (C-order)"); - let tracks_flat = tracks.as_slice().expect("tracks must be contiguous (C-order)"); + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + let tracks_flat = tracks + .as_slice() + .expect("tracks must be contiguous (C-order)"); // Hoist keep flat option once (avoids repeated .as_slice() per hap). - let keep_flat: Option<&[bool]> = - keep.as_ref().map(|k| k.as_slice().expect("keep must be contiguous (C-order)")); + let keep_flat: Option<&[bool]> = keep + .as_ref() + .map(|k| k.as_slice().expect("keep must be contiguous (C-order)")); if parallel { // Build disjoint per-k mutable output slices using the split_at_mut cursor @@ -507,15 +571,14 @@ pub fn shift_and_realign_tracks_sparse( let o_idx = geno_offset_idx[[query, hap]] as usize; let qh_shift = shifts[[query, hap]] as i64; - let qh_keep: Option> = - match (&keep_flat, &keep_offsets) { - (Some(k_flat), Some(ko)) => { - let ks = ko[k] as usize; - let ke = ko[k + 1] as usize; - Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) - } - _ => None, - }; + let qh_keep: Option> = match (&keep_flat, &keep_offsets) { + (Some(k_flat), Some(ko)) => { + let ks = ko[k] as usize; + let ke = ko[k + 1] as usize; + Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) + } + _ => None, + }; let mut qh_out = ndarray::ArrayViewMut1::from(out_chunk); shift_and_realign_track_sparse( @@ -562,15 +625,14 @@ pub fn shift_and_realign_tracks_sparse( // qh_keep = keep[keep_offsets[k_idx]:keep_offsets[k_idx+1]] // ArrayView1::from(&slice[..]) avoids the do_slice call that // k.slice(s![ks..ke]) would generate. - let qh_keep: Option> = - match (&keep_flat, &keep_offsets) { - (Some(k_flat), Some(ko)) => { - let ks = ko[k_idx] as usize; - let ke = ko[k_idx + 1] as usize; - Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) - } - _ => None, - }; + let qh_keep: Option> = match (&keep_flat, &keep_offsets) { + (Some(k_flat), Some(ko)) => { + let ks = ko[k_idx] as usize; + let ke = ko[k_idx + 1] as usize; + Some(ndarray::ArrayView1::from(&k_flat[ks..ke])) + } + _ => None, + }; // Numba: out_s, out_e = out_offsets[k_idx], out_offsets[k_idx + 1] let out_s = out_offsets[k_idx] as usize; @@ -604,6 +666,195 @@ pub fn shift_and_realign_tracks_sparse( } } +/// Shift and realign tracks for a batch of (query, hap) pairs from the SVAR2 +/// two-channel (var_key ⋈ dense) source. Mirrors +/// [`shift_and_realign_tracks_sparse`]'s parallel/serial disjoint-slice carving +/// exactly — only the per-work-item variant source and decode differ. Additive: SVAR +/// 1.0's `shift_and_realign_track[s]_sparse` are untouched by this function. Unlike +/// [`crate::reconstruct::reconstruct_haplotypes_from_svar2`], there are no annotation +/// buffers here (tracks never had them), so the carving is out-chunks only. +/// +/// # Parameters +/// - `out`: flat output buffer (f32), written in place +/// - `out_offsets`: ragged offsets into out, shape (n_q * ploidy + 1,) +/// - `regions`: (n_q, 3) array of (contig_idx, start, end) per query +/// - `shifts`: (n_q, ploidy) shift per (query, hap) +/// - `vk_pos` / `vk_key`: this hap's `var_key` channel: position + uniform key (i32) +/// - `vk_off`: shape (n_work + 1) CSR offsets into `vk_pos`/`vk_key`, indexed by flat +/// work index `k = query * ploidy + hap` +/// - `dense_pos` / `dense_key`: the shared `dense` channel: position + uniform key (i32) +/// - `dense_range`: shape (n_q, 2) as `[ds, de)`, the window into `dense_pos`/`dense_key` +/// for each query row +/// - `dense_present`: packed presence bits over the dense window, LSB-first per byte (u8) +/// - `dense_present_off`: shape (n_work + 1) BIT offsets into `dense_present`, indexed by +/// flat work index `k` +/// - `lut_bytes` / `lut_off`: long-allele-bank bytes + CSR offsets, for `Lookup` keys +/// - `tracks`: flat reference track buffer (f32), ragged by track_offsets +/// - `track_offsets`: (n_q + 1,) offsets into tracks (one track per query) +/// - `params`: per-strategy parameter (f64), shape (1,) +/// - `strategy_id`, `base_seed`: insertion-fill strategy parameters +/// - `query_seed`: optional (n_q,) map from the LOCAL query index (`k / ploidy`) +/// to the GLOBAL batch row used as the FlankSample fill-seed `query` component. +/// `None` seeds with the local index (identity), which is exact for a +/// single-call/single-group batch. Callers that split one logical batch across +/// several kernel invocations (e.g. the SVAR2 read-bound path's per-contig-group +/// loop) MUST pass the group's global row indices so seed-dependent fills match +/// the single fused SVAR1 call. See GenVarLoader issue #267. +/// - `parallel`: if true, use rayon to process work items concurrently +#[allow(clippy::too_many_arguments)] +pub fn shift_and_realign_tracks_from_svar2( + mut out: ndarray::ArrayViewMut1, + out_offsets: ndarray::ArrayView1, + regions: ndarray::ArrayView2, + shifts: ndarray::ArrayView2, + vk_pos: ndarray::ArrayView1, + vk_key: ndarray::ArrayView1, + vk_off: ndarray::ArrayView1, + dense_pos: ndarray::ArrayView1, + dense_key: ndarray::ArrayView1, + dense_range: ndarray::ArrayView2, + dense_present: ndarray::ArrayView1, + dense_present_off: ndarray::ArrayView1, + lut_bytes: ndarray::ArrayView1, + lut_off: ndarray::ArrayView1, + tracks: ndarray::ArrayView1, + track_offsets: ndarray::ArrayView1, + params: ndarray::ArrayView1, + strategy_id: i64, + base_seed: u64, + query_seed: Option>, + parallel: bool, +) { + let ploidy = shifts.ncols(); + let n_work = regions.nrows() * ploidy; + + // Hoist contiguous-slice pointers once, exactly as the SVAR1 driver hoists + // `tracks_flat` — these are Send+Sync so the rayon parallel path is unchanged. + let vk_pos_s: &[i32] = vk_pos.as_slice().unwrap(); + let vk_key_s: &[i32] = vk_key.as_slice().unwrap(); + let dense_pos_s: &[i32] = dense_pos.as_slice().unwrap(); + let dense_key_s: &[i32] = dense_key.as_slice().unwrap(); + let dense_present_s: &[u8] = dense_present.as_slice().unwrap(); + let lut_bytes_s: &[u8] = lut_bytes.as_slice().unwrap(); + let lut_off_s: &[i64] = lut_off.as_slice().unwrap(); + let tracks_flat: &[f32] = tracks + .as_slice() + .expect("tracks must be contiguous (C-order)"); + + // Per-k inner work: merge this hap's var_key ⋈ dense entries, then realign via the + // shared core with a decode closure. All read-only ArrayViews/slices are Send+Sync + // so the closure can borrow them freely. + let do_work = |k: usize, out_chunk: &mut [f32]| { + let query = k / ploidy; + let hap = k % ploidy; + + // FlankSample fill-seed `query` component: GLOBAL batch row when a + // `query_seed` map is supplied (multi-call/per-group batches), else the + // local index (exact for a single fused call). See issue #267. + let q_seed = match query_seed { + Some(qs) => qs[query] as u64, + None => query as u64, + }; + + let t_s = track_offsets[query] as usize; + let t_e = track_offsets[query + 1] as usize; + let q_track = ndarray::ArrayView1::from(&tracks_flat[t_s..t_e]); + let q_start = regions[[query, 1]] as i64; + let qh_shift = shifts[[query, hap]] as i64; + + // var_key window for this hap + let vk_lo = vk_off[k] as usize; + let vk_hi = vk_off[k + 1] as usize; + + // dense window for this query + let ds = dense_range[[query, 0]] as usize; + let de = dense_range[[query, 1]] as usize; + + // presence bits for this hap start at bit `dense_present_off[k]` + let base_bit = dense_present_off[k] as usize; + let present_bit = |j: usize| crate::svar2::present_bit(dense_present_s, base_bit, j); + + let merged = crate::svar2::merge_hap( + vk_pos_s, + vk_key_s, + vk_lo, + vk_hi, + dense_pos_s, + dense_key_s, + ds, + de, + present_bit, + ); + + let provide = |v: usize| { + let (pos, key) = merged[v]; + let (v_diff, _allele) = crate::svar2::decode_alt(key, lut_bytes_s, lut_off_s); + (pos as i64, v_diff) + }; + + let mut qh_out = ndarray::ArrayViewMut1::from(out_chunk); + shift_and_realign_track_core( + merged.len(), + provide, + qh_shift, + q_track, + q_start, + &mut qh_out, + params, + None, // keep: SVAR2 has no per-haplotype keep mask + strategy_id, + base_seed, + q_seed, + hap as u64, + ); + }; + + if parallel { + // Build disjoint per-k mutable output slices using the split_at_mut cursor + // idiom (mirrors shift_and_realign_tracks_sparse's parallel path). + let bounds: Vec<(usize, usize)> = (0..n_work) + .map(|k| (out_offsets[k] as usize, out_offsets[k + 1] as usize)) + .collect(); + + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + let mut out_chunks: Vec<&mut [f32]> = Vec::with_capacity(n_work); + { + let mut rest = &mut out_flat[..]; + let mut cursor = 0usize; + for &(s, e) in &bounds { + debug_assert!( + s >= cursor && e >= s, + "out_offsets must be monotonically non-decreasing (got s={s}, e={e}, cursor={cursor})" + ); + let (_, tail) = rest.split_at_mut(s - cursor); + let (mid, tail2) = tail.split_at_mut(e - s); + out_chunks.push(mid); + rest = tail2; + cursor = e; + } + } + + out_chunks + .into_par_iter() + .enumerate() + .for_each(|(k, out_chunk)| { + do_work(k, out_chunk); + }); + } else { + // Serial path. + let out_flat = out + .as_slice_mut() + .expect("out must be contiguous (C-order)"); + for k in 0..n_work { + let out_s = out_offsets[k] as usize; + let out_e = out_offsets[k + 1] as usize; + do_work(k, &mut out_flat[out_s..out_e]); + } + } +} + /// RLE-encode a ragged f32 track buffer into (starts, ends, values, offsets) intervals. /// /// Mirrors numba `tracks_to_intervals` + `_scanned_mask` + `_compact_mask` in @@ -738,7 +989,12 @@ pub fn tracks_to_intervals( // Build disjoint per-query mutable slices from all_starts/ends/values using // interval_offsets (which have already been computed sequentially above). let itv_bounds: Vec<(usize, usize)> = (0..n_queries) - .map(|q| (interval_offsets[q] as usize, interval_offsets[q + 1] as usize)) + .map(|q| { + ( + interval_offsets[q] as usize, + interval_offsets[q + 1] as usize, + ) + }) .collect(); let mut starts_chunks: Vec<&mut [i32]> = Vec::with_capacity(n_queries); @@ -861,7 +1117,7 @@ pub fn tracks_to_intervals( #[cfg(test)] mod tests { use super::*; - use ndarray::Array1; + use ndarray::{arr1, Array1}; /// Expected values hand-derived from the numba algorithm (verified by running /// the Python reference implementation with np.uint64 arithmetic). @@ -972,7 +1228,10 @@ mod tests { ); assert_eq!(result.len(), writable_length); for &v in &result { - assert_eq!(v, expected_val, "REPEAT_5P_NORM: expected {expected_val}, got {v}"); + assert_eq!( + v, expected_val, + "REPEAT_5P_NORM: expected {expected_val}, got {v}" + ); } // Sum preservation check let sum: f32 = result.iter().sum(); @@ -1090,7 +1349,10 @@ mod tests { hap, ); - assert_eq!(result, expected, "FLANK_SAMPLE: result did not match expected"); + assert_eq!( + result, expected, + "FLANK_SAMPLE: result did not match expected" + ); // Spot-check the first index by computing hash4 explicitly: // hash4(42, 7, 1, 0): @@ -1192,7 +1454,9 @@ mod tests { for a in 0..2usize { let mut term = ys[a]; for b in 0..2usize { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1217,9 +1481,15 @@ mod tests { assert_eq!(result.len(), writable_length); // Endpoint check: at i=0, result should equal ys[0]=track[v_rel_pos]=4.0 - assert_eq!(result[0], 4.0f32, "order=1 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 4.0f32, + "order=1 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=1 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=1 at i={i}: got {got}, expected {exp}" + ); } } @@ -1256,7 +1526,9 @@ mod tests { for a in 0..n { let mut term = ys[a]; for b in 0..n { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1280,9 +1552,15 @@ mod tests { ); // At x=0, result should equal ys[0] = track[v_rel_pos] = 4.0 - assert_eq!(result[0], 4.0f32, "order=2 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 4.0f32, + "order=2 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=2 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=2 at i={i}: got {got}, expected {exp}" + ); } } @@ -1318,7 +1596,9 @@ mod tests { for a in 0..n { let mut term = ys[a]; for b in 0..n { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1342,9 +1622,15 @@ mod tests { ); // At x=0, result should equal track[v_rel_pos]=5.0 - assert_eq!(result[0], 5.0f32, "order=3 left endpoint must equal track[v_rel_pos]"); + assert_eq!( + result[0], 5.0f32, + "order=3 left endpoint must equal track[v_rel_pos]" + ); for (i, (&got, &exp)) in result.iter().zip(expected.iter()).enumerate() { - assert_eq!(got, exp, "INTERPOLATE order=3 at i={i}: got {got}, expected {exp}"); + assert_eq!( + got, exp, + "INTERPOLATE order=3 at i={i}: got {got}, expected {exp}" + ); } } @@ -1390,7 +1676,9 @@ mod tests { for a in 0..2 { let mut term = ys[a]; for b in 0..2 { - if b == a { continue; } + if b == a { + continue; + } term *= (x - xs[b]) / (xs[a] - xs[b]); } acc += term; @@ -1498,7 +1786,11 @@ mod tests { 0, 0, ); - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "no variants: copy track[:length]"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "no variants: copy track[:length]" + ); } /// Deletion: track[v_rel_pos] repeated for writable_length; track advances by @@ -1519,13 +1811,13 @@ mod tests { let track = [10.0f32, 20.0, 30.0, 40.0, 50.0]; let v_starts = [1i32]; // v_start = 1 let ilens = [-2i32]; // deletion of 2 → v_rel_end = 1 - (-2) + 1 = 4... wait - // v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 - // Actually: v_rel_end = 1 - min(0, -2) + 1 = 1 - (-2) + 1 = 4 - // v_len = max(0, -2) + 1 = 0 + 1 = 1 - // track up to v_rel_pos=1: track[0..1] = [10.0], out[0] = 10.0 - // v_len=1 repeated: out[1] = track[1] = 20.0 - // track_idx = 4; remaining: track[4..5] = [50.0] → out[2] = 50.0 - // out[3] = 0.0 (trailing pad) + // v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 + // Actually: v_rel_end = 1 - min(0, -2) + 1 = 1 - (-2) + 1 = 4 + // v_len = max(0, -2) + 1 = 0 + 1 = 1 + // track up to v_rel_pos=1: track[0..1] = [10.0], out[0] = 10.0 + // v_len=1 repeated: out[1] = track[1] = 20.0 + // track_idx = 4; remaining: track[4..5] = [50.0] → out[2] = 50.0 + // out[3] = 0.0 (trailing pad) let geno_v_idxs = [0i32]; let geno_offsets = [0i64, 1]; @@ -1605,8 +1897,14 @@ mod tests { // out[0..4] from main loop; zero-pad covers out[4..8] from out_idx (not index 2). assert_eq!(result[0], 1.0f32, "ref[0]"); assert_eq!(result[1], 2.0f32, "ref[1]"); - assert_eq!(result[2], 3.0f32, "ref[2] — must NOT be overwritten by zero-pad"); - assert_eq!(result[3], 4.0f32, "deletion REPEAT_5P value — must NOT be overwritten"); + assert_eq!( + result[2], 3.0f32, + "ref[2] — must NOT be overwritten by zero-pad" + ); + assert_eq!( + result[3], 4.0f32, + "deletion REPEAT_5P value — must NOT be overwritten" + ); assert_eq!(result[4], 0.0f32, "zero-pad[4]"); assert_eq!(result[5], 0.0f32, "zero-pad[5]"); assert_eq!(result[6], 0.0f32, "zero-pad[6]"); @@ -1696,7 +1994,11 @@ mod tests { 0, ); // SNP is skipped — output equals track[:length] - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "SNP must be skipped for tracks"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "SNP must be skipped for tracks" + ); } /// Insertion with REPEAT_5P strategy: repeated track[v_rel_pos]. @@ -1813,7 +2115,11 @@ mod tests { 0, 0, ); - assert_eq!(result, [0.0f32, 1.0, 2.0, 3.0], "no variants + shift=0: copy track[:4]"); + assert_eq!( + result, + [0.0f32, 1.0, 2.0, 3.0], + "no variants + shift=0: copy track[:4]" + ); } /// Shift=2 with one insertion variant: verify shift-through-variant logic. @@ -1873,7 +2179,7 @@ mod tests { out_len: usize, out_offsets: &[i64], regions: &[[i32; 3]], - shifts: &[i32], // flat, will be reshaped (n_q, ploidy) + shifts: &[i32], // flat, will be reshaped (n_q, ploidy) geno_offset_idx: &[i64], // flat (n_q * ploidy) geno_v_idxs: &[i32], geno_offsets_1d: &[i64], @@ -1900,16 +2206,8 @@ mod tests { regions.iter().flat_map(|r| r.iter().cloned()).collect(), ) .unwrap(); - let shifts_arr = Array2::from_shape_vec( - (n_q, ploidy), - shifts.to_vec(), - ) - .unwrap(); - let goi_arr = Array2::from_shape_vec( - (n_q, ploidy), - geno_offset_idx.to_vec(), - ) - .unwrap(); + let shifts_arr = Array2::from_shape_vec((n_q, ploidy), shifts.to_vec()).unwrap(); + let goi_arr = Array2::from_shape_vec((n_q, ploidy), geno_offset_idx.to_vec()).unwrap(); let out_offsets_arr = Array1::from_vec(out_offsets.to_vec()); let gvi_arr = Array1::from_vec(geno_v_idxs.to_vec()); @@ -1991,7 +2289,11 @@ mod tests { 1, // ploidy false, ); - assert_eq!(result, [1.0f32, 2.0, 3.0, 4.0], "batch single: copy track[:4]"); + assert_eq!( + result, + [1.0f32, 2.0, 3.0, 4.0], + "batch single: copy track[:4]" + ); } /// Batch with 2 queries, 1 hap each, SNPs — must pass through unchanged. @@ -2031,9 +2333,17 @@ mod tests { false, ); // SNP skipped → query 0 output = track[0..3] - assert_eq!(result[..3], [1.0f32, 2.0, 3.0], "q0: SNP skipped, track copied"); + assert_eq!( + result[..3], + [1.0f32, 2.0, 3.0], + "q0: SNP skipped, track copied" + ); // No variants in q1 → track[3..6] - assert_eq!(result[3..], [4.0f32, 5.0, 6.0], "q1: no variants, track copied"); + assert_eq!( + result[3..], + [4.0f32, 5.0, 6.0], + "q1: no variants, track copied" + ); } // ================================================================== // @@ -2054,7 +2364,7 @@ mod tests { // regions: (n_queries, 3) — (contig_idx, start, end) let regions_data = vec![ - 0i32, 0, 0, // q0: empty length + 0i32, 0, 0, // q0: empty length 0i32, 10, 13, // q1: [10, 13), length 3 0i32, 20, 25, // q2: [20, 25), length 5 ]; @@ -2071,7 +2381,11 @@ mod tests { tracks_to_intervals(regions.view(), tracks.view(), track_offsets.view(), false); // offsets: [0, 0, 1, 3] - assert_eq!(offsets.as_slice().unwrap(), &[0i64, 0, 1, 3], "offsets mismatch"); + assert_eq!( + offsets.as_slice().unwrap(), + &[0i64, 0, 1, 3], + "offsets mismatch" + ); // Total intervals = 3 assert_eq!(starts.len(), 3); @@ -2148,7 +2462,11 @@ mod tests { tracks_to_intervals(regions.view(), tracks.view(), track_offsets.view(), false); assert_eq!(offsets.as_slice().unwrap(), &[0i64, 3]); - assert_eq!(starts.len(), 3, "must have 3 intervals including zero-value ones"); + assert_eq!( + starts.len(), + 3, + "must have 3 intervals including zero-value ones" + ); assert_eq!(values[0], 0.0f32, "first interval is zero-value"); assert_eq!(starts[0], 0i32); assert_eq!(ends[0], 2i32); @@ -2157,4 +2475,93 @@ mod tests { assert_eq!(starts[2], 3i32); assert_eq!(ends[2], 4i32); } + + // ------------------------------------------------------------------ // + // shift_and_realign_tracks_from_svar2 // + // ------------------------------------------------------------------ // + + /// SVAR2 two-source track driver: a single pure DEL, carried entirely in the + /// `dense` channel (no `var_key` entries), realigns a track identically to the + /// SVAR1 kernel's `test_singular_deletion` case above (same track, v_start, ilen, + /// strategy_id=REPEAT_5P — any strategy_id works for a DEL, since v_diff<0 always + /// takes the `else` branch and repeats `track[v_rel_pos]`, bypassing + /// `apply_insertion_fill` entirely; REPEAT_5P is reused here only to match that + /// existing test's derivation for an easy cross-check). + /// + /// Setup: R=1, S=1 (ploidy=1) -> n_work=1. + /// track = [10.0, 20.0, 30.0, 40.0, 50.0], query_start = 0, out_len = 4 + /// dense pure DEL at pos=1, ilen=-2 — `svar2_codec::encode_pure_del(x)` round-trips + /// through `decode_alt` to `v_diff = x` (see `DecodedKey::PureDel { ilen }`), so + /// `encode_pure_del(-2)` below yields `v_diff = -2` to match the derivation. + /// + /// Hand-derivation (identical math to `test_singular_deletion`): + /// v_rel_pos = 1 - 0 = 1; v_diff = -2 + /// v_rel_end = v_rel_pos - min(0, v_diff) + 1 = 1 - (-2) + 1 = 4 + /// v_len = max(0, -2) + 1 = 1 + /// track_len = v_rel_pos - track_idx = 1 - 0 = 1 -> out[0] = track[0] = 10.0 + /// writable_length = min(1, 4-1) = 1; v_diff<0 -> else branch: + /// out[1] = track[v_rel_pos=1] = 20.0 + /// track_idx = v_rel_end = 4; out_idx = 2 + /// tail: unfilled_length = 4-2 = 2; writable_ref = min(2, track.len()(5)-4) = 1 + /// -> out[2] = track[4] = 50.0; out_end_idx = 3 + /// out[3] = 0.0 (trailing pad, out_end_idx(3) < length(4)) + /// expected out = [10.0, 20.0, 50.0, 0.0] + #[test] + fn svar2_track_realign_del() { + let track = arr1(&[10.0f32, 20.0, 30.0, 40.0, 50.0]); + let track_offsets = arr1(&[0i64, 5]); + + // One region on contig 0 (unused by tracks beyond regions[[q,1]]=query_start), + // one sample-slot, one ploid -> n_work = 1. + let regions = ndarray::arr2(&[[0i32, 0, 4]]); + let shifts = ndarray::arr2(&[[0i32]]); + + // No var_key entries for this hap. + let vk_pos = arr1::(&[]); + let vk_key = arr1::(&[]); + let vk_off = arr1(&[0i64, 0]); + + // dense channel: one pure DEL (ilen=-2) at pos=1, present for this hap. + let del_key = svar2_codec::encode_pure_del(-2) as i32; + let dense_pos = arr1(&[1i32]); + let dense_key = arr1(&[del_key]); + let dense_range = ndarray::arr2(&[[0i32, 1]]); + // Presence bits, LSB-first: bit 0 (this hap's only dense entry) is set. + let dense_present = arr1(&[0b0000_0001u8]); + let dense_present_off = arr1(&[0i64, 1]); + + // No long-allele-bank lookups exercised in this test. + let lut_bytes = arr1::(&[]); + let lut_off = arr1(&[0i64]); + + let out_offsets = arr1(&[0i64, 4]); + let params = arr1(&[0.0f64]); + let mut out = Array1::::zeros(4); + + shift_and_realign_tracks_from_svar2( + out.view_mut(), + out_offsets.view(), + regions.view(), + shifts.view(), + vk_pos.view(), + vk_key.view(), + vk_off.view(), + dense_pos.view(), + dense_key.view(), + dense_range.view(), + dense_present.view(), + dense_present_off.view(), + lut_bytes.view(), + lut_off.view(), + track.view(), + track_offsets.view(), + params.view(), + REPEAT_5P, + 0, + None, // query_seed: single call, local index is the global row + false, // serial + ); + + assert_eq!(out.as_slice().unwrap(), &[10.0f32, 20.0, 50.0, 0.0]); + } } diff --git a/tests/_builders/case.py b/tests/_builders/case.py index 3298ee1e..5b96f6ee 100644 --- a/tests/_builders/case.py +++ b/tests/_builders/case.py @@ -14,7 +14,7 @@ import numpy as np import polars as pl -from vcfixture import Number, ReferenceBuilder, Type, VcfBuilder +from vcfixture import Number, ReferenceBuilder, Seq, Type, VcfBuilder, VcfVersion SEQ_LEN = 20 @@ -317,11 +317,11 @@ def session_reference(): def session_document(spec): """Fixed standardized source document (relabel of the Phase-1 source VCF). - Replaces Phase-1 ``_synthetic.build_source_vcf``. VCFv4.0 is used so the + Replaces Phase-1 ``_synthetic.build_source_vcf``. VCFv4.1 is used so the Number=. INFO fields parse under the noodles VCF reader SparseVar relies on. """ contigs = list(_SESSION_CONTIGS) - b = VcfBuilder(samples=["s0", "s1", "s2"], contigs=contigs, fileformat="VCFv4.0") + b = VcfBuilder(samples=["s0", "s1", "s2"], contigs=contigs, version=VcfVersion.V4_1) b.info("NS", Number.ONE, Type.INTEGER) b.info("AN", Number.ONE, Type.INTEGER) b.info("AC", Number.DOT, Type.INTEGER) @@ -339,23 +339,25 @@ def session_document(spec): b.filter("s50", "Less than 50% of samples have data") # chr1 block (relabeled from chr19). - b.record("chr1", 111, ref="N", alt=["C"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1010696, ref="GAGA", alt=["G"], gt=["1|0", "0|0", "0/0"]) - b.record("chr1", 1010696, ref="GAGACGG", alt=["G"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1010696, ref="GAGACGGGGCC", alt=["G"], gt=["0|1", "1|1", "0/0"]) - b.record("chr1", 1110696, ref="A", alt=["TTT"], gt=["0|1", "1|1", "0/0"]) - b.record("chr1", 1110696, ref="A", alt=["G"], gt=["0|0", "0|0", "0/1"]) - b.record("chr1", 1210696, ref="C", alt=["G"], gt=["1|.", "0/1", "1|1"]) - b.record("chr1", 1210696, ref="C", alt=["G"], gt=[".|1", "0|0", "0/0"]) - b.record("chr1", 1210697, ref="T", alt=["G"], gt=["0/0", "1|0", "0/1"]) - b.record("chr1", 1210697, ref="T", alt=["A"], gt=["0/0", "1|0", "0/1"]) + b.record("chr1", 111, ref="N", alt=[Seq("C")], gt=["0|0", "0|0", "0/1"]) + b.record("chr1", 1010696, ref="GAGA", alt=[Seq("G")], gt=["1|0", "0|0", "0/0"]) + b.record("chr1", 1010696, ref="GAGACGG", alt=[Seq("G")], gt=["0|0", "0|0", "0/1"]) + b.record( + "chr1", 1010696, ref="GAGACGGGGCC", alt=[Seq("G")], gt=["0|1", "1|1", "0/0"] + ) + b.record("chr1", 1110696, ref="A", alt=[Seq("TTT")], gt=["0|1", "1|1", "0/0"]) + b.record("chr1", 1110696, ref="A", alt=[Seq("G")], gt=["0|0", "0|0", "0/1"]) + b.record("chr1", 1210696, ref="C", alt=[Seq("G")], gt=["1|.", "0/1", "1|1"]) + b.record("chr1", 1210696, ref="C", alt=[Seq("G")], gt=[".|1", "0|0", "0/0"]) + b.record("chr1", 1210697, ref="T", alt=[Seq("G")], gt=["0/0", "1|0", "0/1"]) + b.record("chr1", 1210697, ref="T", alt=[Seq("A")], gt=["0/0", "1|0", "0/1"]) # chr2 block (relabeled from chr20) — carries INFO/IDs/FILTERs. b.record( "chr2", 14370, ref="N", - alt=["A"], + alt=[Seq("A")], ids=["rs6054257"], qual=29.0, filter=(), @@ -366,7 +368,7 @@ def session_document(spec): "chr2", 17330, ref="N", - alt=["A"], + alt=[Seq("A")], qual=3.0, filter=["q10"], gt=["0|0", "0|1", "0/0"], @@ -376,7 +378,7 @@ def session_document(spec): "chr2", 1110696, ref="G", - alt=["A", "T"], + alt=[Seq("A"), Seq("T")], ids=["rs6040355"], qual=67.0, filter=(), @@ -387,7 +389,7 @@ def session_document(spec): "chr2", 1234567, ref="A", - alt=["GA", "AC"], + alt=[Seq("GA"), Seq("AC")], ids=["microsat1"], qual=50.0, filter=(), diff --git a/tests/_builders/test_case.py b/tests/_builders/test_case.py index 595de679..57475679 100644 --- a/tests/_builders/test_case.py +++ b/tests/_builders/test_case.py @@ -6,7 +6,7 @@ import genvarloader as gvl import pysam -from vcfixture import ReferenceBuilder, VcfBuilder +from vcfixture import ReferenceBuilder, Seq, VcfBuilder, VcfVersion def _tiny_spec_and_doc(): @@ -17,9 +17,11 @@ def _tiny_spec_and_doc(): .set_base("chr1", 99, "A") # 0-based; REF for the record below .build() ) - b = VcfBuilder(samples=["s0", "s1"], contigs=[("chr1", 200)], fileformat="VCFv4.0") + b = VcfBuilder( + samples=["s0", "s1"], contigs=[("chr1", 200)], version=VcfVersion.V4_1 + ) b.fmt("GT") - b.record("chr1", 100, ref="A", alt=["C"], gt=["0|1", "1|1"]) # 1-based pos + b.record("chr1", 100, ref="A", alt=[Seq("C")], gt=["0|1", "1|1"]) # 1-based pos return ref, b.build() diff --git a/tests/_oracles/__init__.py b/tests/_oracles/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/_oracles/svar2_readbound_inputs.py b/tests/_oracles/svar2_readbound_inputs.py new file mode 100644 index 00000000..67774858 --- /dev/null +++ b/tests/_oracles/svar2_readbound_inputs.py @@ -0,0 +1,356 @@ +"""Read-bound SVAR2 haplotype reconstruction: one all-Rust FFI call gathering off a +query-only genoray ``Svar2Store`` (``genoray_core::query::gather_haps_readbound``), with +NO interval-search-tree rebuild and NO dense-union rebuild. + +Byte-identical to the existing union-path oracle (``SparseVar2Source.reconstruct``, +``svar2_source.py``), which calls ``reconstruct_haplotypes_from_svar2`` over +``SparseVar2._overlap_batch``'s eagerly-unioned dense channel. This module instead +marshals ``SparseVar2._find_ranges``'s per-class-split ranges through +``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` (Rust side) +and reuses that same validated kernel — see ``reconstruct_haplotypes_from_svar2_readbound`` +in ``src/ffi/mod.rs``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import numpy as np + +from genvarloader._flat import _Flat +from genvarloader.genvarloader import ( + Svar2Store, + decode_variants_from_svar2_readbound, + hap_diffs_from_svar2_readbound, + reconstruct_haplotypes_from_svar2_readbound, + shift_and_realign_tracks_from_svar2_readbound, +) + +if TYPE_CHECKING: + from genoray import SparseVar2 + from numpy.typing import NDArray + from seqpro.rag import Ragged + + from genvarloader._dataset._rag_variants import RaggedVariants + + +def build_readbound_haps( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R + ref_: "NDArray[np.uint8]", # the contig reference bytes + ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) + pad_char: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + output_length: int = -1, + parallel: bool = False, +) -> "Ragged[np.bytes_]": + """Reconstruct the full-cohort haplotypes over ``regions`` via the read-bound kernel. + + Mirrors ``SparseVar2Source.reconstruct``'s signature/return shape exactly (query + order region-major, sample-minor: ``q = r*S + s``), but drives + ``SparseVar2._find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``_overlap_batch``'s eager per-region dense union. + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2._find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + out_data, out_offsets = reconstruct_haplotypes_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + np.ascontiguousarray(ref_, np.uint8), + np.ascontiguousarray(ref_offsets, np.int64), + np.uint8(pad_char), + np.int64(output_length), + parallel, + ) + + shape = (R, S, P, None) + return cast( + "Ragged[np.bytes_]", _Flat.from_offsets(out_data, shape, out_offsets).view("S1") + ) + + +def build_readbound_diffs( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R +) -> "NDArray[np.int32]": + """Per-hap ilen diffs over ``regions`` via the read-bound kernel, WITHOUT + reconstructing haplotypes. + + Mirrors ``build_readbound_haps``'s query order (region-major, sample-minor: + ``q = r*S + s``) and gather-input construction, but stops after + ``svar2::hap_diffs_svar2`` — no ``ref``/``pad_char``/``shifts``/reconstruct + pass. Used by the dataset read path to compute random jitter shifts from + diffs BEFORE reconstructing (mirrors how the SVAR1 path derives shifts from + diffs in ``_prepare_request``). + + Returns the ``(R*S, P)`` diffs array (query order region-major, sample-minor). + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2._find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + diffs = hap_diffs_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + P, + ) + + return cast("NDArray[np.int32]", diffs) + + +def build_readbound_tracks( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R + tracks: "NDArray[np.float32]", # flat per-REGION track buffer + track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks + params: "NDArray[np.float64]", + strategy_id: int, + base_seed: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + parallel: bool = False, +) -> "Ragged[np.float32]": + """Realign the full-cohort tracks over ``regions`` via the read-bound kernel. + + Mirrors ``SparseVar2Source.realign_tracks``'s signature/return shape exactly + (query order region-major, sample-minor: ``q = r*S + s``), but drives + ``SparseVar2._find_ranges`` (search-only, no dense union) + one Rust FFI call + instead of ``_overlap_batch``'s eager per-region dense union. + """ + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2._find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + reg_arr = np.asarray(reg, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray( + np.repeat(reg_arr, S, axis=0), np.int32 + ) # (n_q, 2) + + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + + # `tracks`/`track_offsets` are per-REGION (R of them), but the kernel reads + # `track_offsets` by `query` (= r*S+s) — expand the R track windows to R*S + # by repeating each region's window S times (mirrors + # `SparseVar2Source.realign_tracks`, `tests/_oracles/svar2_source.py`). + t = np.asarray(tracks, np.float32) + toff = np.asarray(track_offsets, np.int64) + tracks_rs = ( + np.concatenate([t[toff[r] : toff[r + 1]] for r in range(R) for _ in range(S)]) + if R + else t + ) + lengths = np.repeat(np.diff(toff), S) + track_offsets_rs = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + out_data, out_offsets = shift_and_realign_tracks_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + np.ascontiguousarray(tracks_rs, np.float32), + track_offsets_rs, + np.ascontiguousarray(params, np.float64), + np.int64(strategy_id), + np.uint64(base_seed), + # Single FFI call over the whole block: local query == global row (#267). + np.arange(n_q, dtype=np.int64), + parallel, + ) + + shape = (R, S, P, None) + return cast("Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets)) + + +def build_readbound_variants( + svar2: "SparseVar2", + contig: str, + regions, # iterable of (start, end), length R +) -> "RaggedVariants": + """Decode the full-cohort overlapping variants over ``regions`` via the + read-bound kernel. + + Mirrors ``SparseVar2.decode``'s return shape exactly (region-major, + sample-minor: ``q = r*S + s``), but drives ``SparseVar2._find_ranges`` + (search-only, no dense union) + one Rust FFI call instead of + ``decode_batch``'s eager per-region dense union. Unlike + ``build_readbound_haps``/``build_readbound_tracks`` there is no reconstruct + sizing pass and no ``shifts``/``ref``/``pad_char`` — decoding a hap's merged + variant set has no output-length dependency on the query region bounds (no + overlap/clip filter; the gather already restricts to overlapping variants). + """ + from genvarloader._dataset._rag_variants import RaggedVariants + + reg = [(int(s), int(e)) for s, e in regions] + R = len(reg) + S = svar2.n_samples + P = svar2.ploidy + + d = svar2._find_ranges( + contig, [s for s, _ in reg], [e for _, e in reg], samples=None + ) + + region_starts_r = np.asarray(d["region_starts"], np.int64) # (R,) + sample_cols = np.asarray(d["sample_cols"], np.int64) # (S,) + # vk_*_range rows are already (R, S, P) row-major == query-major (q = r*S+s, + # row = q*P + p), so they pass through unchanged. + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) # (R, 2) + + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) # (n_q,) + orig_samples = np.tile(sample_cols, R) # (n_q,) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) # (n_q, 2) + + store = Svar2Store(str(svar2.path), svar2.contigs, svar2.n_samples, svar2.ploidy) + + pos, ilen, alt_bytes, str_off, var_off, _field_bufs, _field_itemsizes = ( + decode_variants_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + P, + [], + ) + ) + + from seqpro.rag import Ragged + + shape = (R, S, P, None) + pos_r = Ragged.from_offsets(pos, shape, var_off) + ilen_r = Ragged.from_offsets(ilen, shape, var_off) + alt_r = Ragged.from_offsets( + alt_bytes.view("S1"), shape, var_off, str_offsets=str_off + ) + + return RaggedVariants(alt=alt_r, start=pos_r, ilen=ilen_r) diff --git a/tests/_oracles/svar2_source.py b/tests/_oracles/svar2_source.py new file mode 100644 index 00000000..d86f6d2f --- /dev/null +++ b/tests/_oracles/svar2_source.py @@ -0,0 +1,151 @@ +"""SVAR2 two-source reconstruction adapter — parity oracle only (not a live read path). + +Bridges genoray ``SparseVar2._overlap_batch``'s raw two-channel dict to gvl's SVAR2 kernels +(``reconstruct_haplotypes_from_svar2`` / ``shift_and_realign_tracks_from_svar2``), decoding +``var_key ⋈ dense`` inline with no intermediate variant table. This is the *union* path +(genoray ``_overlap_batch``, whole-cohort). + +Live dataset dispatch is NOT wired through here. ``Dataset`` reconstruction for ``.svar2``-backed +datasets is handled by the read-bound path in ``Svar2Haps`` (``_svar2_haps.py``), which gathers off +the write-time ranges cache and calls the ``*_from_svar2_readbound`` kernels — no interval-search-tree +rebuild and no dense-union rebuild per read. This ``SparseVar2Source`` adapter is retained solely as +the byte-identical *parity oracle* the read-bound kernels are tested against (see +``tests/dataset/test_svar2_readbound_*.py``); it is not imported on any live read path. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import numpy as np +from seqpro.rag import Ragged + +from genvarloader._flat import _Flat +from genvarloader.genvarloader import ( + reconstruct_haplotypes_from_svar2, + shift_and_realign_tracks_from_svar2, +) + +if TYPE_CHECKING: + from genoray import SparseVar2 + from numpy.typing import NDArray + + +class SparseVar2Source: + """Reconstruct haplotypes / realign tracks from a genoray ``SparseVar2`` via the two-source path.""" + + def __init__(self, svar2: "SparseVar2") -> None: + self.svar2 = svar2 + + def _query(self, contig, regions): + d = self.svar2._overlap_batch(contig, [(int(s), int(e)) for s, e in regions]) + R = int(d["n_regions"]) + S = int(d["n_samples"]) + P = int(d["ploidy"]) + reg = np.asarray(regions, dtype=np.int32).reshape(R, 2) + # (R*S, 3): contig_idx=0, start, end — repeat each query region S times. + reg_rs = np.repeat(reg, S, axis=0) # (R*S, 2) + regions_gvl = np.zeros((R * S, 3), dtype=np.int32) + regions_gvl[:, 1:] = reg_rs + dense_range_gvl = np.ascontiguousarray( + np.repeat(np.asarray(d["dense_range"], np.int32), S, axis=0), np.int32 + ) # (R*S, 2) + return d, R, S, P, regions_gvl, dense_range_gvl + + def reconstruct( + self, + contig: str, + regions, # iterable of (start, end), length R + ref_: "NDArray[np.uint8]", # the contig reference bytes + ref_offsets: "NDArray[np.int64]", # e.g. np.array([0, len(ref_)]) + pad_char: int, + shifts: "NDArray[np.int32] | None" = None, # (R*S, P); None -> zeros + output_length: int = -1, + parallel: bool = False, + ) -> "Ragged[np.bytes_]": + d, R, S, P, regions_gvl, dense_range_gvl = self._query(contig, regions) + n_q = R * S + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + out_data, out_offsets = reconstruct_haplotypes_from_svar2( + np.ascontiguousarray(regions_gvl, np.int32), + shifts_a, + np.ascontiguousarray(d["vk_pos"], np.int32), + np.ascontiguousarray(d["vk_key"], np.int32), + np.ascontiguousarray(d["vk_off"], np.int64), + np.ascontiguousarray(d["dense_pos"], np.int32), + np.ascontiguousarray(d["dense_key"], np.int32), + dense_range_gvl, + np.ascontiguousarray(d["dense_present"], np.uint8), + np.ascontiguousarray(d["dense_present_off"], np.int64), + np.ascontiguousarray(d["lut_bytes"], np.uint8), + np.ascontiguousarray(d["lut_off"], np.int64), + np.ascontiguousarray(ref_, np.uint8), + np.ascontiguousarray(ref_offsets, np.int64), + np.uint8(pad_char), + np.int64(output_length), + parallel, + ) + shape = (R, S, P, None) + return cast( + "Ragged[np.bytes_]", + _Flat.from_offsets(out_data, shape, out_offsets).view("S1"), + ) + + def realign_tracks( + self, + contig: str, + regions, + tracks: "NDArray[np.float32]", # flat per-query track buffer + track_offsets: "NDArray[np.int64]", # (R+1) offsets into tracks + params: "NDArray[np.float64]", + strategy_id: int, + base_seed: int, + shifts: "NDArray[np.int32] | None" = None, + parallel: bool = False, + ) -> "Ragged[np.float32]": + d, R, S, P, regions_gvl, dense_range_gvl = self._query(contig, regions) + n_q = R * S + if shifts is None: + shifts_a = np.zeros((n_q, P), dtype=np.int32) + else: + shifts_a = np.ascontiguousarray(shifts, np.int32).reshape(n_q, P) + # tracks are per query REGION (R of them); the driver reads track_offsets by `query` + # (= r*S+s), so expand the R track windows to R*S by repeating each S times. + t = np.asarray(tracks, np.float32) + toff = np.asarray(track_offsets, np.int64) + tracks_rs = ( + np.concatenate( + [t[toff[r] : toff[r + 1]] for r in range(R) for _ in range(S)] + ) + if R + else t + ) + lengths = np.repeat(np.diff(toff), S) + track_offsets_rs = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + out_data, out_offsets = shift_and_realign_tracks_from_svar2( + np.ascontiguousarray(regions_gvl, np.int32), + shifts_a, + np.ascontiguousarray(d["vk_pos"], np.int32), + np.ascontiguousarray(d["vk_key"], np.int32), + np.ascontiguousarray(d["vk_off"], np.int64), + np.ascontiguousarray(d["dense_pos"], np.int32), + np.ascontiguousarray(d["dense_key"], np.int32), + dense_range_gvl, + np.ascontiguousarray(d["dense_present"], np.uint8), + np.ascontiguousarray(d["dense_present_off"], np.int64), + np.ascontiguousarray(d["lut_bytes"], np.uint8), + np.ascontiguousarray(d["lut_off"], np.int64), + np.ascontiguousarray(tracks_rs, np.float32), + track_offsets_rs, + np.ascontiguousarray(params, np.float64), + np.int64(strategy_id), + np.uint64(base_seed), + parallel, + ) + shape = (R, S, P, None) + return cast( + "Ragged[np.float32]", _Flat.from_offsets(out_data, shape, out_offsets) + ) diff --git a/tests/dataset/test_svar2_dataset.py b/tests/dataset/test_svar2_dataset.py new file mode 100644 index 00000000..5ceaa724 --- /dev/null +++ b/tests/dataset/test_svar2_dataset.py @@ -0,0 +1,920 @@ +"""End-to-end SVAR2 dataset read dispatch parity (Task 7b). + +Builds two gvl datasets over the same bed/samples/reference from matched stores +built from the SAME VCF -- one ``.svar`` (SVAR1) and one ``.svar2`` -- and asserts +the SVAR2 read path (``Svar2Haps``) is byte-identical to the SVAR1 path for +``with_seqs('haplotypes')`` and ``with_seqs('variants')``. + +Parity is exact because both sides open with ``deterministic=True`` (shifts=0) +and the datasets are written with ``max_jitter=0``, so no RNG is involved. The +fixture VCF is tie-free (no same-POS SNP+DEL) so the SVAR1 max_ends tie bug +(docs/known-issues/svar1-max-ends-tie-underextension.md) is not exercised. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import polars as pl +import pytest + +import genvarloader as gvl +from genvarloader import VarWindowOpt + +_WIN_OPT = VarWindowOpt( + flank_length=3, token_alphabet=b"ACGT", unknown_token=4, ref="window", alt="window" +) + + +def _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref, opt=_WIN_OPT): + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", opt)[:, :] + return w1, w2 + + +def _assert_window_equal(a, b, name: str) -> None: + """Flat-buffer equality of two _FlatWindow fields (data + both offset levels).""" + assert np.array_equal(np.asarray(a.var_offsets), np.asarray(b.var_offsets)), ( + f"{name} var_offsets differ" + ) + assert np.array_equal(np.asarray(a.seq_offsets), np.asarray(b.seq_offsets)), ( + f"{name} seq_offsets differ" + ) + assert np.array_equal(np.asarray(a.data), np.asarray(b.data)), f"{name} data differ" + + +def test_svar2_variant_windows_ref_window_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """ref_window is a pure reference read over an identical variant SET, so it is + byte-identical to SVAR1 (independent of the deletion-ALT encoding difference).""" + _bcf, ref = _src + w1, w2 = _open_windows_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + assert w2.ref_window is not None + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # scalar start field also identical (same variant SET) — compare _Flat buffers. + assert np.array_equal( + np.asarray(w2.fields["start"].data), np.asarray(w1.fields["start"].data) + ) + assert np.array_equal( + np.asarray(w2.fields["start"].offsets), np.asarray(w1.fields["start"].offsets) + ) + + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 +# (C>CAT), dense SNP@9 (G>C, carried by 3 haps -> dense/snp channel), DEL@11 +# (GTA>G, ilen -2). No same-POS ties. Mirrors the readbound-haps dense-SNP +# fixture so both var_key and dense channels are exercised. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def _src(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_ds_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar_fixture(_src, tmp_path_factory) -> Path: + bcf, _ref = _src + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1") / "store.svar" + SparseVar.from_vcf( + out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return out + + +@pytest.fixture(scope="module") +def svar2_fixture(_src, tmp_path_factory) -> Path: + bcf, ref = _src + from genoray import _core + + out = tmp_path_factory.mktemp("svar2") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +@pytest.fixture(scope="module") +def bed() -> pl.DataFrame: + # Tie-free windows spanning the SNP/INS/dense-SNP/DEL and a variant-free tail. + return pl.DataFrame( + { + "chrom": ["chr1"] * 4, + "chromStart": [0, 0, 5, 20], + "chromEnd": [40, 15, 20, 40], + } + ) + + +def _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref): + from genoray import SparseVar, SparseVar2 + + d1 = tmp_path / "d1.gvl" + d2 = tmp_path / "d2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture), samples=None, overwrite=True) + gvl.write(d2, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + return ( + gvl.Dataset.open(d1, reference=ref), + gvl.Dataset.open(d2, reference=ref), + ) + + +def test_svar2_haplotypes_match_svar1(tmp_path, bed, svar_fixture, svar2_fixture, _src): + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)), ( + f"offsets differ: svar1={np.asarray(a.offsets).tolist()} " + f"svar2={np.asarray(b.offsets).tolist()}" + ) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +def _make_bigwig(path: Path, contig: str, length: int, seed: int) -> None: + """Write a dense per-bp BigWig over ``contig`` (deterministic given ``seed``).""" + import pyBigWig + + rng = np.random.default_rng(seed) + starts = list(range(length)) + ends = list(range(1, length + 1)) + values = [float(v) for v in rng.standard_normal(length).astype(np.float32)] + with pyBigWig.open(str(path), "w") as bw: + bw.addHeader([(contig, length)], maxZooms=0) + bw.addEntries([contig] * length, starts, ends=ends, values=values) + + +@pytest.fixture(scope="module") +def bigwig_fixture(tmp_path_factory): + """Per-sample BigWigs over chr1 (len 40) -> a SAMPLE-indexed gvl.BigWigs track.""" + bw_dir = tmp_path_factory.mktemp("svar2_bw") + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + _make_bigwig(p, "chr1", 40, seed=100 + i) + paths[s] = str(p) + return gvl.BigWigs("signal", paths) + + +def test_svar2_tracks_match_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, bigwig_fixture, _src +): + """Haplotype-realigned tracks byte-identical (f32, NaN-equal) to SVAR1. + + Both datasets are written from the SAME VCF + SAME BigWig track; at read the + SVAR1 backend realigns via the fused kernel and the SVAR2 backend + (``Svar2Haps`` + ``HapsTracks._call_svar2``) realigns via the split + ``intervals_to_tracks`` + ``shift_and_realign_tracks_from_svar2_readbound`` + path. deterministic=True + max_jitter=0 => shifts=0, so parity is exact. + """ + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src + d1 = tmp_path / "t1.gvl" + d2 = tmp_path / "t2.gvl" + gvl.write( + d1, + bed, + variants=SparseVar(svar_fixture), + tracks=bigwig_fixture, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, + bed, + variants=SparseVar2(svar2_fixture), + tracks=bigwig_fixture, + samples=None, + max_jitter=0, + overwrite=True, + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + # HapsTracks (seqs active by default) -> (haps, tracks) tuple. + _h1, a = ds1.with_tracks("signal")[:, :] + _h2, b = ds2.with_tracks("signal")[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "track data differ" + + +def test_svar2_tracks_match_svar1_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """Realigned tracks byte-identical to SVAR1 across a TWO-contig, out-of-order + bed -- exercises ``_call_svar2``'s contig-group split + inverse row-perm + stitching for the track path (single-contig fast path bypassed).""" + import pyBigWig + + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bw_dir = tmp_path / "bw_mc" + bw_dir.mkdir() + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + rng = np.random.default_rng(200 + i) + with pyBigWig.open(str(p), "w") as bw: + bw.addHeader([("chr1", 40), ("chr2", 40)], maxZooms=0) + for contig in ("chr1", "chr2"): + vals = [float(v) for v in rng.standard_normal(40).astype(np.float32)] + bw.addEntries( + [contig] * 40, list(range(40)), ends=list(range(1, 41)), values=vals + ) + paths[s] = str(p) + track = gvl.BigWigs("signal", paths) + + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "tmc1.gvl" + d2 = tmp_path / "tmc2.gvl" + gvl.write( + d1, + bed, + variants=SparseVar(svar_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, + bed, + variants=SparseVar2(svar2_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + _h1, a = ds1.with_tracks("signal")[:, :] + _h2, b = ds2.with_tracks("signal")[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "track data differ" + + +def test_svar2_flanksample_multicontig_matches_svar1( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """FlankSample (seed-dependent fill) + MULTI-contig is byte-identical to SVAR1. + + ``_call_svar2`` realigns per contig group, so ``k / ploidy`` is a contig-LOCAL + query index; SVAR1 realigns the whole batch in one fused call and seeds the + FlankSample fill hash with the GLOBAL row. Issue #267 makes the read-bound + path pass each group's global row indices (``global_query``) into the FFI so + the kernel seeds with the global row too. Without the fix the chr1 group's + queries (which land at global rows 1, 3 in the interleaved bed) would seed off + local indices 0, 1 and diverge in every inserted region. + """ + import pyBigWig + + from genoray import SparseVar, SparseVar2 + + from genvarloader._dataset._insertion_fill import FlankSample + + _bcf, ref = _src2 + bw_dir = tmp_path / "bw_fs" + bw_dir.mkdir() + paths = {} + for i, s in enumerate(["S0", "S1"]): + p = bw_dir / f"{s}.bw" + rng = np.random.default_rng(300 + i) + with pyBigWig.open(str(p), "w") as bw: + bw.addHeader([("chr1", 40), ("chr2", 40)], maxZooms=0) + for contig in ("chr1", "chr2"): + vals = [float(v) for v in rng.standard_normal(40).astype(np.float32)] + bw.addEntries( + [contig] * 40, list(range(40)), ends=list(range(1, 41)), values=vals + ) + paths[s] = str(p) + track = gvl.BigWigs("signal", paths) + + # Interleaved chr2/chr1 bed -> >1 contig group, with each contig appearing at + # multiple GLOBAL rows so local != global for the fill seed. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "fs1.gvl" + d2 = tmp_path / "fs2.gvl" + gvl.write( + d1, + bed, + variants=SparseVar(svar_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + gvl.write( + d2, + bed, + variants=SparseVar2(svar2_fixture2), + tracks=track, + samples=None, + max_jitter=0, + overwrite=True, + ) + fill = {"signal": FlankSample(flank_width=3)} + ds1 = ( + gvl.Dataset.open(d1, reference=ref) + .with_tracks("signal") + .with_insertion_fill(fill) + ) + ds2 = ( + gvl.Dataset.open(d2, reference=ref) + .with_tracks("signal") + .with_insertion_fill(fill) + ) + + _h1, a = ds1[:, :] + _h2, b = ds2[:, :] + + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"track offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad, bd = np.asarray(a.data, np.float32), np.asarray(b.data, np.float32) + assert np.allclose(ad, bd, equal_nan=True), "FlankSample track data differ" + + +def _assert_ragged_equal(a, b, name: str) -> None: + ao, bo = np.asarray(a.offsets), np.asarray(b.offsets) + assert np.array_equal(ao, bo), ( + f"{name} offsets differ: svar1={ao.tolist()} svar2={bo.tolist()}" + ) + ad = np.asarray(a.data).view("u1") + bd = np.asarray(b.data).view("u1") + assert np.array_equal(ad, bd), f"{name} data differ" + + +def test_svar2_variants_positions_match_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """The decoded variant SET (positions + ilens) is byte-identical to SVAR1. + + NOTE: the ALT allele *bytes* are intentionally NOT compared to SVAR1 here. + The two genoray formats encode a deletion's ALT differently -- SVAR1 keeps + the VCF anchor base (e.g. ``G`` for ``GTA>G``) while SVAR2 decodes the + atomized empty ALT (``""``). Haplotype reconstruction is unaffected (see + ``test_svar2_haplotypes_match_svar1``), and the ALT bytes are validated + against the SVAR2 decode oracle in ``test_svar2_variants_match_svar2_oracle``. + """ + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants")[:, :] + b = ds2.with_seqs("variants")[:, :] + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") + + +def test_svar2_variants_unphased_union_matches_svar1( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Ploidy-1 union: start/ilen byte-identical to SVAR1 union (order-preserving + fold, no dedup). ALT differs by encoding, so ALT is not compared to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + a = ds1.with_seqs("variants").with_settings(unphased_union=True)[:, :] + b = ds2.with_seqs("variants").with_settings(unphased_union=True)[:, :] + # Ploidy axis folded 2 -> 1. + assert a.start.shape[-2] == 1 and b.start.shape[-2] == 1 + _assert_ragged_equal(a.start.to_packed(), b.start.to_packed(), "start") + _assert_ragged_equal(a.ilen.to_packed(), b.ilen.to_packed(), "ilen") + + +def test_svar2_variants_match_svar2_oracle( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Full RaggedVariants (start/ilen/alt) match the validated SVAR2 decode oracle. + + ``build_readbound_variants`` (parity-tested against genoray's ``SparseVar2.decode`` + in ``test_svar2_readbound_variants.py``) is driven over the dataset's own regions + in the same (region, sample, ploid) order, so this pins the ``Svar2Haps`` dispatch + (cache slicing + FFI wrapping + contig-group stitching) end-to-end. + """ + from genoray import SparseVar2 + + from tests._oracles.svar2_readbound_inputs import build_readbound_variants + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + b = ds2.with_seqs("variants")[:, :] + + # Dataset regions in on-disk (sorted) order == the getitem's reconstruction + # regions (jitter=0). Build the oracle over the same (start, end) windows. + regions = ds2._full_regions + reg_list = [(int(s), int(e)) for s, e in regions[:, 1:3]] + sv = SparseVar2(svar2_fixture) + oracle = build_readbound_variants(sv, "chr1", reg_list) + + _assert_ragged_equal(b.start.to_packed(), oracle.start.to_packed(), "start") + _assert_ragged_equal(b.ilen.to_packed(), oracle.ilen.to_packed(), "ilen") + _assert_ragged_equal( + b.alt.to_chars().to_packed(), oracle.alt.to_chars().to_packed(), "alt" + ) + + +# -------------------------------------------------------------------------- +# Guard-contract tests: the unsupported combos must RAISE, not silently return +# wrong output. (These lock the guards; the min_af one would have caught the +# open()-drops-min_af bug.) +# -------------------------------------------------------------------------- + + +def test_svar2_min_af_guard_raises_open(tmp_path, bed, svar2_fixture, _src): + """Dataset.open(min_af=...) must reach the NotImplementedError guard. + + Regression for the bug where _build_seqs dropped min_af/max_af for svar2, + leaving Svar2Haps.min_af=None so the guard never fired. + """ + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref, min_af=0.05) + with pytest.raises(NotImplementedError, match="min_af"): + ds.with_seqs("haplotypes")[:, :] + + +def test_svar2_min_af_guard_raises_with_settings(tmp_path, bed, svar2_fixture, _src): + """with_settings(min_af=...) must also reach the guard.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_settings(min_af=0.05) + with pytest.raises(NotImplementedError, match="min_af"): + ds.with_seqs("haplotypes")[:, :] + + +def test_svar2_splice_and_rc_guards_raise(tmp_path, bed, svar2_fixture, _src): + """splice_plan and a real (all-True) in-kernel to_rc must raise for haplotypes.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_seqs("haplotypes") + recon = ds._recon # the Svar2Haps reconstructor (RaggedSeqs kind) + + idx = np.array([0], np.intp) + r_idx = np.array([0], np.intp) + regions = ds._full_regions[[0]].copy() + rng = np.random.default_rng(0) + + with pytest.raises(NotImplementedError, match="[Ss]plice"): + recon(idx, r_idx, regions, "ragged", 0, rng, True, splice_plan=object()) + + with pytest.raises(NotImplementedError, match="reverse-complement"): + recon( + idx, + r_idx, + regions, + "ragged", + 0, + rng, + True, + to_rc=np.ones(len(idx), np.bool_), + ) + + +def test_svar2_variants_jitter_guard_raises(tmp_path, svar2_fixture, _src): + """variants mode must raise when the dataset was written with max_jitter>0. + + The read-bound variants decode does not right-clip, so a padded cache would + silently over-include variants; the guard prevents that. + """ + from genoray import SparseVar2 + + _bcf, ref = _src + # chromStart >= max_jitter so the padded window stays non-negative. + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, + jbed, + variants=SparseVar2(svar2_fixture), + samples=None, + max_jitter=2, + overwrite=True, + ) + ds = gvl.Dataset.open(d, reference=ref) + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variants")[:, :] + + +# -------------------------------------------------------------------------- +# Multi-contig haplotype parity: locks the contig-group split + inverse +# row-permutation stitching in Svar2Haps. +# -------------------------------------------------------------------------- + +# chr2 reference; VCF REF alleles match _REF2 exactly (idx4='C', idx8='T'). +_REF2 = "TTGGCCAATTGGCCAATTACGTACGTTTGGCCAATTGGCC" +_VCF2 = """\ +##fileformat=VCFv4.2 +##contig= +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +chr2\t5\t.\tC\tT\t.\t.\t.\tGT\t1|0\t1|1 +chr2\t9\t.\tT\tTGG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def _src2(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_mc_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n>chr2\n{_REF2}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + vcf = d / "in.vcf" + vcf.write_text(_VCF2) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar_fixture2(_src2, tmp_path_factory) -> Path: + bcf, _ref = _src2 + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1_mc") / "store.svar" + SparseVar.from_vcf( + out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return out + + +@pytest.fixture(scope="module") +def svar2_fixture2(_src2, tmp_path_factory) -> Path: + bcf, ref = _src2 + from genoray import _core + + out = tmp_path_factory.mktemp("svar2_mc") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1", "chr2"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +def test_svar2_haplotypes_match_svar1_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """Haplotypes byte-identical to SVAR1 across a TWO-contig, out-of-order bed. + + The interleaved chr2/chr1 bed forces Svar2Haps' contig-group split + inverse + row-permutation stitching (single-contig fast path is bypassed). + """ + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + # Interleaved contigs + a variant-free tail region, out of sorted order. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "mc1.gvl" + d2 = tmp_path / "mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + + a = ds1.with_seqs("haplotypes")[:, :] + b = ds2.with_seqs("haplotypes")[:, :] + assert np.array_equal(np.asarray(a.offsets), np.asarray(b.offsets)), ( + f"offsets differ: svar1={np.asarray(a.offsets).tolist()} " + f"svar2={np.asarray(b.offsets).tolist()}" + ) + assert np.array_equal(a.data.view("u1"), b.data.view("u1")) + + +# -------------------------------------------------------------------------- +# variant-windows (Task 1): ref_window pinned to SVAR1; alt_window validated via +# ref-flank decomposition + tokenized variants.alt; multi-contig stitch, dummy +# fill, and the ref="allele" / jitter guards. +# -------------------------------------------------------------------------- + + +def test_svar2_variant_windows_alt_window_decomposition( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """alt_window[j] == ref_window[j][:L] + tokenize(alt_j) + ref_window[j][-L:]. + Uses only svar2's own outputs; ref_window is separately pinned to SVAR1.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + w_win = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + alt_opt = VarWindowOpt( + flank_length=L, + token_alphabet=b"ACGT", + unknown_token=4, + ref="window", + alt="allele", + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + + rw = w_win.ref_window + aw = w_win.alt_window + ba = w_alt.alt # bare tokenized alt (_FlatWindow) + assert aw is not None and rw is not None and ba is not None + + # Same variant SET/order across the two reads. + assert np.array_equal(np.asarray(aw.var_offsets), np.asarray(ba.var_offsets)) + n_var = len(np.asarray(aw.seq_offsets)) - 1 + rso, aso, bso = ( + np.asarray(rw.seq_offsets), + np.asarray(aw.seq_offsets), + np.asarray(ba.seq_offsets), + ) + rd, ad, bd = np.asarray(rw.data), np.asarray(aw.data), np.asarray(ba.data) + for j in range(n_var): + rj = rd[rso[j] : rso[j + 1]] + aj = ad[aso[j] : aso[j + 1]] + bj = bd[bso[j] : bso[j + 1]] + expected = np.concatenate([rj[:L], bj, rj[len(rj) - L :]]) + assert np.array_equal(aj, expected), f"alt_window variant {j} mismatch" + + +def test_svar2_variant_windows_bare_alt_tokenizes_variants_alt( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + import awkward as ak + + from genvarloader._dataset._flat_flanks import build_token_lut + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + alt_opt = VarWindowOpt( + flank_length=L, + token_alphabet=b"ACGT", + unknown_token=4, + ref="window", + alt="allele", + ) + w_alt = ds2.with_output_format("flat").with_seqs("variant-windows", alt_opt)[:, :] + v = ds2.with_seqs("variants")[:, :] # RaggedVariants (validated) + + lut, _ = build_token_lut(b"ACGT", 4) + # Flat (b*p) rows, each a list of alt byte-strings in variant order. + alt_rows = ak.to_list(v.alt.to_ak()) # (b*p) -> [bytes,...] + flat_alts: list[bytes] = [] + for per_var in alt_rows: + for a in per_var: + flat_alts.append(bytes(a) if not isinstance(a, bytes) else a) + + ba = w_alt.alt + bso, bd = np.asarray(ba.seq_offsets), np.asarray(ba.data) + assert len(flat_alts) == len(bso) - 1 + for j, a in enumerate(flat_alts): + toks = bd[bso[j] : bso[j + 1]] + expected = np.array([lut[byte] for byte in a], dtype=toks.dtype) + assert np.array_equal(toks, expected), f"bare alt variant {j} mismatch" + + +def test_svar2_variant_windows_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + """ref_window byte-identical to SVAR1 across an interleaved 2-contig bed + (single-contig fast path bypassed -> exercises the group-stitch reorder).""" + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 10, 5], + "chromEnd": [40, 40, 40, 20], + } + ) + d1 = tmp_path / "vw_mc1.gvl" + d2 = tmp_path / "vw_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ds1.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + w2 = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[:, :] + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # alt_window decomposition holds across the stitch too. + w2.alt_window.to_ragged() # offsets/data consistent post-reorder + w2.ref_window.to_ragged() + + +def test_svar2_variant_windows_unphased_union( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + """Union folds ploidy 2->1 for windows; ref_window still byte-identical to + SVAR1 union, and the union row is hap-0's windows then hap-1's, concatenated.""" + _bcf, ref = _src + ds1, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + w1 = ( + ds1.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + # Ploidy axis folded 2 -> 1. Scalar shape is (R,S,p_eff,None) so ploidy is at + # [-2]; window shape is (R,S,p_eff,None,None) so ploidy is at [-3]. + assert w2.fields["start"].shape[-2] == 1 + assert w2.ref_window.shape[-3] == 1 + _assert_window_equal(w2.ref_window, w1.ref_window, "ref_window") + # Union row count == sum over haplotypes: compare to the non-union var counts. + nu = np.asarray(w2.ref_window.var_offsets) + w2_diploid = ds2.with_output_format("flat").with_seqs("variant-windows", _WIN_OPT)[ + :, : + ] + nd = np.asarray(w2_diploid.ref_window.var_offsets) + P = int(ds2._seqs.genotypes.shape[-2]) + # Folded per-row counts == sum of the P per-hap counts (rows q*P+p are contiguous). + diploid_counts = np.diff(nd).reshape(-1, P).sum(1) + union_counts = np.diff(nu) + assert np.array_equal(union_counts, diploid_counts) + w2.ref_window.to_ragged() + w2.alt_window.to_ragged() + + +def test_svar2_variant_windows_union_multicontig( + tmp_path, svar_fixture2, svar2_fixture2, _src2 +): + from genoray import SparseVar, SparseVar2 + + _bcf, ref = _src2 + bed = pl.DataFrame( + {"chrom": ["chr2", "chr1"], "chromStart": [0, 0], "chromEnd": [40, 40]} + ) + d1 = tmp_path / "vwu_mc1.gvl" + d2 = tmp_path / "vwu_mc2.gvl" + gvl.write(d1, bed, variants=SparseVar(svar_fixture2), samples=None, overwrite=True) + gvl.write( + d2, bed, variants=SparseVar2(svar2_fixture2), samples=None, overwrite=True + ) + ds1 = gvl.Dataset.open(d1, reference=ref) + ds2 = gvl.Dataset.open(d2, reference=ref) + w1 = ( + ds1.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + w2 = ( + ds2.with_output_format("flat") + .with_seqs("variant-windows", _WIN_OPT) + .with_settings(unphased_union=True)[:, :] + ) + assert w2.ref_window.shape[-3] == 1 # window ploidy axis + _assert_window_equal( + w2.ref_window, w1.ref_window, "ref_window (union, multicontig)" + ) + w2.alt_window.to_ragged() + + +def test_svar2_variant_windows_dummy_fills_empty_groups( + tmp_path, bed, svar_fixture, svar2_fixture, _src +): + from genvarloader import DummyVariant + + _bcf, ref = _src + _, ds2 = _open_pair(tmp_path, bed, svar_fixture, svar2_fixture, ref) + L = _WIN_OPT.flank_length + dummy = DummyVariant(alt=b"N", ref=b"N") + w = ( + ds2.with_output_format("flat") + .with_settings(dummy_variant=dummy) + .with_seqs("variant-windows", _WIN_OPT)[:, :] + ) + # Every (b*p) row now has >= 1 variant (no empty rows). + vo = np.asarray(w.ref_window.var_offsets) + assert np.all(np.diff(vo) >= 1) + # ref_window dummy width = 2L + len(dummy.ref); alt_window = 2L + len(dummy.alt). + # (For a filled row the sole variant's window length equals the dummy width.) + # Assert at least one dummy-width ref window exists (the tail region rows). + rso = np.asarray(w.ref_window.seq_offsets) + assert (np.diff(rso) == (2 * L + len(dummy.ref))).any() + w.ref_window.to_ragged() + w.alt_window.to_ragged() + + +def test_svar2_variant_windows_ref_allele_guard(tmp_path, bed, svar2_fixture, _src): + """ref='allele' needs stored REF bytes svar2 lacks -> ValueError at with_seqs.""" + from genoray import SparseVar2 + + _bcf, ref = _src + d = tmp_path / "d.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_fixture), samples=None, overwrite=True) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + bad = VarWindowOpt( + flank_length=3, + token_alphabet=b"ACGT", + unknown_token=4, + ref="allele", + alt="window", + ) + with pytest.raises(ValueError, match="REF"): + ds.with_seqs("variant-windows", bad) + + +def test_svar2_variant_windows_jitter_guard(tmp_path, svar2_fixture, _src): + """variant-windows must raise when written with max_jitter>0 (no right-clip).""" + from genoray import SparseVar2 + + _bcf, ref = _src + jbed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [5], "chromEnd": [20]}) + d = tmp_path / "d.gvl" + gvl.write( + d, + jbed, + variants=SparseVar2(svar2_fixture), + samples=None, + max_jitter=2, + overwrite=True, + ) + ds = gvl.Dataset.open(d, reference=ref).with_output_format("flat") + with pytest.raises(NotImplementedError, match="right-clip"): + ds.with_seqs("variant-windows", _WIN_OPT)[:, :] diff --git a/tests/dataset/test_svar2_fields_read.py b/tests/dataset/test_svar2_fields_read.py new file mode 100644 index 00000000..71659fa8 --- /dev/null +++ b/tests/dataset/test_svar2_fields_read.py @@ -0,0 +1,676 @@ +"""Integration oracle test for SVAR2 INFO/FORMAT field routing (Task 3.1). + +Gates the remaining wiring (Tasks 2.3/2.4) that routes scalar-numeric INFO/FORMAT +field values -- discovered and accepted already (Tasks 2.1/2.2) -- into gvl's +``RaggedVariants`` / variant-windows outputs. Written BEFORE that wiring lands, +so it is EXPECTED TO FAIL (RED) right now; it must fail because the field +VALUES are missing from the output, not because the fixture/store/API is broken. + +Oracle: the source VCF, parsed independently with ``cyvcf2`` (never hardcoded +from this repo's own decode output). Coordinate convention: VCF ``POS`` is +1-based; genoray/gvl positions are 0-based, so every oracle key uses +``POS - 1``. + +Fixture routing (self-asserted below via ``SparseVar2._find_ranges``, not +assumed): + - chr1:3 (0-based 2), A>G -- carried by exactly ONE haplotype (S0/hap0) + out of 6 in the cohort -> cost model routes this to the VAR_KEY channel + (VkSnp, call_idx 0 -- first non-empty hap-column in the var_key stream). + - chr1:8 (0-based 7), AT>A (deletion) -- carried by exactly ONE haplotype + (S2/hap0) -> VAR_KEY channel, VkIndel sub-stream (the fixture was + previously all-SNV, leaving VkIndel/DenseIndel dead in this oracle). + - chr1:10 (0-based 9), G>C -- carried by ALL 6 haplotypes (hom in every + sample) -> cost model routes this to the DENSE channel (DenseSnp, on-disk + row 0, the first dense-routed SNP on the contig). + - chr1:13 (0-based 12), T>A -- carried by exactly ONE haplotype (S1/hap1) + -> VAR_KEY channel, VkSnp call_idx 1 (a *later* hap-column than chr1:3's + call_idx 0): pins that call_idx is NOT hardcoded to 0 end-to-end. + - chr1:16 (0-based 15), T>C -- carried by ALL 6 haplotypes -> DENSE + channel, DenseSnp on-disk row 1 (the second dense-routed SNP): a region + query that includes this but excludes chr1:10 (see Test 5, region 3) + resolves an on-disk dense row via a nonzero ``on_disk.start`` offset, + pinning ``dense_abs_row`` is not hardcoded to row 0 end-to-end. + - chr1:18 (0-based 17), GC>G (deletion) -- carried by ALL 6 haplotypes -> + DENSE channel, DenseIndel sub-stream. + - chr2:5 (0-based 4), A>T -- carried by exactly ONE haplotype (S1/hap0) + -> VAR_KEY channel, on the second contig. + +INFO ``AF`` (Float) is deliberately omitted from the chr1:10 record's INFO to +pin the missing-value fill (NaN, per genoray's ``StoredField`` semantics for a +field declared with no explicit ``default``). ``NS`` (Integer) and FORMAT +``DP`` (Integer) are always present. AF/NS are distinct across every variant; +DP is distinct across every sample within a variant -- both are deliberate +(repeated values would make a broken variant<->value or sample<->value +association silently pass). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import polars as pl +import pytest + +# --- fixture: 2 contigs, 3 samples, ploidy 2 (6 haplotypes/contig) ---------- + +_REF1 = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" # chr1, 40bp +# idx2='A' (pos3), idx7:9='AT' (pos8-9), idx9='G' (pos10), idx12='T' (pos13), +# idx15='T' (pos16), idx17:19='GC' (pos18-19) -- all within the first 20bp, so +# the pre-existing chr1:20-40 window (Test 5, region 2) stays variant-free. +_REF2 = "ACGT" * 7 + "AC" # chr2, 30bp; idx4='A' +assert len(_REF1) == 40 +assert _REF1[2] == "A" and _REF1[7:9] == "AT" and _REF1[9] == "G" +assert _REF1[12] == "T" and _REF1[15] == "T" and _REF1[17:19] == "GC" +assert len(_REF2) == 30 and _REF2[4] == "A" + +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##contig= +##INFO= +##INFO= +##FORMAT= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1\tS2 +chr1\t3\t.\tA\tG\t.\t.\tAF=0.1;NS=5\tGT:DP\t1|0:10\t0|0:20\t0|0:30 +chr1\t8\t.\tAT\tA\t.\t.\tAF=0.55;NS=4\tGT:DP\t0|0:14\t0|0:24\t1|0:34 +chr1\t10\t.\tG\tC\t.\t.\tNS=6\tGT:DP\t1|1:11\t1|1:21\t1|1:31 +chr1\t13\t.\tT\tA\t.\t.\tAF=0.66;NS=7\tGT:DP\t0|0:15\t0|1:25\t0|0:35 +chr1\t16\t.\tT\tC\t.\t.\tAF=0.77;NS=3\tGT:DP\t1|1:16\t1|1:26\t1|1:36 +chr1\t18\t.\tGC\tG\t.\t.\tAF=0.88;NS=1\tGT:DP\t1|1:17\t1|1:27\t1|1:37 +chr2\t5\t.\tA\tT\t.\t.\tAF=0.42;NS=2\tGT:DP\t0|0:12\t1|0:22\t0|0:32 +""" + + +@pytest.fixture(scope="module") +def _src(tmp_path_factory) -> tuple[Path, Path]: + d = tmp_path_factory.mktemp("svar2_fields_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF1}\n>chr2\n{_REF2}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar2_fields_store(_src, tmp_path_factory) -> Path: + from genoray import SparseVar2 + from genoray._svar2_fields import FormatField, InfoField + + bcf, ref = _src + out = tmp_path_factory.mktemp("svar2_fields") / "store.svar2" + SparseVar2.from_vcf( + out=out, + source=bcf, + reference=ref, + info_fields=[InfoField("AF"), InfoField("NS")], + format_fields=[FormatField("DP")], + overwrite=True, + ) + assert (out / "meta.json").exists(), "svar2 conversion did not finish" + return out + + +def _build_oracle(bcf_path: Path) -> tuple[dict[tuple[str, int], dict], list[str]]: + """Parse the source VCF with cyvcf2 (the oracle) into + ``{(contig, pos0): {"AF": float | None, "NS": int | None, + "carriers": {(sample_idx, hap_idx): dp_int}}}``. + + ``carriers`` is derived from cyvcf2's own decoded genotypes (allele == 1), + not hardcoded -- it is the ground truth for which (sample, haplotype) + pairs a read-bound decode kernel must emit a variant record for. + """ + from cyvcf2 import VCF as _CyVCF + + vcf = _CyVCF(str(bcf_path)) + try: + samples = list(vcf.samples) + oracle: dict[tuple[str, int], dict] = {} + for rec in vcf: + contig = rec.CHROM + pos0 = rec.POS - 1 # VCF POS is 1-based; genoray/gvl positions are 0-based. + af = rec.INFO.get("AF") + ns = rec.INFO.get("NS") + dp = np.asarray(rec.format("DP")).reshape(-1) + carriers: dict[tuple[int, int], int] = {} + for s_i, gt in enumerate(rec.genotypes): + for p_i, allele in enumerate(gt[:2]): + if allele == 1: + carriers[(s_i, p_i)] = int(dp[s_i]) + oracle[(contig, pos0)] = { + "AF": None if af is None else float(af), + "NS": None if ns is None else int(ns), + "carriers": carriers, + } + return oracle, samples + finally: + vcf.close() + + +@pytest.fixture(scope="module") +def oracle_and_samples(_src) -> tuple[dict[tuple[str, int], dict], list[str]]: + bcf, _ref = _src + return _build_oracle(bcf) + + +def _build_dataset( + tmp_path: Path, name: str, bed: pl.DataFrame, store: Path, ref: Path +): + import genvarloader as gvl + from genoray import SparseVar2 + + d = tmp_path / name + gvl.write(d, bed, variants=SparseVar2(store), samples=None, overwrite=True) + return gvl.Dataset.open(d, reference=ref) + + +_VAR_FIELDS = ["alt", "start", "ilen", "AF", "NS", "DP"] + + +# --- self-assert: fixture actually exercises BOTH channels ------------------ + + +def test_svar2_fields_store_has_fields_and_routes_both_channels(svar2_fields_store): + """Sanity gate for the fixture itself (not the wiring under test). + + ``available_fields`` must list AF/NS/DP (Task 2.1/2.2 discovery), and a + query spanning chr1 must show BOTH a non-empty var_key window (chr1:3, + carried by 1/6 haplotypes) AND a non-empty dense window (chr1:10, carried + by 6/6 haplotypes) -- else half the provenance logic (Task 2.3/2.4) would + be silently untested by the tests below. + """ + import genoray + + sv = genoray.SparseVar2(str(svar2_fields_store)) + assert set(sv.available_fields) == {"AF", "NS", "DP"}, sv.available_fields + + d = sv._find_ranges("chr1", [0], [40], samples=None) + vk_snp_range = np.asarray(d["vk_snp_range"], np.int64) # (R*S*P, 2) + dense_snp_range = np.asarray(d["dense_snp_range"], np.int64) # (R, 2) + + vk_width = int((vk_snp_range[:, 1] - vk_snp_range[:, 0]).sum()) + dense_width = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + assert vk_width >= 1, ( + f"expected chr1:3 (1/6 haplotypes) to route to var_key, but vk_snp_range " + f"is empty ({vk_snp_range.tolist()})" + ) + assert dense_width >= 1, ( + f"expected chr1:10 (6/6 haplotypes) to route to dense, but " + f"dense_snp_range is empty ({dense_snp_range.tolist()})" + ) + + +# --- shared oracle-comparison helper (diploid RaggedVariants) --------------- + + +def _assert_diploid_fields( + rv, + region_contigs: list[str], + samples: list[str], + oracle: dict[tuple[str, int], dict], + sv, +) -> None: + """Compare a diploid (ploidy-2) ``RaggedVariants`` against the oracle. + + Checks, per decoded (region, sample, ploid, variant): AF/NS by position, + DP by (position, sample) -- AND dtype (no widening) -- AND completeness + (every oracle carrier for a queried contig was actually decoded, so a + silently-dropped call is caught, not just a wrong value). + """ + af_dtype = sv.available_fields["AF"].dtype + ns_dtype = sv.available_fields["NS"].dtype + dp_dtype = sv.available_fields["DP"].dtype + assert np.asarray(rv["AF"].data).dtype == af_dtype + assert np.asarray(rv["NS"].data).dtype == ns_dtype + assert np.asarray(rv["DP"].data).dtype == dp_dtype + + start_ak = rv.start.to_ak().to_list() + af_ak = rv["AF"].to_ak().to_list() + ns_ak = rv["NS"].to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + seen: dict[tuple[str, int], set[tuple[int, int]]] = {} + for r, contig in enumerate(region_contigs): + for s_i in range(len(start_ak[r])): + for p_i in range(len(start_ak[r][s_i])): + for v_i, pos0 in enumerate(start_ak[r][s_i][p_i]): + key = (contig, int(pos0)) + assert key in oracle, f"decoded variant not in oracle: {key}" + exp = oracle[key] + + got_af = af_ak[r][s_i][p_i][v_i] + if exp["AF"] is None: + assert got_af != got_af, ( + f"expected NaN AF (missing in VCF) at {key}, got {got_af}" + ) + else: + expected_af = float(np.asarray(exp["AF"], dtype=af_dtype)) + assert got_af == expected_af, ( + f"AF mismatch at {key}: {got_af} != {expected_af}" + ) + + expected_ns = int(np.asarray(exp["NS"], dtype=ns_dtype)) + assert ns_ak[r][s_i][p_i][v_i] == expected_ns, ( + f"NS mismatch at {key}: {ns_ak[r][s_i][p_i][v_i]} != {expected_ns}" + ) + + assert (s_i, p_i) in exp["carriers"], ( + f"decoded a call not marked as a carrier in the oracle: " + f"{key} sample={samples[s_i]} hap={p_i}" + ) + expected_dp = int( + np.asarray(exp["carriers"][(s_i, p_i)], dtype=dp_dtype) + ) + got_dp = dp_ak[r][s_i][p_i][v_i] + assert got_dp == expected_dp, ( + f"DP mismatch at {key} sample={samples[s_i]}: " + f"{got_dp} != {expected_dp}" + ) + + seen.setdefault(key, set()).add((s_i, p_i)) + + for key, exp in oracle.items(): + contig, _pos0 = key + if contig not in region_contigs or not exp["carriers"]: + continue + assert seen.get(key) == set(exp["carriers"]), ( + f"missing/extra decoded carriers at {key}: " + f"expected {set(exp['carriers'])}, got {seen.get(key)}" + ) + + +# --- Test 1: single-contig --------------------------------------------------- + + +def test_svar2_ragged_variants_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genoray + + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d1.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + assert ds.samples == samples + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr1"], samples, oracle, sv) + + +# --- Test 2: multi-contig, interleaved (exercises the row-reorder path) ---- + + +def test_svar2_ragged_variants_fields_multicontig( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genoray + + oracle, samples = oracle_and_samples + _bcf, ref = _src + # Interleaved chr2/chr1/chr2/chr1, out of natural order -> >1 contig group. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr2", "chr1"], + "chromStart": [0, 0, 15, 20], + "chromEnd": [15, 20, 30, 40], + } + ) + ds = _build_dataset(tmp_path, "d2.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr2", "chr1", "chr2", "chr1"], samples, oracle, sv) + + +# --- Test 3: FORMAT per-sample, explicitly NOT sample 0 --------------------- + + +def test_svar2_ragged_variants_format_not_sample0( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d3.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings(var_fields=_VAR_FIELDS) + + rv = ds[:, :] + start_ak = rv.start.to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + dense_pos0 = 9 # chr1:10 (1-based) -> 0-based 9; carried by ALL samples/haps. + exp = oracle[("chr1", dense_pos0)] + + query_sample = "S1" + s_i = samples.index(query_sample) + assert s_i != 0, "must query a sample other than sample 0 (index 0)" + assert exp["carriers"][(s_i, 0)] != exp["carriers"][(0, 0)], ( + "fixture bug: the queried sample's DP must differ from sample 0's DP " + "at this variant, else a broken FORMAT sample-stride would go undetected" + ) + + found = False + for r in range(len(start_ak)): + for p_i in range(len(start_ak[r][s_i])): + for v_i, pos0 in enumerate(start_ak[r][s_i][p_i]): + if int(pos0) != dense_pos0: + continue + found = True + got_dp = dp_ak[r][s_i][p_i][v_i] + assert got_dp == exp["carriers"][(s_i, p_i)], ( + f"DP mismatch for sample {query_sample} at chr1:{dense_pos0}: " + f"{got_dp} != {exp['carriers'][(s_i, p_i)]}" + ) + assert found, ( + f"expected sample {query_sample} to carry the dense variant at " + f"chr1:{dense_pos0}" + ) + + +# --- Test 4: unphased_union --------------------------------------------------- + + +def test_svar2_ragged_variants_fields_unphased_union( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + """``with_settings(unphased_union=True)`` folds ploidy 2->1: ALT occurrences + from both haplotypes are concatenated per sample (no dedup, no sort). AF/NS + are per-variant (hap-independent) and DP is per-sample (identical for every + haplotype of that sample in this fixture), so each decoded occurrence must + still match the oracle by (contig, start) / (contig, start, sample), and the + per-sample occurrence COUNT must equal the number of carrying haplotypes. + """ + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ds = _build_dataset(tmp_path, "d4.gvl", bed, svar2_fields_store, ref) + ds = ds.with_seqs("variants").with_settings( + var_fields=_VAR_FIELDS, unphased_union=True + ) + + rv = ds[:, :] + assert rv.start.shape[-2] == 1, "unphased_union must fold ploidy to 1" + + start_ak = rv.start.to_ak().to_list() + af_ak = rv["AF"].to_ak().to_list() + ns_ak = rv["NS"].to_ak().to_list() + dp_ak = rv["DP"].to_ak().to_list() + + counts: dict[tuple[str, int, int], int] = {} + for s_i in range(len(start_ak[0])): + for pos0, af, ns, dp in zip( + start_ak[0][s_i][0], af_ak[0][s_i][0], ns_ak[0][s_i][0], dp_ak[0][s_i][0] + ): + key = ("chr1", int(pos0)) + assert key in oracle, f"decoded variant not in oracle: {key}" + exp = oracle[key] + if exp["AF"] is None: + assert af != af, f"expected NaN AF at {key}, got {af}" + else: + assert af == pytest.approx(exp["AF"], rel=0, abs=1e-6), ( + f"AF mismatch at {key}: {af} != {exp['AF']}" + ) + assert ns == exp["NS"], f"NS mismatch at {key}: {ns} != {exp['NS']}" + sample_dp = {v for (si, _p), v in exp["carriers"].items() if si == s_i} + assert sample_dp, f"sample {samples[s_i]} unexpectedly carries {key}" + assert len(sample_dp) == 1, "fixture bug: DP must be uniform per sample" + assert dp == next(iter(sample_dp)), ( + f"DP mismatch at {key} sample={samples[s_i]}: {dp} != {sample_dp}" + ) + counts[(*key, s_i)] = counts.get((*key, s_i), 0) + 1 + + for key, exp in oracle.items(): + contig, _pos0 = key + if contig != "chr1": + continue + per_sample_hap_count: dict[int, int] = {} + for si, _p in exp["carriers"]: + per_sample_hap_count[si] = per_sample_hap_count.get(si, 0) + 1 + for s_i, expected_count in per_sample_hap_count.items(): + got_count = counts.get((*key, s_i), 0) + assert got_count == expected_count, ( + f"union occurrence count mismatch at {key} sample={samples[s_i]}: " + f"{got_count} != {expected_count}" + ) + + +# --- Test 5: variant-windows, including an EMPTY-group fill-value case ----- + + +def test_svar2_variant_windows_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + import genvarloader as gvl + + oracle, samples = oracle_and_samples + _bcf, ref = _src + # Interleaved across contigs (chr2, chr1, chr1, chr1) -> >1 contig group, + # so the multi-contig branch of _reconstruct_variant_windows actually runs + # (not just the single-group fast path). Region 0 covers the chr2 variant; + # region 1 covers all 5 chr1:0-20 variants; region 2 (chr1:20-40) has + # NONE (variant-free, dummy-fill case); region 3 (chr1:12-20) covers only + # chr1:13/16/18 -- excluding chr1:10 (the first DenseSnp on-disk row) + # forces the DenseSnp window here to resolve chr1:16 via a NONZERO + # `on_disk.start` offset, pinning `dense_abs_row` end-to-end. + bed = pl.DataFrame( + { + "chrom": ["chr2", "chr1", "chr1", "chr1"], + "chromStart": [0, 0, 20, 12], + "chromEnd": [15, 20, 40, 20], + } + ) + ds = _build_dataset(tmp_path, "d5.gvl", bed, svar2_fields_store, ref) + opt = gvl.VarWindowOpt(flank_length=3, token_alphabet=b"ACGT", unknown_token=4) + ds = ( + ds.with_output_format("flat") + .with_seqs("variant-windows", opt) + .with_settings( + var_fields=_VAR_FIELDS, + dummy_variant=gvl.DummyVariant(alt=b"N", ref=b"N"), + ) + ) + win = ds[:, :] + assert "AF" in win.fields and "NS" in win.fields and "DP" in win.fields + + P = 2 + S = len(samples) + start_off = np.asarray(win.fields["start"].offsets) + start_data = np.asarray(win.fields["start"].data) + af_data = np.asarray(win.fields["AF"].data) + ns_data = np.asarray(win.fields["NS"].data) + dp_data = np.asarray(win.fields["DP"].data) + + def _group(r: int, s_i: int, p_i: int) -> tuple[int, int]: + g = (r * S + s_i) * P + p_i + return int(start_off[g]), int(start_off[g + 1]) + + def _assert_dummy_fill(lo: int, hi: int, where: str) -> None: + assert hi - lo == 1, ( + f"expected exactly 1 dummy variant in the empty group ({where}), " + f"got {hi - lo}" + ) + assert af_data[lo] != af_data[lo], ( + f"expected NaN AF fill for the empty group ({where}), got {af_data[lo]}" + ) + assert ns_data[lo] == 0, f"expected 0 NS fill ({where}), got {ns_data[lo]}" + assert dp_data[lo] == 0, f"expected 0 DP fill ({where}), got {dp_data[lo]}" + + # Regions 0 (chr2:0-15) and 1 (chr1:0-20) carry real oracle variants, but + # NOT every (sample, hap) is a carrier of every variant in a region (e.g. + # chr2:5 has exactly one carrier) -- so a (sample, hap) group with no + # carried variant in that region still gets the dummy fill, same as a + # wholly variant-free region. Compute, per region, which oracle keys fall + # in it, then check each group against exactly the keys it carries. + region_keys = { + 0: [k for k in oracle if k[0] == "chr2"], + 1: [k for k in oracle if k[0] == "chr1" and k[1] < 20], + 3: [k for k in oracle if k[0] == "chr1" and 12 <= k[1] < 20], + } + for r, keys in region_keys.items(): + for s_i in range(S): + for p_i in range(P): + lo, hi = _group(r, s_i, p_i) + expected_keys = {k for k in keys if (s_i, p_i) in oracle[k]["carriers"]} + if not expected_keys: + _assert_dummy_fill( + lo, hi, f"region {r}, sample {samples[s_i]}, hap {p_i}" + ) + continue + assert hi - lo == len(expected_keys), ( + f"expected {len(expected_keys)} variant(s) for region {r}, " + f"sample {samples[s_i]}, hap {p_i}, got {hi - lo}" + ) + seen_pos: set[int] = set() + for i in range(lo, hi): + pos0 = int(start_data[i]) + key = (keys[0][0], pos0) + assert key in expected_keys, ( + f"decoded variant not expected as a carrier here: {key} " + f"sample={samples[s_i]} hap={p_i}" + ) + seen_pos.add(pos0) + exp = oracle[key] + if exp["AF"] is None: + assert af_data[i] != af_data[i], ( + f"expected NaN AF at {key}, got {af_data[i]}" + ) + else: + assert af_data[i] == pytest.approx(exp["AF"], abs=1e-6) + assert ns_data[i] == exp["NS"] + assert dp_data[i] == exp["carriers"][(s_i, p_i)] + assert seen_pos == {k[1] for k in expected_keys} + + # Region 2 (chr1:20-40): variant-free -> the dummy fill must appear + # (exactly 1 entry per group), with the documented fill values: NaN for + # the float AF column, 0 for the integer NS/DP columns (DummyVariant.info + # was left empty). + for s_i in range(S): + for p_i in range(P): + lo, hi = _group(2, s_i, p_i) + _assert_dummy_fill(lo, hi, f"region 2, sample {samples[s_i]}, hap {p_i}") + + +# --- Test 6: Dataset.open(var_fields=...) entry point (not with_settings) --- + + +def test_svar2_dataset_open_var_fields( + tmp_path, svar2_fields_store, oracle_and_samples, _src +): + """``Dataset.open``'s own ``var_fields`` kwarg must actually route to the + svar2 reconstructor. This is a different code path from + ``with_settings(var_fields=...)`` (exercised by every other test in this + file): ``Dataset.open`` forwards ``var_fields`` to ``Svar2Haps.from_path`` + directly, which previously had no such parameter, so the field was + silently dropped -- advertised in ``available_var_fields`` but absent from + the actual output (no error, no warning). + """ + import genoray + import genvarloader as gvl + + oracle, samples = oracle_and_samples + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d6.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + ds = gvl.Dataset.open(d, reference=ref, var_fields=_VAR_FIELDS).with_seqs( + "variants" + ) + assert "NS" in ds.available_var_fields + assert ds.active_var_fields == _VAR_FIELDS, ( + f"Dataset.open(var_fields=...) did not take effect: " + f"{ds.active_var_fields} != {_VAR_FIELDS}" + ) + + rv = ds[:, :] + sv = genoray.SparseVar2(str(svar2_fields_store)) + _assert_diploid_fields(rv, ["chr1"], samples, oracle, sv) + + +# --- Test 7: unknown var_fields name must raise, not be silently dropped ----- + + +def test_svar2_dataset_open_unknown_var_field_raises( + tmp_path, svar2_fields_store, _src +): + """A typo'd/unsupported field name in ``var_fields`` must raise, mirroring + ``with_settings(var_fields=...)`` (``_impl.py``) and SVAR1's + ``Haps.from_path``. Before the fix, ``Svar2Haps.from_path`` silently + filtered unknown names out of ``_requested_store_fields`` -- the dataset + opened successfully, ``active_var_fields`` reported the typo as "active", + and the output simply lacked it (no error, no warning). + """ + import genoray + import genvarloader as gvl + + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d7.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + with pytest.raises(ValueError, match="Missing variant fields"): + gvl.Dataset.open(d, reference=ref, var_fields=["alt", "start", "TYPO"]) + + # "ref" is a builtin var-field name elsewhere but SVAR2 doesn't provide it. + with pytest.raises(ValueError, match="Missing variant fields"): + gvl.Dataset.open(d, reference=ref, var_fields=["ref"]) + + +# --- Test 8: _output_bytes_per_instance must not KeyError on a store field -- + + +def test_svar2_output_bytes_per_instance_with_store_field( + tmp_path, svar2_fields_store, _src +): + """``_output_bytes_per_instance`` (the buffered/double_buffered dataloader + sizing path, ``_torch.py``'s ``_resolve_buffered_inputs`` -> + ``get_dataloader(mode=...)``) must not crash when an INFO/FORMAT store + field is requested. Before the fix, its ``else`` branch always did + ``haps_obj.variants.info[f].dtype`` -- but ``Svar2Haps.variants`` is the + dummy placeholder with ``info={}``, so any store field name raised + ``KeyError`` there, turning a previously-working path into a crash. + """ + import genoray + import genvarloader as gvl + + _bcf, ref = _src + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + d = tmp_path / "d8.gvl" + gvl.write( + d, + bed, + variants=genoray.SparseVar2(svar2_fields_store), + samples=None, + overwrite=True, + ) + + ds = ( + gvl.Dataset.open(d, reference=ref) + .with_seqs("variants") + .with_settings(var_fields=_VAR_FIELDS, deterministic=True) + ) + out = ds._output_bytes_per_instance() + assert out.shape == ds.shape + assert (out >= 0).all() diff --git a/tests/dataset/test_svar2_readbound_diffs.py b/tests/dataset/test_svar2_readbound_diffs.py new file mode 100644 index 00000000..8b097945 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_diffs.py @@ -0,0 +1,207 @@ +"""Parity test for the read-bound SVAR2 per-hap diffs kernel (Task 7a). + +Oracle: the diffs implied by the read-bound HAPLOTYPE reconstruction +(``build_readbound_haps``) — per (region, hap), ``len(haplotype) - (region_end - +region_start)`` is exactly the ilen diff the reconstruct kernel computed internally +via ``svar2::hap_diffs_svar2`` before sizing/writing the output. Under test: +``build_readbound_diffs`` (same gather, but stops after ``hap_diffs_svar2`` and +returns just the diffs — no reconstruct pass). + +This proves the newly-exposed diffs FFI matches what the reconstruct kernel uses +internally, without needing a second independent oracle. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# Same fixtures as tests/dataset/test_svar2_readbound_haps.py: 40 bp reference +# (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), DEL@11 +# (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_diffs") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key) — +# see test_svar2_readbound_haps.py for the routing rationale. +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_diffs_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) -> np.ndarray: + """Diffs implied by the read-bound haplotype reconstruction: per (region, + hap), ``len(haplotype) - (region_end - region_start)``. + + Query order matches ``build_readbound_haps``/``build_readbound_diffs``: + region-major, sample-minor (``q = r*S + s``), hap-minor within a query. + """ + from tests._oracles.svar2_readbound_inputs import build_readbound_haps + + S, P = sv.n_samples, sv.ploidy + R = len(regions) + + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + offsets = np.asarray(rb.offsets) # (R*S*P + 1,) + + lengths = np.diff(offsets) # (R*S*P,) + ref_lens = np.repeat( + np.asarray([e - s for s, e in regions], np.int64), S * P + ) # (R*S*P,) region-major, sample-minor, hap-minor + diffs = (lengths - ref_lens).astype(np.int32) + return diffs.reshape(R * S, P) + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_diffs_matches_implied_haps(svar2_store, regions): + import genoray + + from tests._oracles.svar2_readbound_inputs import build_readbound_diffs + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + implied = _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) + diffs = np.asarray(build_readbound_diffs(sv, contig, regions)) + + assert diffs.shape == implied.shape + assert np.array_equal(diffs, implied), ( + f"diffs mismatch: implied={implied.tolist()} diffs={diffs.tolist()}" + ) + + +def test_readbound_diffs_dense_snp_matches_implied_haps(svar2_store_dense_snp): + """A SNP routed into dense/snp must diff-clip identically to what the + reconstruct kernel implies.""" + import genoray + + from tests._oracles.svar2_readbound_inputs import build_readbound_diffs + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table. + d = sv._find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + implied = _implied_diffs(regions, ref_arr, ref_offsets, sv, contig) + diffs = np.asarray(build_readbound_diffs(sv, contig, regions)) + + assert diffs.shape == implied.shape + assert np.array_equal(diffs, implied), ( + f"diffs mismatch: implied={implied.tolist()} diffs={diffs.tolist()}" + ) diff --git a/tests/dataset/test_svar2_readbound_haps.py b/tests/dataset/test_svar2_readbound_haps.py new file mode 100644 index 00000000..11f53e61 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_haps.py @@ -0,0 +1,415 @@ +"""Parity test for the read-bound SVAR2 haplotype kernel (Task 4). + +Oracle: ``SparseVar2Source.reconstruct`` (genoray ``_overlap_batch``, eager dense-union +path). Under test: ``build_readbound_haps`` (genoray ``_find_ranges`` + one Rust FFI call +via ``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> +the SAME validated ``reconstruct_haplotypes_from_svar2`` kernel the oracle uses). + +Both paths must be byte-identical: same offsets, same data. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_matches_union_oracle(svar2_store, regions): + import genoray + + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + + oracle_offsets = np.asarray(oracle.offsets) + rb_offsets = np.asarray(rb.offsets) + assert np.array_equal(oracle_offsets, rb_offsets), ( + f"offsets mismatch: oracle={oracle_offsets.tolist()} rb={rb_offsets.tolist()}" + ) + + oracle_data = np.asarray(oracle.data).view("u1") + rb_data = np.asarray(rb.data).view("u1") + if not np.array_equal(oracle_data, rb_data): + # Locate the first mismatching (query, hap, byte) for debuggability. + R = len(regions) + H = P + n_q = R * S + for h in range(n_q * H): + s0, e0 = int(oracle_offsets[h]), int(oracle_offsets[h + 1]) + s1, e1 = int(rb_offsets[h]), int(rb_offsets[h + 1]) + a = oracle_data[s0:e0] + b = rb_data[s1:e1] + if not np.array_equal(a, b): + pytest.fail( + f"data mismatch at hap {h}: oracle={a.tobytes()!r} rb={b.tobytes()!r}" + ) + pytest.fail("data mismatch but no single hap slice differed (offset bug?)") + + +def test_readbound_haps_noncontiguous_ref_raises(svar2_store): + """A non-C-contiguous ``ref_`` view must surface as ``ValueError``, not a Rust + panic. + + ``build_readbound_haps`` (the Python oracle wrapper) defensively + ``np.ascontiguousarray``s ``ref_`` before handing it to the FFI, so it can't be + used to inject a strided array here -- this calls + ``reconstruct_haplotypes_from_svar2_readbound`` directly, replaying the same + ``_find_ranges`` marshalling ``build_readbound_haps`` does internally (see + ``tests/_oracles/svar2_readbound_inputs.py::build_readbound_haps``), but with a + genuinely non-contiguous ``ref_``. + """ + import genoray + + from genvarloader.genvarloader import ( + Svar2Store, + reconstruct_haplotypes_from_svar2_readbound, + ) + + contig = "chr1" + regions = [(0, 40)] + ref_bytes = _REF.encode() + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + # A strided (non-contiguous) view carrying the same bytes as `_REF`: double up + # each byte, then stride over every other one to recover the original values. + doubled = np.repeat(np.frombuffer(ref_bytes, np.uint8), 2) + ref_strided = doubled[::2] + assert ref_strided.flags["C_CONTIGUOUS"] is False + assert bytes(ref_strided) == ref_bytes + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + + d = sv._find_ranges( + contig, [s for s, _ in regions], [e for _, e in regions], samples=None + ) + region_starts_r = np.asarray(d["region_starts"], np.int64) + sample_cols = np.asarray(d["sample_cols"], np.int64) + vk_snp_range = np.ascontiguousarray(d["vk_snp_range"], np.int64) + vk_indel_range = np.ascontiguousarray(d["vk_indel_range"], np.int64) + dense_snp_range_r = np.asarray(d["dense_snp_range"], np.int64) + dense_indel_range_r = np.asarray(d["dense_indel_range"], np.int64) + + R = len(regions) + n_q = R * S + region_starts = np.repeat(region_starts_r, S).astype(np.uint32) + orig_samples = np.tile(sample_cols, R) + dense_snp_range = np.ascontiguousarray( + np.repeat(dense_snp_range_r, S, axis=0), np.int64 + ) + dense_indel_range = np.ascontiguousarray( + np.repeat(dense_indel_range_r, S, axis=0), np.int64 + ) + reg_arr = np.asarray(regions, np.int32).reshape(R, 2) + region_bounds = np.ascontiguousarray(np.repeat(reg_arr, S, axis=0), np.int32) + shifts_a = np.zeros((n_q, P), dtype=np.int32) + + store = Svar2Store(str(sv.path), sv.contigs, sv.n_samples, sv.ploidy) + + with pytest.raises(ValueError): + reconstruct_haplotypes_from_svar2_readbound( + store, + contig, + region_starts, + orig_samples, + vk_snp_range, + vk_indel_range, + dense_snp_range, + dense_indel_range, + region_bounds, + shifts_a, + ref_strided, + ref_offsets, + np.uint8(ord("N")), + np.int64(-1), + False, + ) + + +def test_readbound_matches_union_oracle_with_shifts(svar2_store): + """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" + import genoray + + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps + + contig = "chr1" + regions = [(0, 40), (5, 20)] + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + n_q = len(regions) * S + rng = np.random.default_rng(0) + shifts = rng.integers(-2, 3, size=(n_q, P), dtype=np.int32) + + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=shifts, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=shifts, + output_length=-1, + parallel=False, + ) + + assert np.array_equal(np.asarray(oracle.offsets), np.asarray(rb.offsets)) + assert np.array_equal( + np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") + ) + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key), +# so split_to_flat's snp-block concatenation + snp-before-indel window ordering +# are exercised with real data. genoray dense-encodes a SNP when +# dense_bits (POS_BITS + 2 + n_samples*ploidy) < var_key_bits (34*x_calls); with +# 2 samples x ploidy 2 (np=4) that's 38 < 34*x, i.e. any SNP carried by >=2 +# haplotypes goes dense. The SNP@10 below is carried by 3 haps -> dense/snp. +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_readbound_dense_snp_matches_union_oracle(svar2_store_dense_snp): + """A SNP routed into dense/snp must reconstruct byte-identically. + + Also sanity-checks (before asserting parity) that the SNP actually landed in + dense/snp — i.e. ``_find_ranges``' ``dense_snp_range`` is a non-empty window + for a region covering it — so this test genuinely exercises split_to_flat's + snp-block path rather than silently falling back to the var_key channel. + """ + import genoray + + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_haps + + contig = "chr1" + ref_bytes = _REF.encode() + ref_arr = np.frombuffer(ref_bytes, np.uint8) + ref_offsets = np.array([0, len(ref_bytes)], np.int64) + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a + # region spanning it has a non-empty dense_snp window. + d = sv._find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + indel_win = int(dense_indel_range[0, 1] - dense_indel_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + # Non-triviality: dense/indel is also populated (INS@7 + DEL@12), so the + # combined window mixes snp and indel entries (concatenation under test). + assert indel_win >= 1, dense_indel_range.tolist() + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + oracle = SparseVar2Source(sv).reconstruct( + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + rb = build_readbound_haps( + sv, + contig, + regions, + ref_arr, + ref_offsets, + pad_char=ord("N"), + shifts=None, + output_length=-1, + parallel=False, + ) + + assert np.array_equal(np.asarray(oracle.offsets), np.asarray(rb.offsets)) + assert np.array_equal( + np.asarray(oracle.data).view("u1"), np.asarray(rb.data).view("u1") + ) + + +def _svar2_haps_dataset(tmp_path: Path, svar2_store: Path): + """Build a full gvl Dataset over the ``svar2_store`` fixture and return its + haplotypes view (Svar2Haps-backed). + + Lifted/adapted from ``test_svar2_dataset.py::_open_pair`` -- this file has no + existing fixture that yields a live gvl.Dataset (only bare genoray stores), + so this helper builds the minimal one needed to exercise Svar2Haps through + the public Dataset API. + """ + import polars as pl + from genoray import SparseVar2 + + import genvarloader as gvl + + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [0], "chromEnd": [40]}) + ref = svar2_store.parent / "ref.fa" + d = tmp_path / "ds.gvl" + gvl.write(d, bed, variants=SparseVar2(svar2_store), samples=None, overwrite=True) + return gvl.Dataset.open(d, reference=ref).with_seqs("haplotypes") + + +def test_deterministic_haps_read_skips_pre_reconstruct_diffs( + tmp_path: Path, svar2_store: Path, monkeypatch: pytest.MonkeyPatch +): + """A deterministic (shifts=0) haplotypes read must NOT call the separate + hap_diffs readbound kernel -- reconstruct sizes itself internally. Guards the + double-gather regression.""" + import genvarloader._dataset._svar2_haps as m + + calls = {"diffs": 0} + real = m.hap_diffs_from_svar2_readbound + + def counting(*a, **k): + calls["diffs"] += 1 + return real(*a, **k) + + monkeypatch.setattr(m, "hap_diffs_from_svar2_readbound", counting) + + ds2 = _svar2_haps_dataset(tmp_path, svar2_store) + ds2[:, :] + assert calls["diffs"] == 0 diff --git a/tests/dataset/test_svar2_readbound_tracks.py b/tests/dataset/test_svar2_readbound_tracks.py new file mode 100644 index 00000000..a8b4ddf1 --- /dev/null +++ b/tests/dataset/test_svar2_readbound_tracks.py @@ -0,0 +1,201 @@ +"""Parity test for the read-bound SVAR2 track re-alignment kernel (Task 5). + +Oracle: ``SparseVar2Source.realign_tracks`` (genoray ``_overlap_batch``, eager +dense-union path). Under test: ``build_readbound_tracks`` (genoray +``_find_ranges`` + one Rust FFI call via +``genoray_core::query::gather_haps_readbound`` -> ``svar2::split_to_flat`` -> +the SAME validated ``shift_and_realign_tracks_from_svar2`` kernel the oracle +uses). + +Both paths must be byte-identical: same offsets, same (NaN-equal) data. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G, low-carrier, +# routes to var_key), INS@6 (C>CAT), SNP@9 (G>C, carried by 3 haps -> dense/snp +# per the cost model used in test_svar2_readbound_haps.py), DEL@11 (GTA>G, +# ilen -2). Exercises both var_key and dense/snp + dense/indel channels. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_tracks") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _synthetic_track_inputs(regions, seed=0): + """Per-region flat f32 tracks + (R+1) offsets, plus a known-valid + (params, strategy_id) combo mirroring tests/test_svar2_realign_tracks.py.""" + rng = np.random.default_rng(seed) + lengths = [e - s for s, e in regions] + toff = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + tracks = rng.random(int(toff[-1])).astype(np.float32) + strategy_id = 0 # irrelevant for insertion-fill in this test + params = np.zeros(1, np.float64) + base_seed = 0 + return tracks, toff, params, strategy_id, base_seed + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + dense-SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the variant boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_tracks_match_union_oracle(svar2_store, regions): + import genoray + + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_tracks + + contig = "chr1" + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # Self-verify the fixture routes the >=2-carrier SNP@9 to dense/snp (mixed + # with the dense indels) so this parity test genuinely exercises the dense + # path for tracks. Without this, a future cost-model change could silently + # demote the SNP to var_key and the test would still pass while covering less. + d = sv._find_ranges(contig, [0], [40], samples=None) + snp_win = int( + np.asarray(d["dense_snp_range"])[0, 1] - np.asarray(d["dense_snp_range"])[0, 0] + ) + indel_win = int( + np.asarray(d["dense_indel_range"])[0, 1] + - np.asarray(d["dense_indel_range"])[0, 0] + ) + assert snp_win >= 1 and indel_win >= 1, ( + f"fixture must populate both dense channels; got dense_snp_range=" + f"{np.asarray(d['dense_snp_range']).tolist()}, dense_indel_range=" + f"{np.asarray(d['dense_indel_range']).tolist()}" + ) + + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + + union = SparseVar2Source(sv).realign_tracks( + contig, regions, tracks, toff, params, strat, seed, shifts=None, parallel=False + ) + rb = build_readbound_tracks( + sv, + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=None, + parallel=False, + ) + + union_offsets = np.asarray(union.offsets) + rb_offsets = np.asarray(rb.offsets) + assert np.array_equal(union_offsets, rb_offsets), ( + f"offsets mismatch: union={union_offsets.tolist()} rb={rb_offsets.tolist()}" + ) + + union_data = np.asarray(union.data) + rb_data = np.asarray(rb.data) + if not np.allclose(union_data, rb_data, equal_nan=True): + R = len(regions) + n_q = R * S + for h in range(n_q * P): + s0, e0 = int(union_offsets[h]), int(union_offsets[h + 1]) + s1, e1 = int(rb_offsets[h]), int(rb_offsets[h + 1]) + a = union_data[s0:e0] + b = rb_data[s1:e1] + if not np.allclose(a, b, equal_nan=True): + pytest.fail(f"data mismatch at hap {h}: union={a!r} rb={b!r}") + pytest.fail("data mismatch but no single hap slice differed (offset bug?)") + + +def test_readbound_tracks_match_union_oracle_with_shifts(svar2_store): + """Non-trivial per-hap jitter shifts must also match byte-for-byte.""" + import genoray + + from tests._oracles.svar2_source import SparseVar2Source + from tests._oracles.svar2_readbound_inputs import build_readbound_tracks + + contig = "chr1" + regions = [(0, 40), (5, 20)] + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + n_q = len(regions) * S + + tracks, toff, params, strat, seed = _synthetic_track_inputs(regions) + + rng = np.random.default_rng(1) + shifts = rng.integers(-2, 3, size=(n_q, P), dtype=np.int32) + + union = SparseVar2Source(sv).realign_tracks( + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=shifts, + parallel=False, + ) + rb = build_readbound_tracks( + sv, + contig, + regions, + tracks, + toff, + params, + strat, + seed, + shifts=shifts, + parallel=False, + ) + + assert np.array_equal(np.asarray(union.offsets), np.asarray(rb.offsets)) + assert np.allclose(np.asarray(union.data), np.asarray(rb.data), equal_nan=True) diff --git a/tests/dataset/test_svar2_readbound_variants.py b/tests/dataset/test_svar2_readbound_variants.py new file mode 100644 index 00000000..d5527ffd --- /dev/null +++ b/tests/dataset/test_svar2_readbound_variants.py @@ -0,0 +1,206 @@ +"""Parity test for the read-bound SVAR2 variants decode kernel (Task 6). + +Oracle: ``SparseVar2.decode`` (genoray's own record-``Ragged`` decode, no +overlap/clip filter — the gather already restricts to overlapping variants). +Under test: ``build_readbound_variants`` (genoray ``_find_ranges`` + one Rust FFI +call via ``genoray_core::query::gather_haps_readbound`` -> per-hap ``merge_hap`` + +``decode_alt``, mirroring genoray's ``decode_hap``). + +Both paths decode the SAME full cohort (``samples=None``), so the flat per-hap +(pos, ilen, alt) arrays and the shared variant-axis offsets must be identical. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_variants") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _assert_variants_match(oracle, rb) -> None: + """Compare a genoray ``decode()`` record Ragged (pos/ilen/allele) against a + ``RaggedVariants`` (alt/start/ilen) built by the read-bound kernel.""" + oracle_off = np.asarray(oracle.offsets) + rb_off = np.asarray(rb.offsets) + assert np.array_equal(oracle_off, rb_off), ( + f"variant-axis offsets mismatch: oracle={oracle_off.tolist()} " + f"rb={rb_off.tolist()}" + ) + + pos_match = oracle["pos"].to_ak().to_list() == rb["start"].to_ak().to_list() + ilen_match = oracle["ilen"].to_ak().to_list() == rb["ilen"].to_ak().to_list() + allele_match = oracle["allele"].to_ak().to_list() == rb["alt"].to_ak().to_list() + + if pos_match and ilen_match and allele_match: + return + + # Locate the first mismatching (hap, variant) for debuggability. + n_hap = len(oracle_off) - 1 + o_pos, r_pos = oracle["pos"].to_ak().to_list(), rb["start"].to_ak().to_list() + o_ilen, r_ilen = oracle["ilen"].to_ak().to_list(), rb["ilen"].to_ak().to_list() + o_alt, r_alt = oracle["allele"].to_ak().to_list(), rb["alt"].to_ak().to_list() + for h in range(n_hap): + if (o_pos[h], o_ilen[h], o_alt[h]) != (r_pos[h], r_ilen[h], r_alt[h]): + pytest.fail( + f"mismatch at hap {h}: " + f"oracle=(pos={o_pos[h]}, ilen={o_ilen[h]}, alt={o_alt[h]}) " + f"rb=(pos={r_pos[h]}, ilen={r_ilen[h]}, alt={r_alt[h]})" + ) + pytest.fail("mismatch but no single hap differed (offset/field bug?)") + + +@pytest.mark.parametrize( + "regions", + [ + [(0, 40)], # whole contig: SNP + INS + DEL all in play + [(0, 5), (5, 15), (15, 40)], # split around the SNP/INS/DEL boundaries + [(0, 40), (2, 2), (20, 25)], # empty region + a variant-free window + ], +) +def test_readbound_variants_match_decode_oracle(svar2_store, regions): + import genoray + + from tests._oracles.svar2_readbound_inputs import build_readbound_variants + + contig = "chr1" + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + oracle = sv.decode(contig, regions) + rb = build_readbound_variants(sv, contig, regions) + + _assert_variants_match(oracle, rb) + + +# Fixture whose cost model routes a SNP into the DENSE/snp table (not var_key), +# so split_to_flat's snp-block concatenation + snp-before-indel window ordering +# are exercised with real data (see test_svar2_readbound_haps.py's identical +# fixture recipe for the routing rationale). +_VCF_DENSE_SNP = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t10\t.\tG\tC\t.\t.\t.\tGT\t1|1\t1|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store_dense_snp(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_readbound_variants_dense_snp") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF_DENSE_SNP) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_readbound_variants_dense_snp_match_decode_oracle(svar2_store_dense_snp): + """A SNP routed into dense/snp must decode identically to the oracle. + + Also sanity-checks (before asserting parity) that the SNP actually landed in + dense/snp — i.e. ``_find_ranges``' ``dense_snp_range`` is a non-empty window + for a region covering it — so this test genuinely exercises split_to_flat's + snp-block path rather than silently falling back to the var_key channel. + """ + import genoray + + from tests._oracles.svar2_readbound_inputs import build_readbound_variants + + contig = "chr1" + + sv = genoray.SparseVar2(str(svar2_store_dense_snp)) + assert (sv.n_samples, sv.ploidy) == (2, 2) + + # Routing sanity: the SNP@10 (0-based 9) must be in the dense/snp table, so a + # region spanning it has a non-empty dense_snp window. + d = sv._find_ranges(contig, [0], [40], samples=None) + dense_snp_range = np.asarray(d["dense_snp_range"]) # (R, 2) + dense_indel_range = np.asarray(d["dense_indel_range"]) # (R, 2) + snp_win = int(dense_snp_range[0, 1] - dense_snp_range[0, 0]) + indel_win = int(dense_indel_range[0, 1] - dense_indel_range[0, 0]) + assert snp_win >= 1, ( + f"expected the SNP to route to dense/snp, but dense_snp_range is empty " + f"({dense_snp_range.tolist()}); cost model did not dense-encode it" + ) + # Non-triviality: dense/indel is also populated (INS@7 + DEL@12), so the + # combined window mixes snp and indel entries (concatenation under test). + assert indel_win >= 1, dense_indel_range.tolist() + + regions = [(0, 40), (0, 12), (9, 15), (8, 11)] + oracle = sv.decode(contig, regions) + rb = build_readbound_variants(sv, contig, regions) + + _assert_variants_match(oracle, rb) diff --git a/tests/dataset/test_write_svar2.py b/tests/dataset/test_write_svar2.py new file mode 100644 index 00000000..bb759195 --- /dev/null +++ b/tests/dataset/test_write_svar2.py @@ -0,0 +1,424 @@ +"""Tests for the `.svar2` write path: `_write_from_svar2` + dispatch. + +Builds a `.svar2` store (and a matched `.svar` store from the same VCF+FASTA) +using the same recipe as `tests/test_svar2_reconstruct.py`'s `svar2_store` +fixture, then exercises `gvl.write(..., variants=SparseVar2(...))`. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import numpy as np +import polars as pl +import pytest + +import genvarloader as gvl +from genvarloader._dataset._svar2_link import Svar2Link + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +# Mirrors tests/test_svar2_reconstruct.py's svar2_store fixture exactly, so the +# matched .svar (SVAR1) store built from the same VCF is a valid parity oracle. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def vcf_and_ref(tmp_path_factory) -> tuple[Path, Path]: + """A bgzipped/indexed BCF + FASTA shared by the .svar2 and .svar fixtures.""" + d = tmp_path_factory.mktemp("svar2_write_src") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + return bcf, ref + + +@pytest.fixture(scope="module") +def svar2_store(vcf_and_ref, tmp_path_factory) -> Path: + bcf, ref = vcf_and_ref + from genoray import _core + + out = tmp_path_factory.mktemp("svar2_write") / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +@pytest.fixture(scope="module") +def svar1_store(vcf_and_ref, tmp_path_factory) -> Path: + bcf, _ref = vcf_and_ref + from genoray import VCF, SparseVar + + out = tmp_path_factory.mktemp("svar1_write") / "store.svar" + SparseVar.from_vcf(out, VCF(bcf), max_mem="64m", overwrite=True) + return out + + +def test_write_svar2_emits_cache(svar2_store: Path, tmp_path: Path): + from genoray import SparseVar2 + + svar2 = SparseVar2(svar2_store) + bed = pl.DataFrame( + { + "chrom": ["chr1", "chr1"], + "chromStart": [0, 5], + "chromEnd": [20, 15], + } + ) + out = tmp_path / "ds.gvl" + gvl.write(out, bed, variants=svar2, samples=None, overwrite=True) + + rd = out / "genotypes" / "svar2_ranges" + meta = json.loads((rd / "svar2_meta.json").read_text()) + assert set(meta) >= { + "vk_snp_range", + "vk_indel_range", + "dense_snp_range", + "dense_indel_range", + "sample_cols", + } + assert meta["ploidy"] == svar2.ploidy + + md = json.loads((out / "metadata.json").read_text()) + assert md["svar2_link"] is not None + assert md["ploidy"] == svar2.ploidy + Svar2Link.model_validate(md["svar2_link"]) # shape check + + # ---- FIX 1: verify cache CONTENTS (not just shapes/keys) against a direct + # _find_ranges call over the same regions. gvl sorts the written samples, so + # replay _find_ranges with the sorted sample list to match slot ordering. + # This LOCKS the row-major (R, S, P) reshape and per-contig layout: a + # scrambled / mis-transposed cache would fail loudly here. + sorted_samples = sorted( + svar2.available_samples + ) # what gvl.write wrote (samples.sort()) + S, P = len(sorted_samples), svar2.ploidy + + def mm(name: str) -> np.ndarray: + # raw memmaps are written as ".npy" (no .npy header); the meta key + # is the bare name. Read via np.memmap with the recorded shape/dtype. + shape = tuple(meta[name]["shape"]) + return np.array( + np.memmap(rd / f"{name}.npy", dtype=np.int64, mode="r", shape=shape) + ) + + vk_snp = mm("vk_snp_range") # (R, S, P, 2) + vk_indel = mm("vk_indel_range") # (R, S, P, 2) + dense_snp = mm("dense_snp_range") # (R, 2) + dense_indel = mm("dense_indel_range") # (R, 2) + + # sample_cols is written with np.save (has a .npy header): read with np.load. + sample_cols = np.load(rd / "sample_cols.npy") + assert sample_cols.tolist() == [ + svar2.available_samples.index(s) for s in sorted_samples + ] + + contig_offset = 0 + for (c,), df in bed.partition_by( + "chrom", as_dict=True, maintain_order=True + ).items(): + rc = df.height + lo, hi = contig_offset, contig_offset + rc + d = svar2._find_ranges( + c, + df["chromStart"].to_numpy(), + df["chromEnd"].to_numpy(), + samples=sorted_samples, + ) + # vk ranges: reshape (rc, S, P, 2) -> (rc*S*P, 2) must equal _find_ranges' + # row-major (R*S*P, 2). This pins the reshape done in _write_from_svar2. + np.testing.assert_array_equal( + vk_snp[lo:hi].reshape(rc * S * P, 2), + np.asarray(d["vk_snp_range"], np.int64), + ) + np.testing.assert_array_equal( + vk_indel[lo:hi].reshape(rc * S * P, 2), + np.asarray(d["vk_indel_range"], np.int64), + ) + # dense ranges: per-region (rc, 2), upcast int32 -> int64. + np.testing.assert_array_equal( + dense_snp[lo:hi], np.asarray(d["dense_snp_range"], np.int64) + ) + np.testing.assert_array_equal( + dense_indel[lo:hi], np.asarray(d["dense_indel_range"], np.int64) + ) + contig_offset += rc + + +def test_write_svar2_max_ends_matches_svar1( + svar2_store: Path, svar1_store: Path, tmp_path: Path +): + """SVAR1 parity gate: end-extension semantics must match exactly. + + Regions are chosen to overlap the DEL at (0-based) POS 11 with varying + windows, so the extension is non-trivial and exercises the "no variants" + (keep chromEnd) branch too. + """ + from genoray import SparseVar, SparseVar2 + + svar2 = SparseVar2(svar2_store) + svar1 = SparseVar(svar1_store) + + bed = pl.DataFrame( + { + "chrom": ["chr1"] * 5, + "chromStart": [0, 0, 5, 12, 20], + "chromEnd": [15, 20, 10, 13, 30], + } + ) + + out2 = tmp_path / "ds_svar2.gvl" + gvl.write(out2, bed, variants=svar2, samples=None, overwrite=True) + + out1 = tmp_path / "ds_svar1.gvl" + gvl.write(out1, bed, variants=svar1, samples=None, overwrite=True) + + regions2 = np.load(out2 / "regions.npy") + regions1 = np.load(out1 / "regions.npy") + + # columns: chrom_idx, chromStart, chromEnd, strand + chrom_end_2 = regions2[:, 2] + chrom_end_1 = regions1[:, 2] + + assert chrom_end_2.tolist() == chrom_end_1.tolist(), ( + f"svar2 max_ends {chrom_end_2.tolist()} != svar1 max_ends {chrom_end_1.tolist()}" + ) + + +# Same-POS tie fixture (FIX 2): two records at POS 12 (0-based 11) with different +# ends -- a SNP (G>A, end=12) and a DEL (GTA>G, ILEN -2, end=14) -- placed on +# DIFFERENT haplotypes of S0 (SNP on hap0, DEL on hap1). A single haplotype +# cannot carry both an overlapping SNP and DEL, so putting them on the same hap +# would make the svar2 encoder drop one; different haps keeps both variants +# present and reachable. Ordering is the coordinator's exact example (SNP record +# first, DEL record second), so in store order the DEL gets the higher v_idx. +# SVAR1's max_ends picks the max-v_idx variant's end; svar2 picks the max-end +# variant on a POS tie -- here both rules select the DEL (end 14), so the paths +# agree. See the task-2 report for the reverse store order (DEL-first), where +# the two rules provably diverge. +_TIE_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t12\t.\tG\tA\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t0|1\t0|0 +""" + + +@pytest.fixture(scope="module") +def tie_stores(tmp_path_factory) -> tuple[Path, Path]: + """Matched .svar2 and .svar stores from the same two-same-POS-records VCF.""" + from genoray import VCF, SparseVar, _core + + d = tmp_path_factory.mktemp("svar2_tie") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_TIE_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + svar2_out = d / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(svar2_out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (svar2_out / "meta.json").exists(), "svar2 conversion did not finish" + + svar1_out = d / "store.svar" + SparseVar.from_vcf( + svar1_out, VCF(bcf), max_mem="1g", samples=["S0", "S1"], overwrite=True + ) + return svar2_out, svar1_out + + +def test_write_svar2_max_ends_same_pos_tie( + tie_stores: tuple[Path, Path], tmp_path: Path +): + """SVAR1 parity on a same-POS tie: a SNP and a DEL at the same position. + + The bed region ends 1bp short of the DEL's footprint so the extension is + variant-driven (not masked by the region's own chromEnd). Both paths must + agree on the extended chromEnd. + """ + from genoray import SparseVar, SparseVar2 + + svar2_out, svar1_out = tie_stores + svar2 = SparseVar2(svar2_out) + svar1 = SparseVar(svar1_out) + + # POS 12 -> 0-based 11. Region [11, 13) overlaps it; region chromEnd 13 is + # below the DEL end (14), so the max_ends extension is variant-driven. + bed = pl.DataFrame({"chrom": ["chr1"], "chromStart": [11], "chromEnd": [13]}) + + out2 = tmp_path / "tie_svar2.gvl" + gvl.write(out2, bed, variants=svar2, samples=None, overwrite=True) + out1 = tmp_path / "tie_svar1.gvl" + gvl.write(out1, bed, variants=svar1, samples=None, overwrite=True) + + chrom_end_2 = np.load(out2 / "regions.npy")[:, 2] + chrom_end_1 = np.load(out1 / "regions.npy")[:, 2] + + assert chrom_end_2.tolist() == chrom_end_1.tolist(), ( + f"same-POS tie: svar2 max_ends {chrom_end_2.tolist()} != " + f"svar1 max_ends {chrom_end_1.tolist()}" + ) + + +def test_svar2_extend_to_length_false_raises(svar2_store: Path, tmp_path: Path): + """extend_to_length=False is unsupported for a .svar2 source: it must raise + NotImplementedError, not silently produce an extended dataset.""" + from genoray import SparseVar2 + + svar2 = SparseVar2(svar2_store) + bed = pl.DataFrame( + { + "chrom": ["chr1", "chr1"], + "chromStart": [0, 5], + "chromEnd": [20, 15], + } + ) + out = tmp_path / "ds.gvl" + with pytest.raises(NotImplementedError, match="extend_to_length"): + gvl.write( + out, + bed, + variants=svar2, + samples=None, + extend_to_length=False, + overwrite=True, + ) + + +def _reference_region_max_ends(svar2, contig, starts, ends, samples): + """Byte-for-byte copy of the ORIGINAL _svar2_region_max_ends triple-loop, + kept here as the oracle that pins the vectorized rewrite byte-identical.""" + import numpy as np + + R, S_all, P = len(starts), svar2.n_samples, svar2.ploidy + sel = [svar2.available_samples.index(s) for s in samples] + dec = svar2.decode(contig, list(zip(starts.tolist(), ends.tolist()))) + pos_arr = dec.data["pos"] + ilen_arr = dec.data["ilen"] + off = np.asarray(dec.offsets) + out = np.asarray(ends, np.int64).copy() + for r in range(R): + best_pos, best_end = -1, -1 + for s in sel: + for p in range(P): + h = (r * S_all + s) * P + p + a, b = int(off[h]), int(off[h + 1]) + if a == b: + continue + seg_pos = pos_arr[a:b] + seg_ilen = ilen_arr[a:b] + j = int(np.argmax(seg_pos)) + p_pos = int(seg_pos[j]) + p_end = (p_pos + 1) - min(int(seg_ilen[j]), 0) + if p_pos > best_pos or (p_pos == best_pos and p_end > best_end): + best_pos, best_end = p_pos, p_end + if best_pos >= 0: + out[r] = best_end + return out.astype(np.int32) + + +def test_svar2_region_max_ends_matches_reference(svar2_store: Path): + """Vectorized _svar2_region_max_ends must equal the original per-hap loop, + including the pos-then-end tie-break and the empty-region default = chromEnd.""" + from genoray import SparseVar2 + + from genvarloader._dataset._write import _svar2_region_max_ends + + svar2 = SparseVar2(svar2_store) + # Overlaps the DEL at 0-based POS 11 with varying windows + a no-variant + # region ([20,30]) so both the extension and keep-chromEnd branches run. + starts = np.array([0, 0, 5, 12, 20], dtype=np.int64) + ends = np.array([15, 20, 10, 13, 30], dtype=np.int64) + samples = list(svar2.available_samples) + + got = _svar2_region_max_ends(svar2, "chr1", starts, ends, samples) + ref = _reference_region_max_ends(svar2, "chr1", starts, ends, samples) + np.testing.assert_array_equal(got, ref) + + # Anti-vacuity: at least one region must be EXTENDED past its chromEnd (the + # DEL at POS 11 extends windows that overlap it), else the test only checks + # the trivial default path. + assert (got != ends.astype(np.int32)).any(), ( + f"test is vacuous: no region extended (got={got.tolist()}, ends={ends.tolist()})" + ) + + +def test_svar2_region_max_ends_large_positions(): + """Regression: the composite key must pack a BOUNDED tie-break, not the + absolute end. A variant past ~2 Mb (real chromosomes are hundreds of Mb) + must not overflow the packing / assert-fail. Uses a stub whose decode returns + large positions so we can exercise realistic coordinates without a huge store. + """ + from types import SimpleNamespace + + import numpy as np + + from genvarloader._dataset._write import _svar2_region_max_ends + + # 2 regions x 1 sample x ploidy 1 = 2 haps, 1 variant each: + # region 0: SNP at pos 3_000_000 (ilen 0) -> end 3_000_001 + # region 1: DEL at pos 5_000_000 (ilen -2) -> end 5_000_003 + class _StubSvar2: + n_samples = 1 + ploidy = 1 + available_samples = ["S0"] + + def decode(self, contig, regions): + return SimpleNamespace( + data={ + "pos": np.array([3_000_000, 5_000_000], np.int64), + "ilen": np.array([0, -2], np.int64), + }, + offsets=np.array([0, 1, 2], np.int64), + ) + + svar2 = _StubSvar2() + starts = np.array([0, 0], np.int64) + ends = np.array([10, 10], np.int64) # small chromEnd so both variants extend + got = _svar2_region_max_ends(svar2, "chrBig", starts, ends, ["S0"]) + ref = _reference_region_max_ends(svar2, "chrBig", starts, ends, ["S0"]) + np.testing.assert_array_equal(got, ref) + np.testing.assert_array_equal(got, np.array([3_000_001, 5_000_003], np.int32)) diff --git a/tests/test_svar2_realign_tracks.py b/tests/test_svar2_realign_tracks.py new file mode 100644 index 00000000..0d54f42c --- /dev/null +++ b/tests/test_svar2_realign_tracks.py @@ -0,0 +1,164 @@ +"""End-to-end validation of the SVAR2 track-realign adapter path. + +Builds a DEL-only SVAR2 store, realigns a reference track through gvl's SVAR2 +path (SparseVar2Source.realign_tracks, the Rust two-source kernel), and compares +per-(region, sample, ploid) against gvl's INDEPENDENT pure-Python SVAR1 track +realign (shift_and_realign_track_sparse) fed genoray's materialized decode +records. Agreement proves the SVAR2 Rust track kernel matches the trusted SVAR1 +realign semantics — including the DEL anchor. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). Two pure DELs chosen to match the reference exactly: +# POS 4 GTA>G -> 0-based pos 3, ilen -2 (ref[3:6] == "GTA") +# POS 10 GGG>G -> 0-based pos 9, ilen -2 (ref[9:12] == "GGG") +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t4\t.\tGTA\tG\t.\t.\t.\tGT\t1|0\t1|1 +chr1\t10\t.\tGGG\tG\t.\t.\t.\tGT\t0|1\t1|0 +""" + + +@pytest.fixture(scope="module") +def svar2_del_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_del") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_svar2_realign_tracks_matches_svar1_oracle(svar2_del_store): + import genoray + from tests._oracles.svar2_source import SparseVar2Source + from genvarloader._dataset._tracks import shift_and_realign_track_sparse + + contig = "chr1" + q_start, q_end = 0, 40 + region_len = q_end - q_start + regions = [(q_start, q_end)] + + sv = genoray.SparseVar2(str(svar2_del_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # A per-region reference track (f32). Random-but-fixed so a positional bug + # can't hide behind a monotonic ramp. + rng = np.random.default_rng(0) + track = rng.random(region_len).astype(np.float32) + + strategy_id = 0 # irrelevant for DEL-only (insertion-fill unused) + params = np.zeros(1, np.float64) + base_seed = 0 + + # --- SVAR2 path under test: one region, expanded internally to R*S*P haps --- + src = SparseVar2Source(sv) + out_rag = src.realign_tracks( + contig, + regions, + track, # flat per-region track buffer + np.array([0, region_len], np.int64), # (R+1) offsets + params, + strategy_id, + base_seed, + shifts=None, # no jitter + parallel=False, + ) + + # --- oracle: genoray decode records -> pure-Python SVAR1 realign, per hap --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + + # Non-triviality: haps carry a varying number of DELs. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [1, 1, 2, 1], per_hap_counts + + # `out_rag` is a `_Flat` (flat data/offsets buffer) cast to `Ragged` for typing; + # `_Flat.__getitem__` only supports leading-axis slicing, so pull rows out via + # its flat offsets directly — the same pattern used against this adapter's + # sibling `_Flat` result in tests/test_svar2_reconstruct.py. + out_data = np.asarray(out_rag.data) + out_off = np.asarray(out_rag.offsets) + + for s in range(S): + for p in range(P): + h = (0 * S + s) * P + p # region-major h=(r*S+s)*P+p + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = np.ascontiguousarray(d_pos[gi0:gi1], np.int32) + ilen_h = np.ascontiguousarray(d_ilen[gi0:gi1], np.int32) + n_h = gi1 - gi0 + + # Independently size the hap: region length + sum of (negative) ilens. + exp_len = region_len + int(ilen_h.sum()) + + got = out_data[int(out_off[h]) : int(out_off[h + 1])] + assert got.shape[0] == exp_len, ( + f"(s={s},p={p}) SVAR2 len {got.shape[0]} != expected {exp_len} " + f"(ilen={ilen_h.tolist()})" + ) + + # Synthetic single-hap SVAR1 layout: v_idxs 0..n_h, one group. + geno_v_idxs = np.arange(n_h, dtype=np.int32) + geno_offsets = np.array([0, n_h], np.int64) + expected = np.empty(exp_len, np.float32) + shift_and_realign_track_sparse( + offset_idx=0, + geno_v_idxs=geno_v_idxs, + geno_offsets=geno_offsets, + v_starts=pos_h, + ilens=ilen_h, + shift=0, + track=track, + query_start=q_start, + out=expected, + params=params, + strategy_id=strategy_id, + base_seed=base_seed, + query=0, + hap=h, + ) + np.testing.assert_allclose( + got, + expected, + rtol=0, + atol=0, + err_msg=f"(s={s},p={p}) SVAR2 track != SVAR1 oracle " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})", + ) diff --git a/tests/test_svar2_reconstruct.py b/tests/test_svar2_reconstruct.py new file mode 100644 index 00000000..58b21165 --- /dev/null +++ b/tests/test_svar2_reconstruct.py @@ -0,0 +1,154 @@ +"""Cross-repo end-to-end validation of the SVAR2 two-source reconstruction kernel. + +Builds a known SVAR2 store from a VCF+FASTA fixture (genoray's conversion pipeline), +reconstructs haplotypes through gvl's two-source path (SparseVar2Source, which decodes +genoray's raw two-channel overlap_batch inline), and compares byte-for-byte against an +INDEPENDENT pure-Python consensus applied to genoray's materialized decode records +(the M6c oracle). Agreement proves: (a) gvl's var_key⋈dense merge+decode matches +genoray's decode, and (b) gvl's reconstruction loop matches an independent reference. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +# 40 bp reference (chr1). VCF POS (1-based) -> 0-based: SNP@2 (A>G), INS@6 (C>CAT), +# DEL@11 (GTA>G, ilen -2). Genotypes exercise both samples and both ploids. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def _consensus(ref: bytes, pos, ilen, alleles, q_start: int, q_end: int) -> bytes: + """Independent reference reconstruction: apply position-sorted (pos, ilen, allele) + records to `ref[q_start:q_end]`. A pure DEL has an empty allele — the anchor base + ref[pos] is retained and the following |ilen| bases are dropped (genoray's convention). + """ + order = np.argsort(pos, kind="stable") + out = bytearray() + ref_idx = q_start + for i in order: + p = int(pos[i]) + il = int(ilen[i]) + al = bytes(alleles[i]) + v_end = p - min(0, il) + 1 + # DEL spanning the region start: advance ref past it, emit nothing. + if il < 0 and p < q_start and v_end >= q_start: + ref_idx = v_end + continue + if p < ref_idx: # overlapping variant already consumed — first-one-wins + continue + if p >= q_end: + break + out += ref[ref_idx:p] + seq = al if len(al) > 0 else ref[p : p + 1] + out += seq + ref_idx = v_end + out += ref[ref_idx:q_end] + return bytes(out) + + +def test_svar2_two_source_matches_decode_oracle(svar2_store): + import genoray + from tests._oracles.svar2_source import SparseVar2Source + + contig = "chr1" + q_start, q_end = 0, 40 + regions = [(q_start, q_end)] + ref_bytes = _REF.encode() + + sv = genoray.SparseVar2(str(svar2_store)) + S, P = sv.n_samples, sv.ploidy + assert (S, P) == (2, 2) + + # --- two-source reconstruction (the path under test) --- + src = SparseVar2Source(sv) + hap_rag = src.reconstruct( + contig, + regions, + np.frombuffer(ref_bytes, np.uint8), + np.array([0, len(ref_bytes)], np.int64), + pad_char=ord("N"), + shifts=None, # no jitter + output_length=-1, # ragged + parallel=False, + ) + ts_data = np.asarray(hap_rag.data).view("S1").tobytes() + ts_off = np.asarray(hap_rag.offsets) + + # --- oracle: genoray's materialized decode records (raw flat dict) --- + raw = sv._readers[contig].decode_batch([(q_start, q_end)]) + R, So, Po = int(raw["n_regions"]), int(raw["n_samples"]), int(raw["ploidy"]) + assert (R, So, Po) == (1, S, P) + H = R * So * Po + off = np.asarray(raw["off"]) # (H+1,) per-hap variant offsets + str_off = np.asarray(raw["str_off"]) # per-variant allele-byte offsets + d_pos = np.asarray(raw["pos"]) + d_ilen = np.asarray(raw["ilen"]) + d_allele = np.asarray(raw["allele"]).tobytes() + + # Non-triviality: the fixture yields per-hap variant counts [2, 2, 1, 2] + # (S0h0, S0h1, S1h0, S1h1) — SNP/INS/DEL spread across samples and ploids. + per_hap_counts = (off[1:] - off[:-1]).tolist() + assert per_hap_counts == [2, 2, 1, 2], per_hap_counts + + for h in range(H): + gi0, gi1 = int(off[h]), int(off[h + 1]) + pos_h = d_pos[gi0:gi1] + ilen_h = d_ilen[gi0:gi1] + alleles_h = [ + d_allele[int(str_off[gi]) : int(str_off[gi + 1])] for gi in range(gi0, gi1) + ] + expected = _consensus(ref_bytes, pos_h, ilen_h, alleles_h, q_start, q_end) + got = ts_data[int(ts_off[h]) : int(ts_off[h + 1])] + assert got == expected, ( + f"hap {h}: two-source {got!r} != oracle {expected!r} " + f"(pos={pos_h.tolist()}, ilen={ilen_h.tolist()})" + ) + + # Sensitivity anchor: a DEL-carrying hap must be shorter than the reference, + # and an INS-only hap longer — proving indels actually change output length. + hap_lens = (ts_off[1:] - ts_off[:-1]).tolist() + assert min(hap_lens) < len(ref_bytes) < max(hap_lens), hap_lens diff --git a/tests/unit/dataset/test_dataset_utils.py b/tests/unit/dataset/test_dataset_utils.py index 42afc805..3fcee65f 100644 --- a/tests/unit/dataset/test_dataset_utils.py +++ b/tests/unit/dataset/test_dataset_utils.py @@ -3,7 +3,7 @@ from __future__ import annotations import numpy as np -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from genvarloader._dataset._utils import ( bed_to_regions, oidx_to_raveled_idx, diff --git a/tests/unit/dataset/test_svar2_link.py b/tests/unit/dataset/test_svar2_link.py new file mode 100644 index 00000000..fb0c2950 --- /dev/null +++ b/tests/unit/dataset/test_svar2_link.py @@ -0,0 +1,102 @@ +"""Unit tests for ``Svar2Link`` resolution + fingerprint integrity. + +Mirrors ``test_svar_link_models.py`` but for the ``.svar2`` back-reference +(``_svar2_link.py``). Three pure/tmp_path tests exercise the override/no-op +error paths; one integration-flavored test builds a real ``.svar2`` store +(via genoray's conversion pipeline, same fixture recipe as +``tests/test_svar2_reconstruct.py``) to prove the fingerprint actually +detects a mutated store. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from genvarloader._dataset._svar2_link import ( + Svar2Fingerprint, + Svar2Link, + _resolve_svar2, + _verify_svar2_fingerprint, + make_svar2_link, +) + +# Same tiny fixture recipe as tests/test_svar2_reconstruct.py::svar2_store. +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +def test_resolve_prefers_override(tmp_path: Path): + real = tmp_path / "cohort.svar2" + real.mkdir() + link = Svar2Link( + relative_path="nope.svar2", + absolute_path="/nope.svar2", + fingerprint=Svar2Fingerprint(n_files=1, store_bytes=1), + ) + assert _resolve_svar2(tmp_path, link, real) == real + + +def test_resolve_missing_override_raises(tmp_path: Path): + with pytest.raises(FileNotFoundError): + _resolve_svar2(tmp_path, None, tmp_path / "absent.svar2") + + +def test_verify_none_link_is_noop(tmp_path: Path): + _verify_svar2_fingerprint(tmp_path, None) # must not raise + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_link") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "store.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_fingerprint_detects_mutated_store(svar2_store: Path, tmp_path: Path): + gvl_path = tmp_path / "ds.gvl" + gvl_path.mkdir() + + link = make_svar2_link(gvl_path, svar2_store) + _verify_svar2_fingerprint(svar2_store, link) # must not raise + + bin_files = sorted(svar2_store.rglob("*.bin")) + assert bin_files, "expected at least one .bin file in a real .svar2 store" + with open(bin_files[0], "ab") as f: + f.write(b"\x00") + + with pytest.raises(ValueError): + _verify_svar2_fingerprint(svar2_store, link) diff --git a/tests/unit/dataset/test_svar2_store.py b/tests/unit/dataset/test_svar2_store.py new file mode 100644 index 00000000..8f18d5f0 --- /dev/null +++ b/tests/unit/dataset/test_svar2_store.py @@ -0,0 +1,60 @@ +"""Svar2Store pyclass: opens one query-only genoray_core ContigReader per contig +at construction (the SVAR2 analog of SVAR1's cached FFI-static), held for the +store's lifetime. Built from a real .svar2 store via genoray's conversion pipeline. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from genvarloader.genvarloader import Svar2Store # compiled extension + +_REF = "ACAGTACATGGGTACTAGCTAGGCTAACCGGTTAACCGGT" +_VCF = """\ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS0\tS1 +chr1\t3\t.\tA\tG\t.\t.\t.\tGT\t1|0\t0|0 +chr1\t7\t.\tC\tCAT\t.\t.\t.\tGT\t0|1\t1|1 +chr1\t12\t.\tGTA\tG\t.\t.\t.\tGT\t1|1\t0|1 +""" + + +@pytest.fixture(scope="module") +def svar2_store(tmp_path_factory) -> Path: + from genoray import _core + + d = tmp_path_factory.mktemp("svar2_store") + ref = d / "ref.fa" + ref.write_text(f">chr1\n{_REF}\n") + subprocess.run(["samtools", "faidx", str(ref)], check=True) + + vcf = d / "in.vcf" + vcf.write_text(_VCF) + bcf = d / "in.bcf" + subprocess.run(["bcftools", "view", "-Ob", "-o", str(bcf), str(vcf)], check=True) + subprocess.run(["bcftools", "index", str(bcf)], check=True) + + out = d / "cohort.svar2" + _core.run_conversion_pipeline( + str(bcf), + str(ref), + ["chr1"], + str(out), + ["S0", "S1"], + 25_000, + 2, + 1, + 8 * 1024 * 1024, + ) + assert (out / "meta.json").exists(), "conversion did not finish" + return out + + +def test_store_opens_contigs(svar2_store: Path): + store = Svar2Store(str(svar2_store), ["chr1"], n_samples=2, ploidy=2) + assert store.contigs() == ["chr1"] diff --git a/tests/unit/ragged/test_rag_variants.py b/tests/unit/ragged/test_rag_variants.py index a3dc817e..193d1bdf 100644 --- a/tests/unit/ragged/test_rag_variants.py +++ b/tests/unit/ragged/test_rag_variants.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from genoray._svar import POS_TYPE +from genoray._types import POS_TYPE from genvarloader import RaggedVariants from numpy.typing import NDArray from pytest_cases import parametrize_with_cases diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b0bfd560..2a60e465 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,6 +1,6 @@ import numpy as np import polars as pl -from genoray._utils import ContigNormalizer +from genoray._contigs import ContigNormalizer from genvarloader._dataset._utils import bed_to_regions from genvarloader._utils import normalize_contig_name from pytest_cases import parametrize_with_cases