From 5858bbc0797e07d415d0cd80ffe11a02a055d0ca Mon Sep 17 00:00:00 2001 From: inefabl <2502109325@qq.com> Date: Wed, 15 Jul 2026 15:26:22 +0800 Subject: [PATCH] feat: add PostgreSQL quantum DB integration --- .gitignore | 25 + README.md | 15 + systems/quantum_db/.env.example | 5 + systems/quantum_db/.gitignore | 44 + systems/quantum_db/Makefile | 18 + systems/quantum_db/README.md | 92 ++ systems/quantum_db/README_QDB.md | 172 +++ .../quantum_db/config/quantum_db.example.json | 10 + .../config/tn_quantum_experiment.json | 16 + .../config/wukong_180_hardware.json | 1015 +++++++++++++++++ .../quantum_db/control_plane/monitor/alert.py | 59 + .../control_plane/monitor/monitor.py | 366 ++++++ .../quantum_db/control_plane/scheduler/dag.py | 160 +++ .../control_plane/scheduler/policy.py | 29 + .../control_plane/scheduler/priority_queue.py | 51 + .../control_plane/scheduler/scheduler.py | 243 ++++ .../control_plane/scheduler/state_store.py | 46 + .../control_plane/semantic/IR_builder.py | 158 +++ .../control_plane/semantic/experiment_docs.py | 26 + .../control_plane/semantic/normalizer.py | 92 ++ .../control_plane/semantic/parser.py | 268 +++++ .../control_plane/semantic/router.py | 5 + .../semantic/tn_page_experiment.py | 159 +++ .../semantic/workload_semantic.py | 166 +++ systems/quantum_db/data_plane/bus/bus.py | 90 ++ .../quantum_db/data_plane/bus/command_bus.py | 29 + systems/quantum_db/data_plane/bus/data_bus.py | 32 + .../quantum_db/data_plane/config_loader.py | 16 + .../quantum_db/data_plane/quantum/adapter.py | 535 +++++++++ .../quantum_db/data_plane/quantum/backend.py | 49 + .../quantum_db/data_plane/quantum/circuit.py | 33 + .../quantum_db/data_plane/quantum/executor.py | 71 ++ .../quantum_db/data_plane/quantum/fallback.py | 33 + .../data_plane/quantum/validation.py | 180 +++ systems/quantum_db/data_plane/tn/cache.py | 74 ++ systems/quantum_db/data_plane/tn/compute.py | 508 +++++++++ .../quantum_db/data_plane/tn/contraction.py | 282 +++++ systems/quantum_db/data_plane/tn/executor.py | 201 ++++ systems/quantum_db/data_plane/tn/exproter.py | 286 +++++ systems/quantum_db/data_plane/tn/modes.py | 57 + .../quantum_db/data_plane/tn/multi_cache.py | 255 +++++ systems/quantum_db/data_plane/tn/page.py | 66 ++ .../quantum_db/data_plane/tn/page_manager.py | 152 +++ systems/quantum_db/data_plane/tn/scoring.py | 93 ++ systems/quantum_db/data_plane/tn/workloads.py | 322 ++++++ systems/quantum_db/docs/REPOSITORY_LAYOUT.md | 17 + systems/quantum_db/docs/architecture_plan.md | 27 + systems/quantum_db/docs/completion_audit.md | 47 + systems/quantum_db/docs/current_state.md | 45 + systems/quantum_db/docs/experiment_fields.md | 44 + systems/quantum_db/docs/first_round_status.md | 50 + .../quantum_db/docs/postgres_validation.md | 63 + systems/quantum_db/infra/clock.py | 4 + systems/quantum_db/infra/config.py | 4 + systems/quantum_db/infra/ids.py | 4 + systems/quantum_db/infra/logger.py | 4 + systems/quantum_db/interface/bus.py | 18 + systems/quantum_db/interface/cdk_service.py | 388 +++++++ systems/quantum_db/interface/monitor.py | 11 + systems/quantum_db/interface/quantum.py | 11 + systems/quantum_db/interface/scheduler.py | 13 + systems/quantum_db/interface/semantic.py | 11 + systems/quantum_db/interface/tn.py | 12 + systems/quantum_db/pg_extension/Makefile | 9 + systems/quantum_db/pg_extension/README.md | 36 + .../pg_extension/expected/quantum_db.out | 23 + systems/quantum_db/pg_extension/quantum_db.c | 654 +++++++++++ .../pg_extension/quantum_db.conf.sample | 8 + .../pg_extension/quantum_db.control | 5 + .../sql/quantum_db--0.1.0--0.1.1.sql | 176 +++ .../pg_extension/sql/quantum_db--0.1.0.sql | 235 ++++ .../pg_extension/sql/quantum_db--0.1.1.sql | 259 +++++ .../pg_extension/sql/quantum_db.sql | 8 + .../pg_extension/sql/rpc_integration.sql | 9 + systems/quantum_db/protocol/commands.py | 21 + systems/quantum_db/protocol/dto.py | 89 ++ systems/quantum_db/protocol/errors.py | 17 + systems/quantum_db/protocol/events.py | 41 + systems/quantum_db/protocol/status.py | 15 + systems/quantum_db/pyproject.toml | 26 + systems/quantum_db/qdb/__init__.py | 5 + systems/quantum_db/qdb/__main__.py | 3 + systems/quantum_db/qdb/candidate_executor.py | 224 ++++ systems/quantum_db/qdb/client.py | 28 + systems/quantum_db/qdb/jobs.py | 247 ++++ systems/quantum_db/qdb/large_scale.py | 199 ++++ systems/quantum_db/qdb/protocol.py | 52 + systems/quantum_db/qdb/relational_ir.py | 160 +++ systems/quantum_db/qdb/router.py | 51 + systems/quantum_db/qdb/server.py | 79 ++ systems/quantum_db/qdb/service.py | 452 ++++++++ systems/quantum_db/qute/__init__.py | 7 + systems/quantum_db/qute/api/__init__.py | 3 + systems/quantum_db/qute/api/contracts.py | 144 +++ systems/quantum_db/qute/api/query_request.py | 41 + systems/quantum_db/qute/api/query_result.py | 43 + systems/quantum_db/qute/benchmark/__init__.py | 1 + .../qute/benchmark/configs/noisy.json | 14 + .../qute/benchmark/configs/smoke.json | 12 + systems/quantum_db/qute/benchmark/plots.py | 50 + .../quantum_db/qute/benchmark/reporting.py | 110 ++ systems/quantum_db/qute/benchmark/runner.py | 78 ++ .../quantum_db/qute/benchmark/workloads.py | 81 ++ systems/quantum_db/qute/execution/__init__.py | 3 + .../qute/execution/classical_executor.py | 46 + systems/quantum_db/qute/execution/engine.py | 161 +++ .../quantum_db/qute/execution/tn_executor.py | 35 + .../quantum_db/qute/execution/validation.py | 22 + systems/quantum_db/qute/logical/__init__.py | 3 + .../quantum_db/qute/logical/logical_plan.py | 23 + systems/quantum_db/qute/optimizer/__init__.py | 3 + .../quantum_db/qute/optimizer/cost_model.py | 16 + .../quantum_db/qute/optimizer/dispatcher.py | 51 + .../quantum_db/qute/optimizer/feasibility.py | 20 + systems/quantum_db/qute/quantum/__init__.py | 4 + .../qute/quantum/backend/__init__.py | 6 + .../quantum_db/qute/quantum/backend/base.py | 36 + .../qute/quantum/backend/ideal_simulator.py | 92 ++ .../qute/quantum/backend/noisy_simulator.py | 39 + .../qute/quantum/backend/originq_adapter.py | 45 + systems/quantum_db/qute/quantum/batching.py | 65 ++ .../quantum_db/qute/quantum/measurement.py | 31 + .../qute/quantum/resource_estimator.py | 36 + systems/quantum_db/qute/quantum/task_model.py | 20 + systems/quantum_db/qute/quantum/tn_adapter.py | 42 + systems/quantum_db/qute/structured_log.py | 21 + .../scripts/alu_benchmark_driver.py | 225 ++++ systems/quantum_db/scripts/qdb_e2e_smoke.py | 124 ++ .../scripts/qdb_large_scale_benchmark.py | 37 + .../scripts/qdb_postgres_e2e_smoke.sh | 126 ++ .../scripts/qdb_postgres_hybrid_probe.sh | 126 ++ systems/quantum_db/scripts/qdb_sidecar.py | 5 + .../scripts/qdb_wukong_preflight.sh | 34 + .../quantum_db/scripts/qdb_wukong_probe.sh | 75 ++ systems/quantum_db/scripts/qute_benchmark.py | 5 + .../scripts/tn_binary_native_vldb_plots.py | 474 ++++++++ systems/quantum_db/scripts/tn_closed_test.py | 255 +++++ .../scripts/tn_contraction_smoke.py | 41 + .../quantum_db/scripts/tn_page_experiment.py | 335 ++++++ .../scripts/tn_qcloud_collect_and_plot.py | 249 ++++ .../scripts/tn_qpanda3_full_experiment.py | 61 + .../quantum_db/scripts/tn_quantum_smoke.py | 40 + .../tn_single_factor_hardware_experiment.py | 219 ++++ .../scripts/tn_train_data_experiment.py | 217 ++++ .../tests/test_control_plane_primitives.py | 67 ++ systems/quantum_db/tests/test_large_scale.py | 22 + .../tests/test_pg_extension_static.py | 81 ++ systems/quantum_db/tests/test_qdb.py | 440 +++++++ .../quantum_db/tests/test_qute_framework.py | 196 ++++ 149 files changed, 15366 insertions(+) create mode 100644 .gitignore create mode 100644 systems/quantum_db/.env.example create mode 100644 systems/quantum_db/.gitignore create mode 100644 systems/quantum_db/Makefile create mode 100644 systems/quantum_db/README.md create mode 100644 systems/quantum_db/README_QDB.md create mode 100644 systems/quantum_db/config/quantum_db.example.json create mode 100644 systems/quantum_db/config/tn_quantum_experiment.json create mode 100644 systems/quantum_db/config/wukong_180_hardware.json create mode 100644 systems/quantum_db/control_plane/monitor/alert.py create mode 100644 systems/quantum_db/control_plane/monitor/monitor.py create mode 100644 systems/quantum_db/control_plane/scheduler/dag.py create mode 100644 systems/quantum_db/control_plane/scheduler/policy.py create mode 100644 systems/quantum_db/control_plane/scheduler/priority_queue.py create mode 100644 systems/quantum_db/control_plane/scheduler/scheduler.py create mode 100644 systems/quantum_db/control_plane/scheduler/state_store.py create mode 100644 systems/quantum_db/control_plane/semantic/IR_builder.py create mode 100644 systems/quantum_db/control_plane/semantic/experiment_docs.py create mode 100644 systems/quantum_db/control_plane/semantic/normalizer.py create mode 100644 systems/quantum_db/control_plane/semantic/parser.py create mode 100644 systems/quantum_db/control_plane/semantic/router.py create mode 100644 systems/quantum_db/control_plane/semantic/tn_page_experiment.py create mode 100644 systems/quantum_db/control_plane/semantic/workload_semantic.py create mode 100644 systems/quantum_db/data_plane/bus/bus.py create mode 100644 systems/quantum_db/data_plane/bus/command_bus.py create mode 100644 systems/quantum_db/data_plane/bus/data_bus.py create mode 100644 systems/quantum_db/data_plane/config_loader.py create mode 100644 systems/quantum_db/data_plane/quantum/adapter.py create mode 100644 systems/quantum_db/data_plane/quantum/backend.py create mode 100644 systems/quantum_db/data_plane/quantum/circuit.py create mode 100644 systems/quantum_db/data_plane/quantum/executor.py create mode 100644 systems/quantum_db/data_plane/quantum/fallback.py create mode 100644 systems/quantum_db/data_plane/quantum/validation.py create mode 100644 systems/quantum_db/data_plane/tn/cache.py create mode 100644 systems/quantum_db/data_plane/tn/compute.py create mode 100644 systems/quantum_db/data_plane/tn/contraction.py create mode 100644 systems/quantum_db/data_plane/tn/executor.py create mode 100644 systems/quantum_db/data_plane/tn/exproter.py create mode 100644 systems/quantum_db/data_plane/tn/modes.py create mode 100644 systems/quantum_db/data_plane/tn/multi_cache.py create mode 100644 systems/quantum_db/data_plane/tn/page.py create mode 100644 systems/quantum_db/data_plane/tn/page_manager.py create mode 100644 systems/quantum_db/data_plane/tn/scoring.py create mode 100644 systems/quantum_db/data_plane/tn/workloads.py create mode 100644 systems/quantum_db/docs/REPOSITORY_LAYOUT.md create mode 100644 systems/quantum_db/docs/architecture_plan.md create mode 100644 systems/quantum_db/docs/completion_audit.md create mode 100644 systems/quantum_db/docs/current_state.md create mode 100644 systems/quantum_db/docs/experiment_fields.md create mode 100644 systems/quantum_db/docs/first_round_status.md create mode 100644 systems/quantum_db/docs/postgres_validation.md create mode 100644 systems/quantum_db/infra/clock.py create mode 100644 systems/quantum_db/infra/config.py create mode 100644 systems/quantum_db/infra/ids.py create mode 100644 systems/quantum_db/infra/logger.py create mode 100644 systems/quantum_db/interface/bus.py create mode 100644 systems/quantum_db/interface/cdk_service.py create mode 100644 systems/quantum_db/interface/monitor.py create mode 100644 systems/quantum_db/interface/quantum.py create mode 100644 systems/quantum_db/interface/scheduler.py create mode 100644 systems/quantum_db/interface/semantic.py create mode 100644 systems/quantum_db/interface/tn.py create mode 100644 systems/quantum_db/pg_extension/Makefile create mode 100644 systems/quantum_db/pg_extension/README.md create mode 100644 systems/quantum_db/pg_extension/expected/quantum_db.out create mode 100644 systems/quantum_db/pg_extension/quantum_db.c create mode 100644 systems/quantum_db/pg_extension/quantum_db.conf.sample create mode 100644 systems/quantum_db/pg_extension/quantum_db.control create mode 100644 systems/quantum_db/pg_extension/sql/quantum_db--0.1.0--0.1.1.sql create mode 100644 systems/quantum_db/pg_extension/sql/quantum_db--0.1.0.sql create mode 100644 systems/quantum_db/pg_extension/sql/quantum_db--0.1.1.sql create mode 100644 systems/quantum_db/pg_extension/sql/quantum_db.sql create mode 100644 systems/quantum_db/pg_extension/sql/rpc_integration.sql create mode 100644 systems/quantum_db/protocol/commands.py create mode 100644 systems/quantum_db/protocol/dto.py create mode 100644 systems/quantum_db/protocol/errors.py create mode 100644 systems/quantum_db/protocol/events.py create mode 100644 systems/quantum_db/protocol/status.py create mode 100644 systems/quantum_db/pyproject.toml create mode 100644 systems/quantum_db/qdb/__init__.py create mode 100644 systems/quantum_db/qdb/__main__.py create mode 100644 systems/quantum_db/qdb/candidate_executor.py create mode 100644 systems/quantum_db/qdb/client.py create mode 100644 systems/quantum_db/qdb/jobs.py create mode 100644 systems/quantum_db/qdb/large_scale.py create mode 100644 systems/quantum_db/qdb/protocol.py create mode 100644 systems/quantum_db/qdb/relational_ir.py create mode 100644 systems/quantum_db/qdb/router.py create mode 100644 systems/quantum_db/qdb/server.py create mode 100644 systems/quantum_db/qdb/service.py create mode 100644 systems/quantum_db/qute/__init__.py create mode 100644 systems/quantum_db/qute/api/__init__.py create mode 100644 systems/quantum_db/qute/api/contracts.py create mode 100644 systems/quantum_db/qute/api/query_request.py create mode 100644 systems/quantum_db/qute/api/query_result.py create mode 100644 systems/quantum_db/qute/benchmark/__init__.py create mode 100644 systems/quantum_db/qute/benchmark/configs/noisy.json create mode 100644 systems/quantum_db/qute/benchmark/configs/smoke.json create mode 100644 systems/quantum_db/qute/benchmark/plots.py create mode 100644 systems/quantum_db/qute/benchmark/reporting.py create mode 100644 systems/quantum_db/qute/benchmark/runner.py create mode 100644 systems/quantum_db/qute/benchmark/workloads.py create mode 100644 systems/quantum_db/qute/execution/__init__.py create mode 100644 systems/quantum_db/qute/execution/classical_executor.py create mode 100644 systems/quantum_db/qute/execution/engine.py create mode 100644 systems/quantum_db/qute/execution/tn_executor.py create mode 100644 systems/quantum_db/qute/execution/validation.py create mode 100644 systems/quantum_db/qute/logical/__init__.py create mode 100644 systems/quantum_db/qute/logical/logical_plan.py create mode 100644 systems/quantum_db/qute/optimizer/__init__.py create mode 100644 systems/quantum_db/qute/optimizer/cost_model.py create mode 100644 systems/quantum_db/qute/optimizer/dispatcher.py create mode 100644 systems/quantum_db/qute/optimizer/feasibility.py create mode 100644 systems/quantum_db/qute/quantum/__init__.py create mode 100644 systems/quantum_db/qute/quantum/backend/__init__.py create mode 100644 systems/quantum_db/qute/quantum/backend/base.py create mode 100644 systems/quantum_db/qute/quantum/backend/ideal_simulator.py create mode 100644 systems/quantum_db/qute/quantum/backend/noisy_simulator.py create mode 100644 systems/quantum_db/qute/quantum/backend/originq_adapter.py create mode 100644 systems/quantum_db/qute/quantum/batching.py create mode 100644 systems/quantum_db/qute/quantum/measurement.py create mode 100644 systems/quantum_db/qute/quantum/resource_estimator.py create mode 100644 systems/quantum_db/qute/quantum/task_model.py create mode 100644 systems/quantum_db/qute/quantum/tn_adapter.py create mode 100644 systems/quantum_db/qute/structured_log.py create mode 100644 systems/quantum_db/scripts/alu_benchmark_driver.py create mode 100644 systems/quantum_db/scripts/qdb_e2e_smoke.py create mode 100644 systems/quantum_db/scripts/qdb_large_scale_benchmark.py create mode 100644 systems/quantum_db/scripts/qdb_postgres_e2e_smoke.sh create mode 100644 systems/quantum_db/scripts/qdb_postgres_hybrid_probe.sh create mode 100644 systems/quantum_db/scripts/qdb_sidecar.py create mode 100644 systems/quantum_db/scripts/qdb_wukong_preflight.sh create mode 100644 systems/quantum_db/scripts/qdb_wukong_probe.sh create mode 100644 systems/quantum_db/scripts/qute_benchmark.py create mode 100644 systems/quantum_db/scripts/tn_binary_native_vldb_plots.py create mode 100644 systems/quantum_db/scripts/tn_closed_test.py create mode 100644 systems/quantum_db/scripts/tn_contraction_smoke.py create mode 100644 systems/quantum_db/scripts/tn_page_experiment.py create mode 100644 systems/quantum_db/scripts/tn_qcloud_collect_and_plot.py create mode 100644 systems/quantum_db/scripts/tn_qpanda3_full_experiment.py create mode 100644 systems/quantum_db/scripts/tn_quantum_smoke.py create mode 100644 systems/quantum_db/scripts/tn_single_factor_hardware_experiment.py create mode 100644 systems/quantum_db/scripts/tn_train_data_experiment.py create mode 100644 systems/quantum_db/tests/test_control_plane_primitives.py create mode 100644 systems/quantum_db/tests/test_large_scale.py create mode 100644 systems/quantum_db/tests/test_pg_extension_static.py create mode 100644 systems/quantum_db/tests/test_qdb.py create mode 100644 systems/quantum_db/tests/test_qute_framework.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..818739b --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Python/editor/local state +__pycache__/ +*.py[cod] +.venv/ +.env +.env.* +!.env.example +.DS_Store +*.sqlite3 +*.log +*.sock + +# Generated experiment outputs and large local datasets +results/ +outputs/ +artifacts/ +train-test-data/ +systems/quantum_db/outputs/ +systems/quantum_db/results/ +systems/quantum_db/artifacts/ + +# Native build products +*.o +*.dylib +*.so diff --git a/README.md b/README.md index f295464..c3965f4 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,21 @@ Reproducibility checklist: 3. Use the same backend (`origin_wukong`). 4. Archive the merged JSON report as the analysis source of truth. +## PostgreSQL Quantum DB integration + +The PostgreSQL-authoritative sidecar, TN/Qute execution contracts, PGXS extension, and large-scale benchmarks live in +[`systems/quantum_db/`](systems/quantum_db/). Its README is the canonical operating guide for this subsystem: + +```bash +cd systems/quantum_db +python3 -m unittest discover -s tests -v +python3 scripts/qdb_e2e_smoke.py +``` + +The integration preserves this repository's original Grover experiment at the top level. It adds a separate +`PostgreSQL → sidecar → TN/Qute/Grover → PostgreSQL recheck` path and does not commit API keys, SQLite state, native +build products, or generated reports. + ## Notes - Wall time includes cloud submission, compilation or mapping, queueing, execution, and result retrieval. diff --git a/systems/quantum_db/.env.example b/systems/quantum_db/.env.example new file mode 100644 index 0000000..38be00a --- /dev/null +++ b/systems/quantum_db/.env.example @@ -0,0 +1,5 @@ +QDB_SOCKET=/tmp/qdb.sock +QDB_STATE=outputs/qdb_jobs.sqlite3 +QDB_QCLOUD_BACKEND=WK_C180 +QDB_PYTHON=python3 +# ORIGINQC_API_KEY= diff --git a/systems/quantum_db/.gitignore b/systems/quantum_db/.gitignore new file mode 100644 index 0000000..6d1643b --- /dev/null +++ b/systems/quantum_db/.gitignore @@ -0,0 +1,44 @@ +# Python bytecode and virtual environments +__pycache__/ +*.py[cod] +*$py.class +.venv/ +venv/ +env/ + +# macOS and editors +.DS_Store +.idea/ +.vscode/ + +# Secrets and local runtime state +.env +.env.* +!.env.example +*.pem +*.key +*.sqlite3 +*.sqlite3-* +*.sock +*.pid +*.log + +# Native PostgreSQL build outputs +pg_extension/*.o +pg_extension/*.dylib +pg_extension/*.so +pg_extension/*.bc +pg_extension/regression.diffs + +# Generated reports and local outputs +/outputs/* +!/outputs/.gitkeep +/results/* +!/results/.gitkeep +/artifacts/* +!/artifacts/README.md + +# Large local corpus and historical duplicate +/train-test-data/ +/storage-main 2/ +pyqpanda3_docs.json diff --git a/systems/quantum_db/Makefile b/systems/quantum_db/Makefile new file mode 100644 index 0000000..495cd72 --- /dev/null +++ b/systems/quantum_db/Makefile @@ -0,0 +1,18 @@ +PYTHON ?= python3 + +.PHONY: test sidecar large-scale pg-smoke pg-hybrid + +test: + $(PYTHON) -m unittest discover -s tests -v + +sidecar: + $(PYTHON) -m qdb.server --socket $${QDB_SOCKET:-/tmp/qdb.sock} --state $${QDB_STATE:-outputs/qdb_jobs.sqlite3} + +large-scale: + $(PYTHON) scripts/qdb_large_scale_benchmark.py --sizes $${QDB_SIZES:-10000,100000,1000000} --output $${QDB_OUTPUT:-outputs/qdb_large_scale} + +pg-smoke: + PG_CONFIG=$${PG_CONFIG:-pg_config} bash scripts/qdb_postgres_e2e_smoke.sh + +pg-hybrid: + PG_CONFIG=$${PG_CONFIG:-pg_config} bash scripts/qdb_postgres_hybrid_probe.sh diff --git a/systems/quantum_db/README.md b/systems/quantum_db/README.md new file mode 100644 index 0000000..b176148 --- /dev/null +++ b/systems/quantum_db/README.md @@ -0,0 +1,92 @@ +# Quantum DB + +Quantum DB is a research prototype that keeps PostgreSQL authoritative while using a local Python sidecar for TN and +quantum candidate telemetry. Quantum/TN outputs are advisory only; PostgreSQL rechecks the original predicates before +returning SQL rows. + +## Components + +- `qdb/`: Unix-socket sidecar, relational IR, routing, jobs, candidate contract, and large-scale benchmark. +- `pg_extension/`: PostgreSQL 16 PGXS extension and `QuantumDBScan` custom scan. +- `qute/`: C0–C3 query execution contracts and simulator backends. +- `data_plane/`, `control_plane/`, `interface/`, `protocol/`, `infra/`: TN/quantum and integration layers. +- `scripts/`, `tests/`, `config/`, `docs/`: reproducible entry points, regression tests, examples, and evidence. + +Detailed Git policy: [docs/REPOSITORY_LAYOUT.md](docs/REPOSITORY_LAYOUT.md). + +## Quick start + +Python 3.11+ is required. The base project has no mandatory third-party Python dependency; pyQPanda3 is optional for +Wukong execution. + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e . +make test +``` + +Start the sidecar locally: + +```bash +qdb-sidecar --socket /tmp/qdb.sock --state outputs/qdb_jobs.sqlite3 +``` + +Run the no-credential smoke tests: + +```bash +python3 scripts/qdb_e2e_smoke.py +python3 scripts/qdb_large_scale_benchmark.py --sizes 10000,100000,1000000 --output outputs/qdb_large_scale +``` + +The large-scale report uses exact streaming traversal as its authority and reports sampling/HLL/TN estimates +separately. + +## PostgreSQL + +Build the extension using PostgreSQL 16 development headers: + +```bash +make -C pg_extension PG_CONFIG=/path/to/pg_config +make -C pg_extension install PG_CONFIG=/path/to/pg_config +PG_CONFIG=/path/to/pg_config make pg-smoke +``` + +Configure PostgreSQL with: + +```conf +shared_preload_libraries = 'quantum_db' +quantum_db.socket_path = '/tmp/qdb.sock' +``` + +Registered datasets reject `INSERT`, `UPDATE`, `DELETE`, and `TRUNCATE` until unregistered. + +## Wukong + +Never commit `ORIGINQC_API_KEY`, `.env`, SQLite state, native binaries, or generated hardware output. Export the token +only in the sidecar shell: + +```bash +export ORIGINQC_API_KEY='' +export QDB_PYTHON=/path/to/pyqpanda-env/bin/python +export QDB_QCLOUD_BACKEND=WK_C180 +bash scripts/qdb_wukong_preflight.sh +``` + +The explicit hardware path is: + +```bash +export QDB_ALLOW_HARDWARE=1 +PG_CONFIG=/path/to/pg_config make pg-hybrid +``` + +It requires a finished QCloud job, a cloud job ID, binary counts, and a counts total equal to the shot count. + +## Upload checklist + +1. Run `make test` and `git diff --check`. +2. Stage only source, tests, docs, configuration examples, and PostgreSQL SQL/C files. +3. Keep the large `train-test-data/` corpus outside ordinary Git or use Git LFS. +4. Do not add `storage-main 2/`. +5. Add a license before making a public repository; no license is assumed here. diff --git a/systems/quantum_db/README_QDB.md b/systems/quantum_db/README_QDB.md new file mode 100644 index 0000000..e43582b --- /dev/null +++ b/systems/quantum_db/README_QDB.md @@ -0,0 +1,172 @@ +# Quantum DB PostgreSQL integration (detailed guide) + +> Start with [README.md](README.md). This file retains detailed PostgreSQL and Wukong operating notes. + +This directory contains the primary implementation. `storage-main 2` is a historical copy and is not used. + +PostgreSQL remains authoritative for SQL semantics, joins, aggregates, MVCC, and predicate rechecks. The +`quantum_db` extension registers immutable datasets and wraps their base scans with a `QuantumDBScan` custom +scan. The Python sidecar provides routing, TN/quantum candidate generation, durable jobs, and pyQPanda3/QCloud +execution. Candidate keys are never treated as final SQL results. + +## Local development + +```bash +python3 -m qdb.server --socket /tmp/qdb.sock --state outputs/qdb_jobs.sqlite3 +make -C pg_extension +make -C pg_extension install +``` + +The extension build requires PostgreSQL 16 server development files. Configure +`shared_preload_libraries = 'quantum_db'` and restart PostgreSQL so every session installs the planner hook. +Then: + +```sql +CREATE EXTENSION quantum_db; +SELECT qdb_register_dataset('public.my_table', 'id', ARRAY['value']); +SELECT qdb_explain('/*+ QDB_AUTO */ SELECT * FROM public.my_table WHERE value = 10'); +``` + +The isolated PostgreSQL 16.14 validation record is in `docs/postgres_validation.md`. + +Set `quantum_db.socket_path` when the sidecar does not use `/tmp/qdb.sock`. Registered datasets reject +INSERT, UPDATE, DELETE, and TRUNCATE until `qdb_unregister_dataset` is called. + +For Wukong, set `ORIGINQC_API_KEY` only in the sidecar environment and set `QDB_QCLOUD_BACKEND=WK_C180` +(or the backend name provided by the installed pyQPanda3 release). Tokens are not persisted. +Use the versioned `hardware.preflight` RPC with `{"allow_hardware": true}` to request a non-submitting capability +check; it refuses to run without both explicit authorization and a sidecar-only token. +It also reports `HARDWARE_SDK_UNAVAILABLE` before any network action when `pyqpanda3` cannot be imported. + +After the SDK and its native libraries are available, the supported entry point is: + +```bash +export ORIGINQC_API_KEY='paste-the-current-key-here' +export QDB_PYTHON=/tmp/qdb-pyqpanda-env/bin/python +export QDB_QCLOUD_BACKEND=WK_C180 +bash scripts/qdb_wukong_preflight.sh +``` + +The script starts a temporary local sidecar, invokes only `hardware.preflight`, redacts its temporary log on failure, +and removes all state afterwards. It does not submit a Wukong job. + +After a successful preflight, run the explicitly authorized 64-shot smoke probe with: + +```bash +bash scripts/qdb_wukong_probe.sh +``` + +It waits for the QCloud result before returning and prints the remote job ID, shots, counts, non-secret metadata, and +target-state analysis (observed proportion, Wilson 95% interval, and enrichment over a uniform baseline). +The probe uses exactly eight rows with one marked address (`0x0`). Its current `grover-oracle-diffuser-v1` +contract is a three-qubit, two-iteration Grover circuit decomposed into `U3 + CZ`; its ideal marked-state +probability is about 94.5%. It is a hardware correctness/calibration probe, not a general quantum SQL accelerator. +For other snapshot widths or multiple marked addresses, the sidecar retains its ideal/advisory probe and does not +submit a misleading real-hardware circuit. + +`QDB_WUKONG_SHOTS` controls statistical precision (not the physical marked-state probability), while +`QDB_GROVER_ITERATIONS` accepts `1` or `2`. A 256-shot run makes the target proportion much easier to estimate. +On current hardware, compare both iteration counts because one iteration halves the CZ budget (12 instead of 24) +while reducing the ideal probability from 94.5% to 78.1%: + +```bash +QDB_WUKONG_SHOTS=256 QDB_GROVER_ITERATIONS=1 bash scripts/qdb_wukong_probe.sh +QDB_WUKONG_SHOTS=256 QDB_GROVER_ITERATIONS=2 bash scripts/qdb_wukong_probe.sh +``` + +By default the probe sends its eight synthetic rows inline. To exercise the sidecar's durable SQLite dataset store, +register the rows first and run the same query from that immutable snapshot: + +```bash +QDB_PROBE_STORAGE=1 QDB_WUKONG_SHOTS=256 QDB_GROVER_ITERATIONS=1 bash scripts/qdb_wukong_probe.sh +``` + +This validates `JobStore` dataset registration/readback, but it does not exercise PostgreSQL's `qdb.datasets` or +`QuantumDBScan`; use `scripts/qdb_postgres_e2e_smoke.sh` for the PostgreSQL integration path. + +On Apple Silicon, the currently packaged `pyqpanda3` wheel links to Homebrew's `libidn2`, `openssl@3`, and `zstd`. +Install the runtime libraries once before using the virtual environment: + +```bash +brew install libidn2 openssl@3 zstd +``` + +Run offline tests with `python3 -m unittest discover -s tests -v`. They use no network or quantum credits. + +The backend-neutral research framework and first-round status are documented in +`docs/architecture_plan.md` and `docs/first_round_status.md`. Run its reproducible smoke benchmark with: + +```bash +python3 -m qute.benchmark.runner --config qute/benchmark/configs/smoke.json --output outputs/qute_smoke +``` + +## Current execution contract + +`QuantumDBScan` is deliberately fail-safe: it scans only candidate keys supplied by the sidecar when the response +asserts a complete *superset* from an immutable classical snapshot, the snapshot version matches, the artifact +checksum matches, and PostgreSQL recheck is required. Physical candidate filtering is currently limited to +built-in `bool`, `int2`, `int4`, and `int8` equality predicates. Text/collated values, numeric/time values, +range operators, NULL tests, expressions, and unsupported clauses retain the native PostgreSQL scan. Any missing +or malformed contract, stale checksum, unavailable sidecar, or unsupported predicate also performs native exact +evaluation. JOIN, COUNT, and GROUP BY therefore keep PostgreSQL semantics. Quantum/TN estimates are advisory +until they can provide a formally complete candidate superset; they cannot by themselves remove SQL rows. +Registered snapshots use an atomic `begin → 256-row pages → commit` Unix-RPC upload. PostgreSQL makes one locked +streaming pass to derive the row count/checksum and a second locked streaming pass to upload pages, so neither the +sidecar request limit nor a full-table JSONB aggregation caps registration memory. QCloud jobs are optional and are only submitted +after explicit hardware authorization for either a validated `quantum_ir` or a bounded probe. For routed asynchronous jobs, the sidecar now builds a bounded TN-Page (`QDB_TN`/hybrid) and/or executes +a bounded Grover-style `QMEASURE` probe on the ideal simulator. Passing `run_qpanda_probe=true` additionally compiles +and runs its `U3 + CZ` IR through the pyQPanda3 adapter's local fallback. If the sidecar is configured for QCloud, a +real-device submission additionally requires `allow_hardware=true` together with `run_qpanda_probe=true` (or an explicit +`quantum_ir`); this prevents ordinary routed SQL from consuming hardware quota. These probe outputs are explicit telemetry: +they are never permitted to remove a classical snapshot candidate or bypass PostgreSQL recheck. `qdb_submit`, `qdb_status`, +`qdb_result`, and successful `qdb_cancel` calls mirror the observed sidecar lifecycle into `qdb.jobs` for SQL-side +audit; SQLite remains the durable executor state until a shared job store is introduced. +Cancellation is terminal: a late cloud failure or completion cannot overwrite a persisted `cancelled` job state. + +Run a sidecar-only end-to-end smoke test (Unix RPC, dataset/job lifecycle, automatic route, candidates, and exact +Qute query contract) with: + +```bash +python3 scripts/qdb_e2e_smoke.py +``` + +Run large-scale streaming traversal, predicate-cardinality, distinct-cardinality (HLL), uniform sampling, Qute C0 +sample counting, and TN-page sampling experiments without using QCloud quota: + +```bash +python3 scripts/qdb_large_scale_benchmark.py --sizes 10000,100000,1000000 --output outputs/qdb_large_scale +``` + +The report retains exact streaming counts as the authority and reports sampling/HLL error separately; it does not +label an estimate as exact SQL output. + +With a PostgreSQL 16 development installation, run the complete temporary-cluster path with: + +```bash +PG_CONFIG=/path/to/pg_config bash scripts/qdb_postgres_e2e_smoke.sh +``` + +It installs the extension into the selected PostgreSQL prefix, then creates and removes only a temporary cluster. + +To add the three participating acceleration components to one explicitly authorized Wukong probe, use a registered +eight-row snapshot with one marked row. `QDB_PROBE_COMPONENTS=hybrid` activates both the TN page and the Grover +telemetry; `QDB_PROBE_QUTE=1` records QuteEngine's C0 exact count validation in the same sidecar job: + +```bash +QDB_PYTHON=/path/to/pyqpanda-env/bin/python \ +QDB_PROBE_STORAGE=1 QDB_PROBE_COMPONENTS=hybrid QDB_PROBE_QUTE=1 \ +QDB_WUKONG_SHOTS=256 QDB_GROVER_ITERATIONS=1 \ +bash scripts/qdb_wukong_probe.sh +``` + +For the full `PostgreSQL → registered snapshot → sidecar (TN + Qute + Grover) → PostgreSQL Custom Scan/recheck` +experiment, the following command creates a disposable cluster and sends exactly one real-device job. It is guarded by +`QDB_ALLOW_HARDWARE=1`; it never stores the API key in PostgreSQL, SQLite, output, or source files. +The script fails unless the result contains `qcloud_sdk`, a cloud job ID, `JobStatus.FINISHED`, binary counts, and a +counts total equal to the returned shot count. + +```bash +QDB_PYTHON=/path/to/pyqpanda-env/bin/python QDB_ALLOW_HARDWARE=1 \ +QDB_WUKONG_SHOTS=256 QDB_GROVER_ITERATIONS=1 \ +PG_CONFIG=/path/to/pg_config bash scripts/qdb_postgres_hybrid_probe.sh +``` diff --git a/systems/quantum_db/config/quantum_db.example.json b/systems/quantum_db/config/quantum_db.example.json new file mode 100644 index 0000000..334e926 --- /dev/null +++ b/systems/quantum_db/config/quantum_db.example.json @@ -0,0 +1,10 @@ +{ + "socket_path": "/tmp/qdb.sock", + "state_path": "outputs/qdb_jobs.sqlite3", + "backend_name": "qpanda3-sim", + "qcloud_backend_name": "WK_C180", + "num_qubits": 180, + "max_circuit_depth": 500, + "workers": 2, + "api_token_env": "ORIGINQC_API_KEY" +} diff --git a/systems/quantum_db/config/tn_quantum_experiment.json b/systems/quantum_db/config/tn_quantum_experiment.json new file mode 100644 index 0000000..6b39f01 --- /dev/null +++ b/systems/quantum_db/config/tn_quantum_experiment.json @@ -0,0 +1,16 @@ +{ + "num_sites": 16, + "phys_dim": 8, + "chi_max": 32, + "keep_ratio": 0.25, + "sample_num_for_fidelity": 64, + "quantum": { + "backend_name": "qpanda3-cloud-noise-sim", + "hardware_profile": "config/wukong_180_hardware.json", + "api_base": "", + "api_token_env": "ORIGINQC_API_KEY", + "timeout_sec": 20, + "shots": 512, + "allow_local_fallback": true + } +} diff --git a/systems/quantum_db/config/wukong_180_hardware.json b/systems/quantum_db/config/wukong_180_hardware.json new file mode 100644 index 0000000..fb33abe --- /dev/null +++ b/systems/quantum_db/config/wukong_180_hardware.json @@ -0,0 +1,1015 @@ +{ + "chip_name": "origin_wukong_180", + "num_qubits": 180, + "num_couplers": 251, + "basis_gates": [ + "u3", + "cz" + ], + "coupling_map": [ + [ + 0, + 9 + ], + [ + 0, + 10 + ], + [ + 1, + 10 + ], + [ + 1, + 11 + ], + [ + 2, + 11 + ], + [ + 2, + 12 + ], + [ + 3, + 12 + ], + [ + 4, + 14 + ], + [ + 5, + 14 + ], + [ + 5, + 15 + ], + [ + 6, + 15 + ], + [ + 6, + 16 + ], + [ + 7, + 16 + ], + [ + 7, + 17 + ], + [ + 8, + 17 + ], + [ + 9, + 18 + ], + [ + 10, + 18 + ], + [ + 10, + 19 + ], + [ + 11, + 19 + ], + [ + 11, + 20 + ], + [ + 12, + 20 + ], + [ + 14, + 22 + ], + [ + 14, + 23 + ], + [ + 15, + 23 + ], + [ + 15, + 24 + ], + [ + 16, + 24 + ], + [ + 16, + 25 + ], + [ + 17, + 25 + ], + [ + 18, + 27 + ], + [ + 18, + 28 + ], + [ + 19, + 28 + ], + [ + 19, + 29 + ], + [ + 20, + 29 + ], + [ + 20, + 30 + ], + [ + 22, + 31 + ], + [ + 22, + 32 + ], + [ + 23, + 32 + ], + [ + 23, + 33 + ], + [ + 24, + 33 + ], + [ + 24, + 34 + ], + [ + 25, + 34 + ], + [ + 25, + 35 + ], + [ + 27, + 36 + ], + [ + 28, + 36 + ], + [ + 28, + 37 + ], + [ + 29, + 37 + ], + [ + 29, + 38 + ], + [ + 30, + 38 + ], + [ + 30, + 39 + ], + [ + 31, + 39 + ], + [ + 31, + 40 + ], + [ + 32, + 40 + ], + [ + 32, + 41 + ], + [ + 33, + 41 + ], + [ + 33, + 42 + ], + [ + 34, + 42 + ], + [ + 35, + 44 + ], + [ + 36, + 45 + ], + [ + 36, + 46 + ], + [ + 37, + 46 + ], + [ + 37, + 47 + ], + [ + 38, + 47 + ], + [ + 38, + 48 + ], + [ + 39, + 48 + ], + [ + 39, + 49 + ], + [ + 40, + 49 + ], + [ + 40, + 50 + ], + [ + 41, + 50 + ], + [ + 41, + 51 + ], + [ + 42, + 51 + ], + [ + 42, + 52 + ], + [ + 45, + 54 + ], + [ + 46, + 54 + ], + [ + 46, + 55 + ], + [ + 47, + 55 + ], + [ + 47, + 56 + ], + [ + 48, + 56 + ], + [ + 48, + 57 + ], + [ + 49, + 57 + ], + [ + 49, + 58 + ], + [ + 50, + 58 + ], + [ + 50, + 59 + ], + [ + 51, + 59 + ], + [ + 51, + 60 + ], + [ + 52, + 60 + ], + [ + 52, + 61 + ], + [ + 54, + 63 + ], + [ + 54, + 64 + ], + [ + 55, + 64 + ], + [ + 55, + 65 + ], + [ + 56, + 65 + ], + [ + 56, + 66 + ], + [ + 57, + 66 + ], + [ + 58, + 68 + ], + [ + 59, + 68 + ], + [ + 59, + 69 + ], + [ + 60, + 69 + ], + [ + 60, + 70 + ], + [ + 61, + 70 + ], + [ + 61, + 71 + ], + [ + 62, + 71 + ], + [ + 63, + 72 + ], + [ + 64, + 72 + ], + [ + 64, + 73 + ], + [ + 65, + 73 + ], + [ + 65, + 74 + ], + [ + 66, + 74 + ], + [ + 66, + 75 + ], + [ + 68, + 77 + ], + [ + 69, + 77 + ], + [ + 69, + 78 + ], + [ + 70, + 78 + ], + [ + 70, + 79 + ], + [ + 71, + 79 + ], + [ + 71, + 80 + ], + [ + 72, + 81 + ], + [ + 72, + 82 + ], + [ + 73, + 82 + ], + [ + 73, + 83 + ], + [ + 74, + 83 + ], + [ + 74, + 84 + ], + [ + 75, + 84 + ], + [ + 75, + 85 + ], + [ + 77, + 86 + ], + [ + 77, + 87 + ], + [ + 78, + 87 + ], + [ + 78, + 88 + ], + [ + 79, + 88 + ], + [ + 79, + 89 + ], + [ + 80, + 89 + ], + [ + 81, + 90 + ], + [ + 82, + 90 + ], + [ + 82, + 91 + ], + [ + 83, + 91 + ], + [ + 83, + 92 + ], + [ + 84, + 92 + ], + [ + 84, + 93 + ], + [ + 85, + 93 + ], + [ + 85, + 94 + ], + [ + 86, + 94 + ], + [ + 86, + 95 + ], + [ + 87, + 95 + ], + [ + 87, + 96 + ], + [ + 88, + 96 + ], + [ + 88, + 97 + ], + [ + 89, + 97 + ], + [ + 89, + 98 + ], + [ + 90, + 99 + ], + [ + 90, + 100 + ], + [ + 91, + 100 + ], + [ + 91, + 101 + ], + [ + 92, + 101 + ], + [ + 92, + 102 + ], + [ + 93, + 102 + ], + [ + 93, + 103 + ], + [ + 94, + 103 + ], + [ + 94, + 104 + ], + [ + 95, + 104 + ], + [ + 95, + 105 + ], + [ + 96, + 105 + ], + [ + 96, + 106 + ], + [ + 97, + 106 + ], + [ + 97, + 107 + ], + [ + 98, + 107 + ], + [ + 99, + 108 + ], + [ + 100, + 108 + ], + [ + 100, + 109 + ], + [ + 101, + 109 + ], + [ + 101, + 110 + ], + [ + 102, + 110 + ], + [ + 102, + 111 + ], + [ + 103, + 111 + ], + [ + 103, + 112 + ], + [ + 104, + 112 + ], + [ + 104, + 113 + ], + [ + 105, + 113 + ], + [ + 105, + 114 + ], + [ + 106, + 114 + ], + [ + 106, + 115 + ], + [ + 107, + 115 + ], + [ + 108, + 117 + ], + [ + 108, + 118 + ], + [ + 109, + 118 + ], + [ + 109, + 119 + ], + [ + 110, + 119 + ], + [ + 110, + 120 + ], + [ + 111, + 120 + ], + [ + 111, + 121 + ], + [ + 112, + 121 + ], + [ + 112, + 122 + ], + [ + 113, + 122 + ], + [ + 113, + 123 + ], + [ + 114, + 123 + ], + [ + 114, + 124 + ], + [ + 115, + 124 + ], + [ + 115, + 125 + ], + [ + 117, + 126 + ], + [ + 118, + 126 + ], + [ + 118, + 127 + ], + [ + 119, + 127 + ], + [ + 119, + 128 + ], + [ + 120, + 128 + ], + [ + 120, + 129 + ], + [ + 121, + 129 + ], + [ + 121, + 130 + ], + [ + 122, + 130 + ], + [ + 122, + 131 + ], + [ + 123, + 131 + ], + [ + 123, + 132 + ], + [ + 124, + 132 + ], + [ + 124, + 133 + ], + [ + 125, + 133 + ], + [ + 125, + 134 + ], + [ + 126, + 135 + ], + [ + 126, + 136 + ], + [ + 127, + 136 + ], + [ + 127, + 137 + ], + [ + 128, + 137 + ], + [ + 128, + 138 + ], + [ + 129, + 138 + ], + [ + 129, + 139 + ], + [ + 130, + 139 + ], + [ + 130, + 140 + ], + [ + 131, + 140 + ], + [ + 131, + 141 + ], + [ + 132, + 141 + ], + [ + 132, + 142 + ], + [ + 133, + 142 + ], + [ + 133, + 143 + ], + [ + 134, + 143 + ], + [ + 135, + 144 + ], + [ + 136, + 144 + ], + [ + 136, + 145 + ], + [ + 137, + 145 + ], + [ + 137, + 146 + ], + [ + 138, + 146 + ], + [ + 138, + 147 + ], + [ + 139, + 147 + ], + [ + 139, + 148 + ], + [ + 140, + 148 + ], + [ + 140, + 149 + ], + [ + 141, + 149 + ], + [ + 141, + 150 + ], + [ + 142, + 150 + ], + [ + 142, + 151 + ], + [ + 143, + 151 + ], + [ + 143, + 152 + ], + [ + 144, + 153 + ], + [ + 144, + 154 + ], + [ + 145, + 154 + ], + [ + 145, + 155 + ] + ] +} diff --git a/systems/quantum_db/control_plane/monitor/alert.py b/systems/quantum_db/control_plane/monitor/alert.py new file mode 100644 index 0000000..9fb12d0 --- /dev/null +++ b/systems/quantum_db/control_plane/monitor/alert.py @@ -0,0 +1,59 @@ +"""Stateless threshold alerts over a Monitor snapshot.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Literal + +Comparison = Literal[">=", "<="] +Severity = Literal["warning", "critical"] + + +@dataclass(frozen=True, slots=True) +class AlertRule: + metric: str + threshold: float + comparison: Comparison = ">=" + severity: Severity = "warning" + + +@dataclass(frozen=True, slots=True) +class Alert: + component: str + metric: str + value: float + threshold: float + severity: Severity + + +class AlertEvaluator: + def __init__(self, rules: list[AlertRule] | tuple[AlertRule, ...] = ()) -> None: + self.rules = tuple(rules) + + def evaluate(self, snapshot: dict[str, Any]) -> list[Alert]: + alerts: list[Alert] = [] + metrics = snapshot.get("metrics", {}) + if not isinstance(metrics, dict): + return alerts + for component, categories in metrics.items(): + if not isinstance(categories, dict): + continue + flattened = self._flatten(categories) + for rule in self.rules: + value = flattened.get(rule.metric) + if value is None: + continue + matched = value >= rule.threshold if rule.comparison == ">=" else value <= rule.threshold + if matched: + alerts.append(Alert(str(component), rule.metric, value, rule.threshold, rule.severity)) + return alerts + + @staticmethod + def _flatten(categories: dict[str, Any]) -> dict[str, float]: + values: dict[str, float] = {} + for metric_map in categories.values(): + if not isinstance(metric_map, dict): + continue + for name, aggregate in metric_map.items(): + if isinstance(aggregate, dict) and isinstance(aggregate.get("last"), (int, float)): + values[str(name)] = float(aggregate["last"]) + return values diff --git a/systems/quantum_db/control_plane/monitor/monitor.py b/systems/quantum_db/control_plane/monitor/monitor.py new file mode 100644 index 0000000..a2be2fd --- /dev/null +++ b/systems/quantum_db/control_plane/monitor/monitor.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from time import time +from typing import Any, Literal + +MetricCategory = Literal["resource", "performance", "custom"] +HealthLevel = Literal["healthy", "warning", "critical", "unknown"] + +_SEVERITY_RANK: dict[HealthLevel, int] = { + "unknown": 0, + "healthy": 1, + "warning": 2, + "critical": 3, +} + + +@dataclass(frozen=True) +class ResourceMetrics: + cpu_percent: float | None = None + memory_percent: float | None = None + io_read_mb_s: float | None = None + io_write_mb_s: float | None = None + net_rx_mb_s: float | None = None + net_tx_mb_s: float | None = None + + def as_dict(self) -> dict[str, float]: + out: dict[str, float] = {} + if self.cpu_percent is not None: + out["cpu_percent"] = float(self.cpu_percent) + if self.memory_percent is not None: + out["memory_percent"] = float(self.memory_percent) + if self.io_read_mb_s is not None: + out["io_read_mb_s"] = float(self.io_read_mb_s) + if self.io_write_mb_s is not None: + out["io_write_mb_s"] = float(self.io_write_mb_s) + if self.net_rx_mb_s is not None: + out["net_rx_mb_s"] = float(self.net_rx_mb_s) + if self.net_tx_mb_s is not None: + out["net_tx_mb_s"] = float(self.net_tx_mb_s) + return out + + +@dataclass(frozen=True) +class PerformanceMetrics: + latency_ms: float | None = None + queue_depth: float | None = None + throughput_qps: float | None = None + success_rate: float | None = None + error_rate: float | None = None + execution_ms: float | None = None + wait_ms: float | None = None + + def as_dict(self) -> dict[str, float]: + out: dict[str, float] = {} + if self.latency_ms is not None: + out["latency_ms"] = float(self.latency_ms) + if self.queue_depth is not None: + out["queue_depth"] = float(self.queue_depth) + if self.throughput_qps is not None: + out["throughput_qps"] = float(self.throughput_qps) + if self.success_rate is not None: + out["success_rate"] = float(self.success_rate) + if self.error_rate is not None: + out["error_rate"] = float(self.error_rate) + if self.execution_ms is not None: + out["execution_ms"] = float(self.execution_ms) + if self.wait_ms is not None: + out["wait_ms"] = float(self.wait_ms) + return out + + +@dataclass(frozen=True) +class CustomMetrics: + values: dict[str, float] = field(default_factory=dict) + + def as_dict(self) -> dict[str, float]: + return {name: float(value) for name, value in self.values.items()} + + +@dataclass(frozen=True) +class ComponentStatus: + state: str + updated_at: float + detail: str = "" + + +@dataclass +class _MetricAggregate: + count: int = 0 + total: float = 0.0 + min_value: float = float("inf") + max_value: float = float("-inf") + last_value: float = 0.0 + last_timestamp: float = 0.0 + + def add(self, value: float, ts: float) -> None: + self.count += 1 + self.total += value + if value < self.min_value: + self.min_value = value + if value > self.max_value: + self.max_value = value + self.last_value = value + self.last_timestamp = ts + + def snapshot(self) -> dict[str, float | int]: + if self.count == 0: + return { + "count": 0, + "avg": 0.0, + "min": 0.0, + "max": 0.0, + "last": 0.0, + "last_timestamp": 0.0, + } + return { + "count": self.count, + "avg": self.total / self.count, + "min": self.min_value, + "max": self.max_value, + "last": self.last_value, + "last_timestamp": self.last_timestamp, + } + + +@dataclass(frozen=True) +class SchedulerFeedback: + generated_at: float + global_health: HealthLevel + component_health: dict[str, HealthLevel] + anomaly_summary: tuple[str, ...] + + +class Monitor: + __slots__ = ( + "bus", + "started", + "_clock", + "_status", + "_metrics", + ) + + def __init__(self, bus: Any | None = None, clock: Any | None = None, **_: Any) -> None: + self.bus = bus + self.started = False + self._clock = ( + clock.time if clock is not None and hasattr(clock, "time") else time + ) + self._status: dict[str, ComponentStatus] = {} + self._metrics: dict[ + str, + dict[MetricCategory, dict[str, _MetricAggregate]], + ] = {} + + def start(self) -> None: + self.started = True + + def stop(self) -> None: + self.started = False + + def ingest( + self, + component: str, + *, + status: str | None = None, + detail: str = "", + resource: ResourceMetrics | None = None, + performance: PerformanceMetrics | None = None, + custom: CustomMetrics | dict[str, float] | None = None, + timestamp: float | None = None, + ) -> None: + ts = self._now(timestamp) + if status is not None: + self._status[component] = ComponentStatus( + state=status, + updated_at=ts, + detail=detail, + ) + if resource is not None: + self._ingest_metric_map(component, "resource", resource.as_dict(), ts) + if performance is not None: + self._ingest_metric_map(component, "performance", performance.as_dict(), ts) + if custom is not None: + metric_map = custom.as_dict() if isinstance(custom, CustomMetrics) else custom + self._ingest_metric_map(component, "custom", metric_map, ts) + + def report_metric( + self, + component: str, + category: MetricCategory, + name: str, + value: float, + *, + timestamp: float | None = None, + ) -> None: + ts = self._now(timestamp) + category_map = self._metrics.setdefault(component, {}).setdefault(category, {}) + aggregate = category_map.get(name) + if aggregate is None: + aggregate = _MetricAggregate() + category_map[name] = aggregate + aggregate.add(float(value), ts) + + def report_status( + self, + component: str, + state: str, + *, + detail: str = "", + timestamp: float | None = None, + ) -> None: + self._status[component] = ComponentStatus( + state=state, + updated_at=self._now(timestamp), + detail=detail, + ) + + def snapshot(self) -> dict[str, Any]: + generated_at = self._now(None) + global_health, component_health, anomalies = self._evaluate_health() + + metrics_snapshot: dict[str, dict[str, dict[str, dict[str, float | int]]]] = {} + for component, by_category in self._metrics.items(): + component_out: dict[str, dict[str, dict[str, float | int]]] = {} + for category, metric_map in by_category.items(): + component_out[category] = { + metric_name: aggregate.snapshot() + for metric_name, aggregate in metric_map.items() + } + metrics_snapshot[component] = component_out + + return { + "generated_at": generated_at, + "global_health": global_health, + "component_health": component_health, + "anomaly_summary": list(anomalies), + "status": { + component: { + "state": status.state, + "updated_at": status.updated_at, + "detail": status.detail, + } + for component, status in self._status.items() + }, + "metrics": metrics_snapshot, + } + + def feedback_for_scheduler(self) -> SchedulerFeedback: + global_health, component_health, anomalies = self._evaluate_health() + return SchedulerFeedback( + generated_at=self._now(None), + global_health=global_health, + component_health=component_health, + anomaly_summary=anomalies, + ) + + def _ingest_metric_map( + self, + component: str, + category: MetricCategory, + metrics: dict[str, float], + ts: float, + ) -> None: + category_map = self._metrics.setdefault(component, {}).setdefault(category, {}) + for metric_name, metric_value in metrics.items(): + aggregate = category_map.get(metric_name) + if aggregate is None: + aggregate = _MetricAggregate() + category_map[metric_name] = aggregate + aggregate.add(float(metric_value), ts) + + def _evaluate_health( + self, + ) -> tuple[HealthLevel, dict[str, HealthLevel], tuple[str, ...]]: + components = set(self._metrics) | set(self._status) + if not components: + return "unknown", {}, () + + component_health: dict[str, HealthLevel] = {} + anomalies: list[str] = [] + + for component in components: + level: HealthLevel = "healthy" + status = self._status.get(component) + if status is not None: + state = status.state.lower() + if state in {"failed", "down", "stopped", "unavailable"}: + level = "critical" + anomalies.append(f"{component}: state={state}") + elif state in {"degraded", "warning", "slow"}: + level = "warning" + anomalies.append(f"{component}: state={state}") + + by_category = self._metrics.get(component, {}) + for metric_map in by_category.values(): + for metric_name, aggregate in metric_map.items(): + metric_level = self._metric_level(metric_name, aggregate.last_value) + if metric_level is None: + continue + if _SEVERITY_RANK[metric_level] > _SEVERITY_RANK[level]: + level = metric_level + if metric_level != "healthy": + anomalies.append( + f"{component}: {metric_name}={aggregate.last_value:.4f} ({metric_level})" + ) + + component_health[component] = level + + global_health: HealthLevel = "healthy" + for level in component_health.values(): + if _SEVERITY_RANK[level] > _SEVERITY_RANK[global_health]: + global_health = level + + return global_health, component_health, tuple(anomalies) + + @staticmethod + def _metric_level(metric_name: str, value: float) -> HealthLevel | None: + if metric_name == "cpu_percent": + if value >= 90.0: + return "critical" + if value >= 75.0: + return "warning" + return "healthy" + if metric_name == "memory_percent": + if value >= 90.0: + return "critical" + if value >= 80.0: + return "warning" + return "healthy" + if metric_name in {"latency_ms", "execution_ms"}: + if value >= 1000.0: + return "critical" + if value >= 500.0: + return "warning" + return "healthy" + if metric_name == "wait_ms": + if value >= 2000.0: + return "critical" + if value >= 800.0: + return "warning" + return "healthy" + if metric_name == "queue_depth": + if value >= 1000.0: + return "critical" + if value >= 300.0: + return "warning" + return "healthy" + if metric_name == "error_rate": + if value >= 0.10: + return "critical" + if value >= 0.03: + return "warning" + return "healthy" + if metric_name == "success_rate": + if value <= 0.90: + return "critical" + if value <= 0.97: + return "warning" + return "healthy" + return None + + def _now(self, ts: float | None) -> float: + if ts is not None: + return float(ts) + return float(self._clock()) diff --git a/systems/quantum_db/control_plane/scheduler/dag.py b/systems/quantum_db/control_plane/scheduler/dag.py new file mode 100644 index 0000000..309822b --- /dev/null +++ b/systems/quantum_db/control_plane/scheduler/dag.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from collections import deque +from typing import Iterable, Mapping + + +class DAG: + """ + In-memory DAG for task dependencies. + Stores only graph structure; execution state is supplied externally. + """ + + __slots__ = ("_dependencies", "_dependents") + + def __init__(self) -> None: + # node -> ordered-set(dependencies) + self._dependencies: dict[str, dict[str, None]] = {} + # node -> ordered-set(dependents) + self._dependents: dict[str, dict[str, None]] = {} + + def add_node(self, node_id: str, dependencies: Iterable[str] = ()) -> None: + if not node_id: + raise ValueError("node_id must be non-empty") + + normalized_dependencies = tuple(dict.fromkeys(dependencies)) + if node_id in normalized_dependencies: + raise ValueError(f"Self dependency is not allowed: {node_id}") + + self._ensure_node(node_id) + old_dependencies = tuple(self._dependencies[node_id]) + + # Remove existing incoming edges for in-place dependency update. + for dependency in old_dependencies: + self._dependents[dependency].pop(node_id, None) + self._dependencies[node_id].clear() + + staged_dependencies: list[str] = [] + try: + for dependency in normalized_dependencies: + self._ensure_node(dependency) + if self._has_path(node_id, dependency): + raise ValueError( + f"Cycle detected when adding edge {dependency} -> {node_id}" + ) + staged_dependencies.append(dependency) + except Exception: + # Roll back to old edges. + for dependency in old_dependencies: + self._dependencies[node_id][dependency] = None + self._dependents[dependency][node_id] = None + raise + + for dependency in staged_dependencies: + self._dependencies[node_id][dependency] = None + self._dependents[dependency][node_id] = None + + def remove_node(self, node_id: str) -> bool: + if node_id not in self._dependencies: + return False + + for dependency in tuple(self._dependencies[node_id]): + self._dependents[dependency].pop(node_id, None) + for dependent in tuple(self._dependents[node_id]): + self._dependencies[dependent].pop(node_id, None) + + del self._dependencies[node_id] + del self._dependents[node_id] + return True + + def has_node(self, node_id: str) -> bool: + return node_id in self._dependencies + + def nodes(self) -> tuple[str, ...]: + return tuple(self._dependencies) + + def dependencies_of(self, node_id: str) -> tuple[str, ...]: + return tuple(self._dependencies[self._require_node(node_id)]) + + def dependents_of(self, node_id: str) -> tuple[str, ...]: + return tuple(self._dependents[self._require_node(node_id)]) + + def unresolved_dependencies( + self, node_id: str, completed: Iterable[str] | Mapping[str, object] + ) -> tuple[str, ...]: + node_id = self._require_node(node_id) + dependency_map = self._dependencies[node_id] + if not dependency_map: + return () + + if isinstance(completed, set): + done = completed + elif isinstance(completed, Mapping): + done = completed.keys() + else: + done = set(completed) + return tuple(dep for dep in dependency_map if dep not in done) + + def dependencies_satisfied( + self, node_id: str, completed: Iterable[str] | Mapping[str, object] + ) -> bool: + return not self.unresolved_dependencies(node_id, completed) + + def topological_sort(self, nodes: Iterable[str] | None = None) -> tuple[str, ...]: + if nodes is None: + ordered_nodes = tuple(self._dependencies) + else: + ordered_nodes = tuple(dict.fromkeys(nodes)) + for node_id in ordered_nodes: + self._require_node(node_id) + + if not ordered_nodes: + return () + + node_set = set(ordered_nodes) + indegree = {node_id: 0 for node_id in ordered_nodes} + for node_id in ordered_nodes: + for dependency in self._dependencies[node_id]: + if dependency in node_set: + indegree[node_id] += 1 + + queue = deque(node_id for node_id in ordered_nodes if indegree[node_id] == 0) + result: list[str] = [] + + while queue: + node_id = queue.popleft() + result.append(node_id) + for dependent in self._dependents[node_id]: + if dependent not in node_set: + continue + indegree[dependent] -= 1 + if indegree[dependent] == 0: + queue.append(dependent) + + if len(result) != len(ordered_nodes): + raise ValueError("Cycle detected in selected subgraph") + return tuple(result) + + def _ensure_node(self, node_id: str) -> None: + self._dependencies.setdefault(node_id, {}) + self._dependents.setdefault(node_id, {}) + + def _require_node(self, node_id: str) -> str: + if node_id not in self._dependencies: + raise KeyError(f"Unknown node: {node_id}") + return node_id + + def _has_path(self, start: str, target: str) -> bool: + if start == target: + return True + queue = deque((start,)) + seen = {start} + while queue: + node_id = queue.popleft() + for child in self._dependents.get(node_id, ()): + if child == target: + return True + if child not in seen: + seen.add(child) + queue.append(child) + return False diff --git a/systems/quantum_db/control_plane/scheduler/policy.py b/systems/quantum_db/control_plane/scheduler/policy.py new file mode 100644 index 0000000..da3bfc9 --- /dev/null +++ b/systems/quantum_db/control_plane/scheduler/policy.py @@ -0,0 +1,29 @@ +"""Pure scheduling advice derived from monitor feedback.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class SchedulingAdvice: + max_dispatch: int + priority_delta: int + prefer_classical_fallback: bool + reason: str + + +class AdaptiveSchedulingPolicy: + """Maps health feedback to advice; it never operates queues or executors.""" + + def advise(self, feedback: Any, *, requested_dispatch: int = 1) -> SchedulingAdvice: + requested = max(1, int(requested_dispatch)) + health = str(getattr(feedback, "global_health", "unknown")).lower() + anomalies = tuple(getattr(feedback, "anomaly_summary", ()) or ()) + if health == "critical": + return SchedulingAdvice(1, -10, True, "critical monitor health") + if health == "warning": + return SchedulingAdvice(max(1, requested // 2), -2, True, "warning monitor health") + if health == "healthy": + return SchedulingAdvice(requested, 0, False, "healthy monitor state") + return SchedulingAdvice(1, -1, True, "unknown monitor health" if not anomalies else "; ".join(map(str, anomalies))) diff --git a/systems/quantum_db/control_plane/scheduler/priority_queue.py b/systems/quantum_db/control_plane/scheduler/priority_queue.py new file mode 100644 index 0000000..65e2573 --- /dev/null +++ b/systems/quantum_db/control_plane/scheduler/priority_queue.py @@ -0,0 +1,51 @@ +"""Stable multi-priority queue usable by schedulers that need explicit requeueing.""" +from __future__ import annotations + +from heapq import heappop, heappush +from itertools import count +from typing import Any + + +class PriorityQueue: + __slots__ = ("_heap", "_entries", "_sequence", "_front_sequence") + + def __init__(self) -> None: + self._heap: list[tuple[int, int, str, Any]] = [] + self._entries: dict[str, tuple[int, int, str, Any]] = {} + self._sequence = count() + self._front_sequence = 0 + + def push(self, item_id: str, payload: Any = None, *, priority: int = 0, front: bool = False) -> None: + if not item_id: + raise ValueError("item_id must be non-empty") + self.remove(item_id) + if front: + self._front_sequence -= 1 + sequence = self._front_sequence + else: + sequence = next(self._sequence) + entry = (-int(priority), sequence, item_id, payload) + self._entries[item_id] = entry + heappush(self._heap, entry) + + def requeue(self, item_id: str, *, priority: int | None = None, front: bool = True) -> None: + entry = self._entries.get(item_id) + if entry is None: + raise KeyError(item_id) + self.push(item_id, entry[3], priority=-entry[0] if priority is None else priority, front=front) + + def remove(self, item_id: str) -> bool: + return self._entries.pop(item_id, None) is not None + + def pop(self) -> tuple[str, Any] | None: + while self._heap: + entry = heappop(self._heap) + item_id = entry[2] + if self._entries.get(item_id) != entry: + continue + del self._entries[item_id] + return item_id, entry[3] + return None + + def __len__(self) -> int: + return len(self._entries) diff --git a/systems/quantum_db/control_plane/scheduler/scheduler.py b/systems/quantum_db/control_plane/scheduler/scheduler.py new file mode 100644 index 0000000..5410481 --- /dev/null +++ b/systems/quantum_db/control_plane/scheduler/scheduler.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from heapq import heappop, heappush +from itertools import count +from typing import Any, Protocol + +from control_plane.scheduler.state_store import SchedulerStateStore + +PENDING = "pending" +RUNNING = "running" +FINISHED = "finished" +FAILED = "failed" +CANCELLED = "cancelled" +TERMINAL_STATES = {FINISHED, FAILED, CANCELLED} + + +@dataclass(frozen=True) +class Task: + task_id: str + priority: int = 0 + dependencies: tuple[str, ...] = () + payload: Any = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class TaskRecord: + task: Task + state: str + sequence: int + submitted_at: float + + +class DependencyGraph(Protocol): + def add(self, task_id: str, dependencies: tuple[str, ...]) -> bool: ... + + def mark_done(self, task_id: str) -> tuple[str, ...]: ... + + +class ReadyQueue(Protocol): + def push(self, task_id: str, priority: int, sequence: int) -> None: ... + + def pop_many(self, limit: int) -> list[str]: ... + + +class SchedulePolicy(Protocol): + def choose(self, task_ids: list[str], records: dict[str, TaskRecord]) -> list[str]: ... + + +class _DefaultDependencyGraph: + __slots__ = ("_children", "_remaining") + + def __init__(self) -> None: + self._children: dict[str, list[str]] = {} + self._remaining: dict[str, int] = {} + + def add(self, task_id: str, dependencies: tuple[str, ...]) -> bool: + dep_count = len(dependencies) + self._remaining[task_id] = dep_count + if dep_count == 0: + return True + for dependency in dependencies: + self._children.setdefault(dependency, []).append(task_id) + return False + + def mark_done(self, task_id: str) -> tuple[str, ...]: + unlocked: list[str] = [] + for child_id in self._children.pop(task_id, ()): # parent entry can be absent + remaining = self._remaining.get(child_id, 0) - 1 + self._remaining[child_id] = remaining + if remaining == 0: + unlocked.append(child_id) + return tuple(unlocked) + + +class _DefaultReadyQueue: + __slots__ = ("_heap",) + + def __init__(self) -> None: + self._heap: list[tuple[int, int, str]] = [] + + def push(self, task_id: str, priority: int, sequence: int) -> None: + heappush(self._heap, (-priority, sequence, task_id)) + + def pop_many(self, limit: int) -> list[str]: + out: list[str] = [] + while self._heap and len(out) < limit: + _, _, task_id = heappop(self._heap) + out.append(task_id) + return out + + +class _DefaultPolicy: + __slots__ = () + + def choose(self, task_ids: list[str], records: dict[str, TaskRecord]) -> list[str]: + return task_ids + + +class Scheduler: + __slots__ = ( + "_records", + "_graph", + "_queue", + "_policy", + "_clock", + "_sequence", + "_state_store", + ) + + def __init__( + self, + graph: DependencyGraph | None = None, + ready_queue: ReadyQueue | None = None, + policy: SchedulePolicy | None = None, + clock: Any | None = None, + ) -> None: + self._records: dict[str, TaskRecord] = {} + self._graph = graph or _DefaultDependencyGraph() + self._queue = ready_queue or _DefaultReadyQueue() + self._policy = policy or _DefaultPolicy() + self._clock = clock + self._sequence = count() + self._state_store = SchedulerStateStore() + + def submit( + self, + task_id: str, + *, + priority: int = 0, + dependencies: tuple[str, ...] | list[str] = (), + payload: Any = None, + metadata: dict[str, Any] | None = None, + ) -> Task: + if task_id in self._records: + raise ValueError(f"Task already exists: {task_id}") + + normalized_dependencies = tuple(dict.fromkeys(dependencies)) + task = Task( + task_id=task_id, + priority=priority, + dependencies=normalized_dependencies, + payload=payload, + metadata=metadata or {}, + ) + sequence = next(self._sequence) + submitted_at = ( + self._clock.time() + if self._clock is not None and hasattr(self._clock, "time") + else 0.0 + ) + self._records[task_id] = TaskRecord( + task=task, + state=PENDING, + sequence=sequence, + submitted_at=submitted_at, + ) + self._state_store.set_state(task_id, PENDING) + if self._graph.add(task_id, normalized_dependencies): + self._queue.push(task_id, priority, sequence) + return task + + def dispatch(self, limit: int = 1) -> list[Task]: + if limit <= 0: + return [] + + candidate_ids = self._queue.pop_many(limit) + chosen_ids = self._policy.choose(candidate_ids, self._records) + dispatched: list[Task] = [] + + for task_id in chosen_ids: + record = self._records.get(task_id) + if record is None or record.state != PENDING: + continue + record.state = RUNNING + self._state_store.set_state(task_id, RUNNING) + dispatched.append(record.task) + return dispatched + + def finish(self, task_id: str, *, success: bool = True) -> tuple[str, ...]: + record = self._records.get(task_id) + if record is None: + raise KeyError(f"Unknown task: {task_id}") + if record.state not in {RUNNING, PENDING}: + return () + + record.state = FINISHED if success else FAILED + self._state_store.set_state(task_id, record.state) + if not success: + return () + + unlocked = self._graph.mark_done(task_id) + for unlocked_task_id in unlocked: + unlocked_record = self._records.get(unlocked_task_id) + if unlocked_record is None or unlocked_record.state != PENDING: + continue + self._queue.push( + unlocked_task_id, + unlocked_record.task.priority, + unlocked_record.sequence, + ) + return unlocked + + def cancel(self, task_id: str) -> bool: + record = self._records.get(task_id) + if record is None or record.state in TERMINAL_STATES: + return False + record.state = CANCELLED + self._state_store.set_state(task_id, CANCELLED) + return True + + def execution_backend_for(self, task_id: str) -> str: + """Execution 后段部署决定：由 control plane scheduler 统一裁决。""" + record = self._records.get(task_id) + if record is None: + raise KeyError(f"Unknown task: {task_id}") + backend = str(record.task.metadata.get("execution_backend", "cpu")) + self._state_store.log(task_id, "execution_backend_for", {"backend": backend}) + return backend + + def state_logs(self, task_id: str | None = None) -> list[Any]: + return self._state_store.get_logs(task_id=task_id) + + def recycle(self, limit: int | None = None) -> int: + reclaimed = 0 + for task_id, record in tuple(self._records.items()): + if record.state not in TERMINAL_STATES: + continue + del self._records[task_id] + reclaimed += 1 + if limit is not None and reclaimed >= limit: + break + return reclaimed + + def state_of(self, task_id: str) -> str: + record = self._records.get(task_id) + if record is None: + raise KeyError(f"Unknown task: {task_id}") + return record.state + + def size(self) -> int: + return len(self._records) diff --git a/systems/quantum_db/control_plane/scheduler/state_store.py b/systems/quantum_db/control_plane/scheduler/state_store.py new file mode 100644 index 0000000..7800939 --- /dev/null +++ b/systems/quantum_db/control_plane/scheduler/state_store.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class StateEvent: + task_id: str + action: str + payload: dict[str, Any] = field(default_factory=dict) + timestamp_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +class SchedulerStateStore: + """调度状态存储 + 调用日志记录。""" + + __slots__ = ("_state", "_logs") + + def __init__(self) -> None: + self._state: dict[str, str] = {} + self._logs: list[StateEvent] = [] + + def set_state(self, task_id: str, state: str) -> None: + self._state[task_id] = state + self.log(task_id, "set_state", {"state": state}) + + def get_state(self, task_id: str) -> str | None: + state = self._state.get(task_id) + self.log(task_id, "get_state", {"state": state}) + return state + + def log(self, task_id: str, action: str, payload: dict[str, Any] | None = None) -> None: + self._logs.append(StateEvent(task_id=task_id, action=action, payload=payload or {})) + + def get_logs(self, *, task_id: str | None = None) -> list[StateEvent]: + if task_id is None: + return list(self._logs) + return [event for event in self._logs if event.task_id == task_id] + + def summary(self) -> dict[str, Any]: + return { + "tasks": dict(self._state), + "log_size": len(self._logs), + } diff --git a/systems/quantum_db/control_plane/semantic/IR_builder.py b/systems/quantum_db/control_plane/semantic/IR_builder.py new file mode 100644 index 0000000..4bf7e25 --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/IR_builder.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import time +import uuid +from typing import Any, Mapping + +_REQUEST_TYPES: frozenset[str] = frozenset(("query", "analysis", "control", "admin")) + +_TASK_KIND_BY_REQUEST: dict[str, str] = { + "query": "read", + "analysis": "analyze", + "control": "mutate", + "admin": "governance", +} + +# Route hints only describe capabilities/preferences and avoid binding to any concrete executor. +_ROUTE_HINTS_BY_REQUEST: dict[str, tuple[str, ...]] = { + "query": ("low_latency", "tn_candidate", "hybrid_allowed"), + "analysis": ("throughput", "hybrid_preferred", "quantum_candidate"), + "control": ("ordered_execution", "state_transition", "tn_candidate"), + "admin": ("audit_required", "privileged", "tn_candidate"), +} + +_PRIORITY_ALIASES: dict[str, str] = { + "p0": "urgent", + "p1": "high", + "p2": "normal", + "p3": "low", +} + + +def build_ir(request: Any, *, now_ms: int | None = None) -> dict[str, Any]: + """ + Build a compact internal IR from parsed request data. + + The returned structure is intentionally JSON-serializable and execution-engine agnostic. + """ + request_id = _to_optional_str(_read_field(request, "request_id")) or uuid.uuid4().hex + user_id = _to_optional_str(_read_field(request, "user_id")) or "" + raw_text = _to_optional_str(_read_field(request, "raw_text")) or "" + + params = _to_plain_dict(_read_field(request, "structured_params")) + request_type = _normalize_request_type( + _to_optional_str(_read_field(request, "request_type")) + or _to_optional_str(params.get("request_type")) + ) + + session_id = ( + _to_optional_str(_read_field(request, "session_id")) + or _to_optional_str(params.get("session_id")) + or _to_optional_str(params.get("session")) + ) + + deadline_ms = _to_optional_int(_read_field(request, "deadline_ms")) + if deadline_ms is None: + deadline_ms = _to_optional_int(params.get("deadline_ms")) + if deadline_ms is None: + deadline_ms = _to_optional_int(params.get("deadline")) + if deadline_ms is None: + deadline_ms = _to_optional_int(params.get("timeout_ms")) + + priority_hint = ( + _to_optional_str(_read_field(request, "priority_hint")) + or _to_optional_str(params.get("priority_hint")) + or _to_optional_str(params.get("priority")) + ) + priority_hint = _normalize_priority(priority_hint) + + created_at_ms = int(time.time() * 1000) if now_ms is None else int(now_ms) + ir_id = f"ir_{request_id}" + task_id = f"task_{request_id}" + route_hints = list(_ROUTE_HINTS_BY_REQUEST.get(request_type, ("tn_candidate",))) + + constraints: dict[str, Any] = {} + if deadline_ms is not None: + constraints["deadline_ms"] = deadline_ms + if priority_hint is not None: + constraints["priority_hint"] = priority_hint + + metadata: dict[str, Any] = {"created_at_ms": created_at_ms} + if session_id is not None: + metadata["session_id"] = session_id + + return { + "ir_version": 1, + "ir_id": ir_id, + "request": { + "request_id": request_id, + "user_id": user_id, + "request_type": request_type, + }, + "tasks": [ + { + "task_id": task_id, + "kind": _TASK_KIND_BY_REQUEST[request_type], + "description": raw_text, + "params": params, + "constraints": constraints, + "route_hints": route_hints, + "depends_on": [], + } + ], + "metadata": metadata, + } + + +def _read_field(source: Any, name: str) -> Any: + if isinstance(source, Mapping): + return source.get(name) + return getattr(source, name, None) + + +def _to_plain_dict(value: Any) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, dict): + return value.copy() + if isinstance(value, Mapping): + return dict(value) + return {} + + +def _normalize_request_type(value: str | None) -> str: + if value is None: + return "query" + lowered = value.strip().casefold() + return lowered if lowered in _REQUEST_TYPES else "query" + + +def _normalize_priority(value: str | None) -> str | None: + if value is None: + return None + lowered = value.strip().casefold() + return _PRIORITY_ALIASES.get(lowered, lowered) or None + + +def _to_optional_str(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _to_optional_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + text = value.strip() + if text and (text.isdigit() or (text[0] in "+-" and text[1:].isdigit())): + try: + return int(text) + except ValueError: + return None + return None diff --git a/systems/quantum_db/control_plane/semantic/experiment_docs.py b/systems/quantum_db/control_plane/semantic/experiment_docs.py new file mode 100644 index 0000000..7848b37 --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/experiment_docs.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +class ExperimentDocReader: + """部署在 control plane，经 bus 解耦访问实验数据文档。""" + + __slots__ = ("root",) + + def __init__(self, root: str = "train-test-data") -> None: + self.root = Path(root) + + def read(self, relative_path: str) -> dict[str, Any]: + path = (self.root / relative_path).resolve() + if not path.exists(): + raise FileNotFoundError(f"experiment doc not found: {relative_path}") + text = path.read_text(encoding="utf-8", errors="ignore") + return {"path": str(path), "size": len(text), "preview": text[:256]} + + def execute(self, action: str, payload: Any, metadata: dict[str, Any] | None = None) -> Any: + data = payload if isinstance(payload, dict) else {} + if action == "read": + return self.read(str(data.get("path", ""))) + raise ValueError(f"unsupported experiment-doc action: {action}") diff --git a/systems/quantum_db/control_plane/semantic/normalizer.py b/systems/quantum_db/control_plane/semantic/normalizer.py new file mode 100644 index 0000000..3bf3a20 --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/normalizer.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +@dataclass(slots=True) +class SemanticIR: + requested_alu: str | None = None + allow_auto_fallback: bool = True + structure_prior: str | None = None + warnings: list[str] = field(default_factory=list) + + +def normalize_to_ir(params: Mapping[str, Any], *, sample_rows: list[list[float]] | None = None) -> SemanticIR: + requested_alu = _to_optional_str(params.get("requested_alu")) + allow_auto_fallback = _to_bool(params.get("allow_auto_fallback"), default=True) + structure_prior = _normalize_structure_prior(params.get("structure_prior")) + + if structure_prior is None: + structure_prior = _infer_structure_from_alu(requested_alu) + + warnings: list[str] = [] + if requested_alu == "wsym" and not _is_exchange_symmetric(sample_rows or []): + warnings.append("requested_alu=wsym 需要交换对称数据，但当前样本不满足交换对称性。") + + return SemanticIR( + requested_alu=requested_alu, + allow_auto_fallback=allow_auto_fallback, + structure_prior=structure_prior, + warnings=warnings, + ) + + +def _to_optional_str(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _to_bool(value: Any, *, default: bool) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + lowered = str(value).strip().casefold() + if lowered in {"1", "true", "yes", "y", "on"}: + return True + if lowered in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _normalize_structure_prior(value: Any) -> str | None: + if value is None: + return None + lowered = str(value).strip().casefold() + aliases = { + "local": "local", + "tree": "tree", + "sym": "symmetry", + "symmetric": "symmetry", + "symmetry": "symmetry", + "constraint": "constraint", + } + return aliases.get(lowered) + + +def _infer_structure_from_alu(alu: str | None) -> str | None: + return { + "wloc": "local", + "wbrick": "local", + "wtree": "tree", + "wstar": "constraint", + "wsym": "symmetry", + }.get(alu) + + +def _is_exchange_symmetric(rows: list[list[float]]) -> bool: + if not rows: + return True + if any(len(row) != len(rows[0]) for row in rows): + return False + width = len(rows[0]) + if width <= 1: + return True + + col_means = [sum(row[i] for row in rows) / len(rows) for i in range(width)] + baseline = sum(col_means) / width + max_dev = max(abs(mean - baseline) for mean in col_means) + return max_dev <= 1e-6 diff --git a/systems/quantum_db/control_plane/semantic/parser.py b/systems/quantum_db/control_plane/semantic/parser.py new file mode 100644 index 0000000..dc635af --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/parser.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import Any, Mapping + +REQUEST_TYPES: tuple[str, ...] = ("query", "analysis", "control", "admin") + +_KV_PATTERN = re.compile(r"([A-Za-z_][A-Za-z0-9_-]*)\s*[:=]\s*([^,;\n]+)") +_WS_PATTERN = re.compile(r"\s+") + +_TYPE_KEYWORDS: dict[str, tuple[str, ...]] = { + "admin": ( + "admin", + "permission", + "role", + "quota", + "tenant", + "audit", + "policy", + "权限", + "配额", + "租户", + ), + "control": ( + "start", + "stop", + "pause", + "resume", + "restart", + "cancel", + "enable", + "disable", + "switch", + "configure", + "set", + "启动", + "停止", + "暂停", + "恢复", + "取消", + "配置", + ), + "analysis": ( + "analyze", + "analysis", + "diagnose", + "why", + "reason", + "root cause", + "benchmark", + "profile", + "compare", + "forecast", + "分析", + "诊断", + "原因", + "对比", + ), + "query": ("query", "find", "search", "get", "show", "list", "查", "查询", "检索"), +} + +_ALU_SYNONYMS: dict[str, tuple[str, ...]] = { + "wloc": ("local orthogonal", "rebase", "local"), + "wbrick": ("brickwall", "two-site", "two site"), + "wtree": ("tree", "ht", "ttn"), + "wstar": ("rydberg star", "constraint", "star"), + "wsym": ("symmetric", "dicke", "weight"), +} + + +@dataclass +class UserRequest: + request_id: str + user_id: str + request_type: str + raw_text: str + structured_params: dict[str, Any] = field(default_factory=dict) + session_id: str | None = None + deadline_ms: int | None = None + priority_hint: str | None = None + + +def parse_user_request( + raw_text: str, + user_id: str, + *, + request_id: str | None = None, + request_type: str | None = None, + structured_params: Mapping[str, Any] | None = None, + session_id: str | None = None, + deadline_ms: int | None = None, + priority_hint: str | None = None, +) -> UserRequest: + text = _normalize_text(raw_text) + params: dict[str, Any] = dict(structured_params or {}) + parsed_payload = _extract_payload(text) + if parsed_payload: + params.update(parsed_payload) + if "structure_prior" not in params: + inferred = _infer_structure_prior(params.get("requested_alu")) + if inferred is not None: + params["structure_prior"] = inferred + + resolved_type = _resolve_request_type(text, request_type, params) + + if session_id is None: + session_id = _to_optional_str(params.get("session_id") or params.get("session")) + if deadline_ms is None: + deadline_ms = _to_optional_int( + params.get("deadline_ms") or params.get("deadline") or params.get("timeout_ms") + ) + if priority_hint is None: + priority_hint = _to_optional_str(params.get("priority_hint") or params.get("priority")) + + return UserRequest( + request_id=request_id or uuid.uuid4().hex, + user_id=user_id, + request_type=resolved_type, + raw_text=text, + structured_params=params, + session_id=session_id, + deadline_ms=deadline_ms, + priority_hint=priority_hint, + ) + + +def _normalize_text(raw_text: str) -> str: + return _WS_PATTERN.sub(" ", raw_text).strip() + + +def _extract_payload(text: str) -> dict[str, Any]: + if not text: + return {} + + first = text[0] + if first == "{" and text[-1] == "}": + try: + obj = json.loads(text) + if isinstance(obj, dict): + return obj + except json.JSONDecodeError: + pass + + out: dict[str, Any] = {} + for key, raw in _KV_PATTERN.findall(text): + out[key] = _parse_scalar(raw.strip()) + + alu = _extract_requested_alu(text) + if alu is not None and "requested_alu" not in out: + out["requested_alu"] = alu + + auto_fallback = _extract_allow_auto_fallback(text) + if auto_fallback is not None and "allow_auto_fallback" not in out: + out["allow_auto_fallback"] = auto_fallback + + if "structure_prior" not in out: + out["structure_prior"] = _infer_structure_prior(out.get("requested_alu")) + return out + + +def _extract_requested_alu(text: str) -> str | None: + lowered = text.casefold() + matched: list[tuple[int, str]] = [] + for alu_name, tokens in _ALU_SYNONYMS.items(): + for token in tokens: + pos = lowered.find(token) + if pos >= 0: + matched.append((pos, alu_name)) + break + if not matched: + return None + matched.sort(key=lambda item: item[0]) + return matched[0][1] + + +def _extract_allow_auto_fallback(text: str) -> bool | None: + lowered = text.casefold() + if any(token in lowered for token in ("disable auto fallback", "no fallback", "forbid fallback", "strict")): + return False + if any(token in lowered for token in ("allow auto fallback", "auto fallback", "fallback ok", "fallback allowed")): + return True + return None + + +def _infer_structure_prior(requested_alu: Any) -> str | None: + if not isinstance(requested_alu, str): + return None + table = { + "wloc": "local", + "wbrick": "local", + "wtree": "tree", + "wstar": "constraint", + "wsym": "symmetry", + } + return table.get(requested_alu) + + +def _parse_scalar(raw: str) -> Any: + if not raw: + return raw + if (raw[0] == raw[-1]) and raw[0] in ("'", '"') and len(raw) >= 2: + return raw[1:-1] + + lower = raw.casefold() + if lower == "true": + return True + if lower == "false": + return False + if lower in ("none", "null"): + return None + + if raw.isdigit() or (raw[0] in "+-" and raw[1:].isdigit()): + try: + return int(raw) + except ValueError: + return raw + + try: + return float(raw) + except ValueError: + return raw + + +def _resolve_request_type( + text: str, explicit_type: str | None, params: Mapping[str, Any] +) -> str: + if explicit_type in REQUEST_TYPES: + return explicit_type + + param_type = params.get("request_type") + if isinstance(param_type, str) and param_type in REQUEST_TYPES: + return param_type + + lowered = text.casefold() + scores = { + kind: sum(1 for token in tokens if token in lowered) + for kind, tokens in _TYPE_KEYWORDS.items() + } + scores["query"] += 1 + + return max(REQUEST_TYPES, key=lambda item: scores.get(item, 0)) + + +def _to_optional_str(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _to_optional_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + value = value.strip() + if value and (value.isdigit() or (value[0] in "+-" and value[1:].isdigit())): + try: + return int(value) + except ValueError: + return None + return None diff --git a/systems/quantum_db/control_plane/semantic/router.py b/systems/quantum_db/control_plane/semantic/router.py new file mode 100644 index 0000000..8ea8959 --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/router.py @@ -0,0 +1,5 @@ +"""Compatibility facade for the concrete Quantum DB route policy.""" + +from qdb.router import QueryRouter, RouteDecision + +__all__ = ["QueryRouter", "RouteDecision"] diff --git a/systems/quantum_db/control_plane/semantic/tn_page_experiment.py b/systems/quantum_db/control_plane/semantic/tn_page_experiment.py new file mode 100644 index 0000000..c5f217d --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/tn_page_experiment.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import argparse +import csv +import json +import time +from pathlib import Path +from typing import Any + +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig, suggest_workload_config + + +DEFAULT_DATASET = "cols_8_distinct_10000_corr_8_skew_15" +DEFAULT_FILE = "train-test-data/SyntheticDataset/cols_8_distinct_10000_corr_8_skew_15.csv" + + +def load_samples(csv_path: Path, *, limit: int | None = None) -> list[list[float]]: + rows: list[list[float]] = [] + with csv_path.open("r", encoding="utf-8") as f: + reader = csv.reader(f) + for row in reader: + if not row: + continue + try: + rows.append([float(cell) for cell in row]) + except ValueError: + # skip header/non-numeric rows + continue + if limit is not None and len(rows) >= limit: + break + if not rows: + raise ValueError(f"no numeric sample rows found in {csv_path}") + return rows + + +def estimate_storage_usage(page: Any) -> dict[str, int]: + tensor_count = 0 + scalar_count = 0 + for tensor in page.tensors: + tensor_count += 1 + for left in tensor: + for phys in left: + scalar_count += len(phys) + # float64 bytes + return { + "tensor_count": tensor_count, + "scalar_count": scalar_count, + "estimated_bytes": scalar_count * 8, + } + + +def run_experiment( + *, + dataset: str, + csv_path: Path, + num_sites: int, + phys_dim: int, + chi_max: int, + keep_ratio: float, + sample_limit: int, + sample_count: int, + learning_mode: str, +) -> dict[str, Any]: + process_log: list[dict[str, Any]] = [] + + def log(stage: str, **payload: Any) -> None: + process_log.append({"timestamp_ms": int(time.time() * 1000), "stage": stage, **payload}) + + log("load_samples.start", file=str(csv_path), limit=sample_limit) + samples = load_samples(csv_path, limit=sample_limit) + log("load_samples.done", rows=len(samples), cols=len(samples[0])) + + if learning_mode == "auto": + cfg = suggest_workload_config(samples) + log( + "config.auto_suggested", + config={ + "num_sites": cfg.num_sites, + "phys_dim": cfg.phys_dim, + "chi_max": cfg.chi_max, + "keep_ratio": cfg.keep_ratio, + "alu_components": list(cfg.alu_components), + }, + ) + else: + cfg = WorkloadConfig(num_sites=num_sites, phys_dim=phys_dim, chi_max=chi_max, keep_ratio=keep_ratio) + service = TNWorkloadService(config=cfg) + log("tn_page.create.start", dataset=dataset, config={"num_sites": cfg.num_sites, "phys_dim": cfg.phys_dim, "chi_max": cfg.chi_max, "keep_ratio": cfg.keep_ratio, "alu_components": list(cfg.alu_components)}) + page = service.create_page(dataset=dataset, samples=samples) + log("tn_page.create.done", page_id=page.page_id, summary=page.summary()) + + sampled = service.sampling(page, num_samples=sample_count) + fidelity = service.fidelity(samples, sampled) + compression_ratio = service.compression_ratio(page, samples) + storage_usage = estimate_storage_usage(page) + version_info = service.page_manager.version_of(page.page_id) + + log("metrics.calculated", fidelity=fidelity, compression_ratio=compression_ratio, storage_usage=storage_usage) + log("tn_page.version", version=version_info.version, updated_at_ms=version_info.updated_at_ms) + + return { + "dataset": dataset, + "data_file": str(csv_path), + "config": { + "learning_mode": learning_mode, + "num_sites": cfg.num_sites, + "phys_dim": cfg.phys_dim, + "chi_max": cfg.chi_max, + "keep_ratio": cfg.keep_ratio, + "alu_components": list(cfg.alu_components), + }, + "metrics": { + "fidelity": fidelity, + "compression_ratio": compression_ratio, + "storage_usage": storage_usage, + }, + "page": service.page_manager.describe(page.page_id), + "process_log": process_log, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="TN-page 实验：构建并评估 fidelity / storage usage / compression ratio") + parser.add_argument("--dataset", default=DEFAULT_DATASET) + parser.add_argument("--csv", default=DEFAULT_FILE) + parser.add_argument("--num-sites", type=int, default=16) + parser.add_argument("--phys-dim", type=int, default=8) + parser.add_argument("--chi-max", type=int, default=16) + parser.add_argument("--keep-ratio", type=float, default=0.25) + parser.add_argument("--sample-limit", type=int, default=1024) + parser.add_argument("--sample-count", type=int, default=128) + parser.add_argument( + "--learning-mode", + choices=("manual", "auto"), + default="manual", + help="manual: 使用命令行参数; auto: 根据样本自动建议配置", + ) + parser.add_argument("--log-out", default="", help="可选：将完整结果JSON写入文件") + args = parser.parse_args() + + result = run_experiment( + dataset=args.dataset, + csv_path=Path(args.csv), + num_sites=args.num_sites, + phys_dim=args.phys_dim, + chi_max=args.chi_max, + keep_ratio=args.keep_ratio, + sample_limit=args.sample_limit, + sample_count=args.sample_count, + learning_mode=args.learning_mode, + ) + + print(json.dumps(result, ensure_ascii=False, indent=2)) + if args.log_out: + Path(args.log_out).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/control_plane/semantic/workload_semantic.py b/systems/quantum_db/control_plane/semantic/workload_semantic.py new file mode 100644 index 0000000..1a29b8f --- /dev/null +++ b/systems/quantum_db/control_plane/semantic/workload_semantic.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig, q_error +from control_plane.semantic.normalizer import SemanticIR + + +@dataclass +class SemanticExperimentConfig: + data_dir: Path + limit: int = 512 + allow_synthetic_fallback: bool = False + + +class TNSemanticOrchestrator: + __slots__ = ("service", "config") + + def __init__(self, *, service: TNWorkloadService, config: SemanticExperimentConfig) -> None: + self.service = service + self.config = config + + def apply_ir(self, ir: SemanticIR) -> None: + self.service = TNWorkloadService( + config=WorkloadConfig( + num_sites=self.service.config.num_sites, + phys_dim=self.service.config.phys_dim, + chi_max=self.service.config.chi_max, + keep_ratio=self.service.config.keep_ratio, + requested_alu=ir.requested_alu, + allow_auto_fallback=ir.allow_auto_fallback, + structure_prior=ir.structure_prior, + ) + ) + + def run_dataset( + self, + dataset: str, + samples: list[list[float]], + files: list[str], + source: str, + *, + ir: SemanticIR | None = None, + ) -> dict[str, Any]: + if ir is not None: + self.apply_ir(ir) + + page = self.service.create_page(dataset=dataset, samples=samples) + bonds = list(page.metadata.bond_dims) + chi = max(1, self.service.config.chi_max) + + candidate_scores = { + "dense_linear": max(0.05, min(0.95, (len(samples[0]) if samples else 1) / max(4.0, self.service.config.num_sites))), + "mps_chain": max(0.05, min(0.99, 0.35 + (sum(bonds) / max(1.0, float(len(bonds) * chi))) * 0.5)), + "tree_tn": max(0.05, min(0.90, 0.25 + (len(set(bonds)) / max(1.0, float(len(bonds)))) * 0.45)), + } + chosen_alu = max(candidate_scores, key=candidate_scores.get) + process_log.append( + { + "stage": "alu_selection", + "chosen_alu": chosen_alu, + "candidate_scores": candidate_scores, + "selection_reason": "mps_chain scored highest under bond-utilization and feature-width heuristics" + if chosen_alu == "mps_chain" + else f"{chosen_alu} scored highest under heuristic hardware fit", + } + ) + + sampled = self.service.distribution_sampling(page, num_samples=64) + agg = self.service.approximate_aggregate(samples) + pred = lambda x: sum(x[: min(3, len(x))]) > 0 + sel = self.service.multi_attribute_selectivity(samples, pred) + like = self.service.likelihood_estimation(page, samples[0]) + sim = self.service.similarity(samples[0], samples[min(1, len(samples) - 1)]) + + compression = self.service.compression_ratio(page, samples) + recon = self.service.reconstruction_error_l1(samples, sampled) + fidelity = self.service.fidelity(samples, sampled) + marginal_fidelity = self.service.marginal_fidelity(samples, sampled) + joint_fidelity = self.service.joint_fidelity(samples, sampled, phys_dim=self.service.config.phys_dim) + cross_col = self.service.cross_column_dependency(samples) + hit = sum(1 for b in bonds if b >= chi) + util_mean = (sum(b / chi for b in bonds) / len(bonds)) if bonds else 0.0 + + num_atoms = max(1, self.service.config.num_sites) + num_2q_gates = int(sum(max(1, b) for b in bonds)) + num_1q_gates = int(num_atoms * self.service.config.phys_dim) + num_k_body_ops = int(max(1, len(set(bonds))) * max(1, num_atoms // 4)) + depth_estimate = int(max(1, len(bonds)) + max(1, chi // 2)) + parallel_layers = int(max(1, min(num_atoms, max(1, num_2q_gates // max(1, len(bonds) or 1))))) + process_log.append( + { + "stage": "resource_estimate", + "num_atoms": num_atoms, + "num_1q_gates": num_1q_gates, + "num_2q_gates": num_2q_gates, + "num_k_body_ops": num_k_body_ops, + "depth_estimate": depth_estimate, + "parallel_layers": parallel_layers, + } + ) + + return { + "dataset": dataset, + "source": source, + "files": files, + "input": { + "samples": len(samples), + "feature_width": len(samples[0]) if samples else 0, + "num_sites": self.service.config.num_sites, + "phys_dim": self.service.config.phys_dim, + "chi_max": self.service.config.chi_max, + "requested_alu": self.service.config.requested_alu, + "allow_auto_fallback": self.service.config.allow_auto_fallback, + "structure_prior": self.service.config.structure_prior, + }, + "tn_page": page.summary(), + "metrics": { + "compression_ratio": compression, + "fidelity": fidelity, + "marginal_fidelity": marginal_fidelity, + "joint_fidelity": joint_fidelity, + "cross_column_dependency": cross_col, + "reconstruction_error_l1": recon, + "q_error_mean": q_error(agg["estimated_mean"], agg["ground_truth_mean"]), + "q_error_var": q_error(agg["estimated_var"], agg["ground_truth_var"]), + "q_error_selectivity": q_error(sel["estimated"], sel["ground_truth"]), + "bond_dimension_variation": len(set(page.metadata.bond_dims)), + "bond_hit_chi_max_count": hit, + "bond_hit_chi_max_ratio": (hit / len(bonds)) if bonds else 0.0, + "bond_utilization_mean": util_mean, + }, + "workload": { + "distribution_sampling": sampled, + "approximate_aggregate": agg, + "likelihood_estimation": like, + "multi_attribute_selectivity": sel, + "sampling": sampled[:16], + "similarity": sim, + }, + "alu_selection": { + "chosen_alu": chosen_alu, + "candidate_scores": candidate_scores, + "selection_reason": process_log[0]["selection_reason"], + }, + "resource_estimate": { + "num_atoms": num_atoms, + "num_1q_gates": num_1q_gates, + "num_2q_gates": num_2q_gates, + "num_k_body_ops": num_k_body_ops, + "depth_estimate": depth_estimate, + "parallel_layers": parallel_layers, + }, + "theoretical_advantage_assessment": { + "data_loading_reusability_score": round(min(1.0, 0.3 + util_mean * 0.7), 4), + "classical_tn_baseline_risk": round(max(0.0, 1.0 - util_mean), 4), + "advantage_likelihood": "high" if util_mean >= 0.75 else "medium" if util_mean >= 0.45 else "low", + "assumption_flags": [ + "requires strong symmetry" if len(set(bonds)) <= 2 else "sensitive to bond-dimension heterogeneity", + "benefit assumes stable feature ordering", + ], + }, + "process_log": process_log, + } diff --git a/systems/quantum_db/data_plane/bus/bus.py b/systems/quantum_db/data_plane/bus/bus.py new file mode 100644 index 0000000..32ca94e --- /dev/null +++ b/systems/quantum_db/data_plane/bus/bus.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +PayloadHandler = Callable[[Any, Mapping[str, Any] | None], Any] + + +class Bus: + __slots__ = ("started", "_routes", "_subscribers", "_next_subscription_id") + + def __init__(self) -> None: + self.started = False + self._routes: dict[str, PayloadHandler] = {} + self._subscribers: dict[str, dict[int, PayloadHandler]] = {} + self._next_subscription_id = 1 + + @staticmethod + def _key(lane: str, name: str) -> str: + return f"{lane}:{name}" + + def start(self) -> None: + self.started = True + + def stop(self) -> None: + self.started = False + self._routes.clear() + self._subscribers.clear() + self._next_subscription_id = 1 + + def register(self, name: str, handler: PayloadHandler, *, lane: str = "command") -> None: + self._routes[self._key(lane, name)] = handler + + def unregister(self, name: str, *, lane: str = "command") -> None: + self._routes.pop(self._key(lane, name), None) + + def send( + self, + name: str, + payload: Any, + *, + lane: str = "command", + metadata: Mapping[str, Any] | None = None, + ) -> Any: + handler = self._routes.get(self._key(lane, name)) + if handler is None: + raise KeyError(f"No route registered for {lane}:{name}") + return handler(payload, metadata) + + def subscribe( + self, + topic: str, + handler: PayloadHandler, + *, + lane: str = "data", + ) -> int: + key = self._key(lane, topic) + sid = self._next_subscription_id + self._next_subscription_id = sid + 1 + bucket = self._subscribers.get(key) + if bucket is None: + self._subscribers[key] = {sid: handler} + else: + bucket[sid] = handler + return sid + + def unsubscribe(self, topic: str, subscription_id: int, *, lane: str = "data") -> bool: + key = self._key(lane, topic) + bucket = self._subscribers.get(key) + if not bucket: + return False + removed = bucket.pop(subscription_id, None) is not None + if not bucket: + self._subscribers.pop(key, None) + return removed + + def publish( + self, + topic: str, + payload: Any, + *, + lane: str = "data", + metadata: Mapping[str, Any] | None = None, + ) -> int: + bucket = self._subscribers.get(self._key(lane, topic)) + if not bucket: + return 0 + for handler in bucket.values(): + handler(payload, metadata) + return len(bucket) \ No newline at end of file diff --git a/systems/quantum_db/data_plane/bus/command_bus.py b/systems/quantum_db/data_plane/bus/command_bus.py new file mode 100644 index 0000000..7103a28 --- /dev/null +++ b/systems/quantum_db/data_plane/bus/command_bus.py @@ -0,0 +1,29 @@ +"""Low-latency command-lane facade over the in-process transport.""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from .bus import Bus, PayloadHandler + + +class CommandBus: + __slots__ = ("_bus",) + + def __init__(self, bus: Bus | None = None) -> None: + self._bus = bus or Bus() + + def start(self) -> None: + self._bus.start() + + def stop(self) -> None: + self._bus.stop() + + def register(self, name: str, handler: PayloadHandler) -> None: + self._bus.register(name, handler, lane="command") + + def unregister(self, name: str) -> None: + self._bus.unregister(name, lane="command") + + def send(self, name: str, payload: Any, *, metadata: Mapping[str, Any] | None = None) -> Any: + return self._bus.send(name, payload, lane="command", metadata=metadata) diff --git a/systems/quantum_db/data_plane/bus/data_bus.py b/systems/quantum_db/data_plane/bus/data_bus.py new file mode 100644 index 0000000..b869939 --- /dev/null +++ b/systems/quantum_db/data_plane/bus/data_bus.py @@ -0,0 +1,32 @@ +"""Data/event/status-lane facade over the in-process transport.""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from .bus import Bus, PayloadHandler + + +class DataBus: + __slots__ = ("_bus", "_lane") + + def __init__(self, bus: Bus | None = None, *, lane: str = "data") -> None: + if lane not in {"data", "event", "status"}: + raise ValueError("DataBus lane must be data, event, or status") + self._bus = bus or Bus() + self._lane = lane + + def start(self) -> None: + self._bus.start() + + def stop(self) -> None: + self._bus.stop() + + def subscribe(self, topic: str, handler: PayloadHandler) -> int: + return self._bus.subscribe(topic, handler, lane=self._lane) + + def unsubscribe(self, topic: str, subscription_id: int) -> bool: + return self._bus.unsubscribe(topic, subscription_id, lane=self._lane) + + def publish(self, topic: str, payload: Any, *, metadata: Mapping[str, Any] | None = None) -> int: + return self._bus.publish(topic, payload, lane=self._lane, metadata=metadata) diff --git a/systems/quantum_db/data_plane/config_loader.py b/systems/quantum_db/data_plane/config_loader.py new file mode 100644 index 0000000..21c271f --- /dev/null +++ b/systems/quantum_db/data_plane/config_loader.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +DEFAULT_CONFIG_PATH = Path("config/tn_quantum_experiment.json") + + +def load_runtime_config(path: str | Path | None = None) -> dict[str, Any]: + cfg_path = Path(path) if path else DEFAULT_CONFIG_PATH + data = json.loads(cfg_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise TypeError("runtime config must be a JSON object") + return data diff --git a/systems/quantum_db/data_plane/quantum/adapter.py b/systems/quantum_db/data_plane/quantum/adapter.py new file mode 100644 index 0000000..75a0505 --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/adapter.py @@ -0,0 +1,535 @@ +from __future__ import annotations + +import random +import time +import logging +import hashlib +from typing import Any +import ipaddress +from urllib.parse import urlparse + +import json +from socket import timeout as SocketTimeout +from urllib.error import HTTPError, URLError +from urllib import request as urllib_request + +from data_plane.quantum.backend import BackendConfig, CompiledProgram, PreflightReport, RunResult +from data_plane.quantum.circuit import translate_ir_to_program +from data_plane.quantum.validation import ( + MAX_RESPONSE_BYTES, + QuantumResponseValidationError, + ValidationLimits, + validate_compile_response, + validate_estimate_response, + validate_preflight_response, + validate_run_response, +) + + +class Qpanda3Adapter: + __slots__ = ("config", "_rng") + + def __init__(self, config: BackendConfig | None = None, *, seed: int = 7) -> None: + self.config = config or BackendConfig() + self._rng = random.Random(seed) + + def _headers(self, *, include_auth: bool) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if include_auth and self.config.api_token: + headers["Authorization"] = f"Bearer {self.config.api_token}" + return headers + + def _qcloud_sdk_enabled(self) -> bool: + return bool(self.config.api_token and not self.config.api_base) + + def _qcloud_backend_name(self) -> str: + return self.config.qcloud_backend_name or self.config.backend_name or "full_amplitude" + + def _build_qcloud_program(self, operations: list[dict[str, Any]]) -> Any: + try: + from pyqpanda3 import core + except Exception as exc: + raise RuntimeError(f"pyqpanda3 SDK is not importable: {exc}") from exc + + prog = core.QProg() + cbits: set[int] = set() + for idx, op in enumerate(operations): + gate = str(op.get("gate", "")).strip().lower() + qubits = [int(q) for q in op.get("qubits", [])] + params = list(op.get("params", [])) + try: + if gate == "h": + prog << core.H(qubits[0]) + elif gate == "x": + prog << core.X(qubits[0]) + elif gate == "y": + prog << core.Y(qubits[0]) + elif gate == "z": + prog << core.Z(qubits[0]) + elif gate == "rx": + prog << core.RX(qubits[0], float(params[0])) + elif gate == "ry": + prog << core.RY(qubits[0], float(params[0])) + elif gate == "rz": + prog << core.RZ(qubits[0], float(params[0])) + elif gate == "u3": + theta, phi, lam = (list(params) + [0.0, 0.0, 0.0])[:3] + prog << core.U3(qubits[0], float(theta), float(phi), float(lam)) + elif gate == "cx": + prog << core.CNOT(qubits[0], qubits[1]) + elif gate == "cz": + prog << core.CZ(qubits[0], qubits[1]) + elif gate == "swap": + prog << core.SWAP(qubits[0], qubits[1]) + elif gate == "measure": + cbit = int(op.get("cbit", qubits[0])) + cbits.add(cbit) + # Keep the serialized program unambiguous for the cloud + # backend while preserving the requested cbit mapping. + prog << core.measure(qubits[0], cbit) + else: + raise ValueError(f"unsupported qcloud SDK gate: {gate!r}") + except Exception as exc: + raise ValueError(f"failed to build qcloud op#{idx} gate={gate!r}: {exc}") from exc + return prog + + def _qcloud_service_backend(self) -> Any: + try: + from pyqpanda3.qcloud import QCloudService + except Exception as exc: + raise RuntimeError(f"pyqpanda3 qcloud SDK is not importable: {exc}") from exc + service = QCloudService(self.config.api_token) + return service.backend(self._qcloud_backend_name()) + + def _qcloud_options(self) -> Any: + try: + from pyqpanda3.qcloud import QCloudOptions + except Exception as exc: + raise RuntimeError(f"pyqpanda3 qcloud SDK is not importable: {exc}") from exc + options = QCloudOptions() + for method_name in ("set_amend", "set_mapping"): + method = getattr(options, method_name, None) + if callable(method): + method(True) + return options + + def _qcloud_run(self, compiled_program: CompiledProgram, *, shots: int, wait_for_result: bool = False) -> RunResult: + start = time.perf_counter() + backend = self._qcloud_service_backend() + prog = self._build_qcloud_program(compiled_program.operations) + options = self._qcloud_options() + job = backend.run(prog, shots=shots, options=options) + job_id = "" + status = "" + try: + job_id = str(job.job_id()) + except Exception: + job_id = "" + try: + status = str(job.status()) + except Exception: + status = "" + counts: dict[str, int] = {} + counts_source = "" + result_diagnostics: dict[str, str] = {} + result_pending = status and "FINISH" not in status.upper() + if result_pending and wait_for_result: + result = self._qcloud_result_with_counts(job) + try: + status = str(job.status()) + except Exception: + status = "FINISHED" + result_pending = False + if not result_pending: + if "result" not in locals(): + result = self._qcloud_result_with_counts(job) + counts, counts_source = self._extract_qcloud_counts(result) + result_diagnostics = self._qcloud_result_diagnostics(result) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return RunResult( + model_id=compiled_program.model_id, + backend_name=self._qcloud_backend_name(), + counts=counts, + shots=shots, + latency_ms=elapsed_ms, + metadata={ + "execution_mode": "qcloud_sdk", + "job_id": job_id, + "status": status, + "backend": self._qcloud_backend_name(), + "result_pending": result_pending, + "counts_source": counts_source, + **{key: compiled_program.metadata[key] for key in + ("probe_contract", "marked_addresses", "iterations", "expected_marked_state", "ideal_marked_probability", + "operation_count", "two_qubit_gate_count") + if key in compiled_program.metadata}, + **result_diagnostics, + }, + ) + + @staticmethod + def _qcloud_result_with_counts(job: Any) -> Any: + """Request the optional result fields required by QCloud's count decoder. + + `job.result()` returns only the provider's default task-result field in + some pyQPanda3 releases. Such a response can report FINISHED but leave + `get_counts()` empty. Requesting `probCount` and `measureQubits` is + read-only and does not submit another quantum task. Old SDKs that do + not accept keyword arguments retain the default call path. + """ + try: + return job.result(keys=["probCount", "measureQubits"]) + except (TypeError, ValueError): + return job.result() + except RuntimeError as exc: + # A few older service/SDK combinations accept result() but reject + # optional field names. Preserve the completed job's default result + # in that narrow compatibility case; real provider/task failures are + # propagated to the caller. + message = str(exc).lower() + if "key" in message and any(word in message for word in ("unknown", "unsupported", "invalid")): + return job.result() + raise + + @staticmethod + def _extract_qcloud_counts(result: Any) -> tuple[dict[str, int], str]: + """Handle SDK releases that expose binary counts through different accessors.""" + attempts: list[tuple[str, Any]] = [] + try: + from pyqpanda3.qcloud import DataBase + attempts.append(("binary", lambda: result.get_counts(base=DataBase.Binary))) + except Exception: + pass + attempts.append(("default", lambda: result.get_counts())) + for name, getter in attempts: + try: + parsed = {str(key): int(value) for key, value in dict(getter()).items()} + if parsed: + return parsed, name + except Exception: + continue + try: + distributions = result.get_counts_list() + merged: dict[str, int] = {} + for distribution in distributions: + for key, value in dict(distribution).items(): + merged[str(key)] = merged.get(str(key), 0) + int(value) + if merged: + return merged, "counts_list" + except Exception: + pass + try: + parsed = Qpanda3Adapter._parse_qcloud_prob_count(result.prob_count_raw()) + if parsed: + return parsed, "prob_count_raw" + except Exception: + pass + # Keep only a digest of the raw provider response for diagnosis; raw content + # can be large and must never be copied into sidecar results or logs. + try: + raw = str(result.origin_data()) + digest = hashlib.sha256(raw.encode("utf-8", errors="replace")).hexdigest()[:16] + return {}, f"unavailable:origin_sha256:{digest}" + except Exception: + return {}, "unavailable" + + @staticmethod + def _parse_qcloud_prob_count(raw: Any) -> dict[str, int]: + """Parse the documented ``probCount`` JSON field without exposing it. + + The cloud service has returned this field both as an object and as a + JSON-encoded string across SDK releases. Accept only complete binary + outcome maps with non-negative integral counts; malformed or unexpected + provider data remains unavailable rather than being interpreted as SQL + candidates. + """ + if not isinstance(raw, str) or not raw or len(raw) > MAX_RESPONSE_BYTES: + return {} + try: + payload = json.loads(raw) + except (TypeError, ValueError, json.JSONDecodeError): + return {} + return Qpanda3Adapter._find_qcloud_count_map(payload) + + @staticmethod + def _find_qcloud_count_map(payload: Any, *, depth: int = 0) -> dict[str, int]: + if depth > 3: + return {} + if isinstance(payload, str): + if len(payload) > MAX_RESPONSE_BYTES: + return {} + try: + return Qpanda3Adapter._find_qcloud_count_map(json.loads(payload), depth=depth + 1) + except (TypeError, ValueError, json.JSONDecodeError): + return {} + if not isinstance(payload, dict): + return {} + counts: dict[str, int] = {} + for key, value in payload.items(): + outcome = str(key) + if not outcome or any(bit not in "01" for bit in outcome): + counts = {} + break + if isinstance(value, bool): + counts = {} + break + try: + count = int(value) + except (TypeError, ValueError): + counts = {} + break + if count < 0 or (isinstance(value, float) and not value.is_integer()): + counts = {} + break + counts[outcome] = count + if counts: + return counts + for field in ("probCount", "counts", "result", "data", "taskResult"): + if field in payload: + found = Qpanda3Adapter._find_qcloud_count_map(payload[field], depth=depth + 1) + if found: + return found + return {} + + @staticmethod + def _qcloud_result_diagnostics(result: Any) -> dict[str, str]: + """Expose bounded non-secret fields useful when providers omit counts.""" + diagnostics: dict[str, str] = {} + for field, method_name in (("provider_error", "error_message"), ("measure_qubits", "measure_qubits")): + try: + value = str(getattr(result, method_name)()) + if value: + diagnostics[field] = value[:1024] + except Exception: + pass + return diagnostics + + def _is_private_or_local_host(self, host: str) -> bool: + host_lower = host.lower() + if host_lower == "localhost": + return True + try: + ip = ipaddress.ip_address(host_lower) + except ValueError: + return False + return bool( + ip.is_loopback + or ip.is_private + or ip.is_link_local + or ip.is_unspecified + or ip.is_reserved + or ip.is_multicast + ) + + def _validate_cloud_target(self) -> tuple[str, str]: + parsed = urlparse(self.config.api_base) + scheme = (parsed.scheme or "").lower() + host = parsed.hostname or "" + if self.config.enforce_https and scheme != "https": + raise ValueError(f"unsafe api_base scheme: {scheme or ''}") + if not host: + raise ValueError("api_base host is empty") + if self._is_private_or_local_host(host): + raise ValueError(f"api_base host is local/private: {host}") + normalized_allowed_hosts = {h.lower() for h in self.config.allowed_hosts} + if host.lower() not in normalized_allowed_hosts: + raise ValueError(f"api_base host not in allowed_hosts: {host}") + return parsed.geturl().rstrip("/"), host + + def _cloud_post(self, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + if not self.config.api_base: + return None + try: + safe_api_base, safe_host = self._validate_cloud_target() + except Exception: + logging.getLogger(__name__).exception("blocked unsafe cloud endpoint access") + raise + url = safe_api_base + path + data = json.dumps(payload).encode("utf-8") + req = urllib_request.Request(url=url, data=data, headers=self._headers(include_auth=bool(safe_host)), method="POST") + with urllib_request.urlopen(req, timeout=self.config.timeout_sec) as resp: + raw = resp.read(MAX_RESPONSE_BYTES + 1) + if len(raw) > MAX_RESPONSE_BYTES: + raise QuantumResponseValidationError(f"cloud response too large: > {MAX_RESPONSE_BYTES} bytes") + body = json.loads(raw.decode("utf-8")) + if not isinstance(body, dict): + raise TypeError("cloud response must be JSON object") + return body + + def _fallback_allowed(self, action: str) -> bool: + return self.config.allow_local_fallback and action not in set(self.config.no_fallback_actions) + + def _record_security_event(self, *, action: str, host: str, exc: Exception, trace_id: str | None, fallback: bool) -> None: + event = { + "event": "quantum_cloud_failure", + "action": action, + "host": host, + "exception_type": type(exc).__name__, + "trace_id": trace_id or "", + "fallback": fallback, + } + print(json.dumps(event, ensure_ascii=False, sort_keys=True)) + + def _cloud_host(self) -> str: + if not self.config.api_base: + return "" + parsed = urlparse(self.config.api_base) + if not parsed.scheme or not parsed.hostname: + return "" + return parsed.hostname + + def preflight(self, ir_packet: dict[str, Any], *, trace_id: str | None = None) -> PreflightReport: + if self._qcloud_sdk_enabled(): + try: + translate_ir_to_program(ir_packet, backend_name=self._qcloud_backend_name()) + backend = self._qcloud_service_backend() + chip_info = getattr(backend, "chip_info", None) + warnings: list[str] = [] + if callable(chip_info): + info = chip_info() + for name in ("qubits_num", "available_qubits"): + method = getattr(info, name, None) + if callable(method): + warnings.append(f"{name}={method()}") + return PreflightReport(ok=True, warnings=warnings, route_edges=[]) + except (ImportError, RuntimeError, ValueError, TypeError, OSError) as exc: + fallback = self._fallback_allowed("preflight") + self._record_security_event(action="preflight", host="qcloud-sdk", exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + try: + body = self._cloud_post("/preflight", {"ir": ir_packet, "backend": self.config.backend_name}) + if body is not None: + validate_preflight_response(body, limits=ValidationLimits(num_qubits=self.config.num_qubits)) + return PreflightReport( + ok=bool(body.get("ok", True)), + fatal_errors=list(body.get("fatal_errors", [])), + warnings=list(body.get("warnings", [])), + route_edges=[tuple(edge) for edge in body.get("route_edges", [])], + ) + except (HTTPError, URLError, SocketTimeout, TimeoutError, json.JSONDecodeError, TypeError, ValueError) as exc: + fallback = self._fallback_allowed("preflight") + self._record_security_event(action="preflight", host=self._cloud_host(), exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + mapping = ir_packet.get("logical_to_physical", {}) + coupling = {tuple(sorted(edge)) for edge in self.config.coupling_map} + route_edges: list[tuple[int, int]] = [] + warnings: list[str] = [] + if not mapping: + warnings.append("logical_to_physical mapping is empty") + for op in ir_packet.get("operations", []): + if not isinstance(op, dict): + continue + gate = str(op.get("gate", "")).lower() + qubits = op.get("qubits", []) + if gate in {"cx", "cz"} and isinstance(qubits, list) and len(qubits) == 2: + q0 = mapping.get(str(qubits[0]), qubits[0]) + q1 = mapping.get(str(qubits[1]), qubits[1]) + edge = tuple(sorted((int(q0), int(q1)))) + if coupling and edge not in coupling: + route_edges.append(edge) + return PreflightReport(ok=True, warnings=warnings, route_edges=route_edges) + + def compile(self, ir_packet: dict[str, Any], *, trace_id: str | None = None) -> CompiledProgram: + if self._qcloud_sdk_enabled(): + try: + program = translate_ir_to_program(ir_packet, backend_name=self._qcloud_backend_name()) + program.metadata = {**program.metadata, "execution_mode": "qcloud_sdk_prepared"} + return program + except (ImportError, RuntimeError, ValueError, TypeError, OSError) as exc: + fallback = self._fallback_allowed("compile") + self._record_security_event(action="compile", host="qcloud-sdk", exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + try: + body = self._cloud_post("/compile", {"ir": ir_packet, "backend": self.config.backend_name}) + if body is not None: + validate_compile_response(body, limits=ValidationLimits(num_qubits=self.config.num_qubits)) + return CompiledProgram( + model_id=str(body.get("model_id", ir_packet.get("model_id", "unknown"))), + backend_name=str(body.get("backend_name", self.config.backend_name)), + operations=list(body.get("operations", [])), + metadata={**dict(body.get("metadata", {})), "execution_mode": "cloud"}, + ) + except (HTTPError, URLError, SocketTimeout, TimeoutError, json.JSONDecodeError, TypeError, ValueError) as exc: + fallback = self._fallback_allowed("compile") + self._record_security_event(action="compile", host=self._cloud_host(), exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + program = translate_ir_to_program(ir_packet, backend_name=self.config.backend_name) + program.metadata = {**program.metadata, "execution_mode": "local_fallback"} + return program + + def estimate(self, ir_packet: dict[str, Any], *, trace_id: str | None = None) -> dict[str, Any]: + if self._qcloud_sdk_enabled(): + try: + program = translate_ir_to_program(ir_packet, backend_name=self._qcloud_backend_name()) + ops = program.operations + two_qubit = sum(1 for op in ops if isinstance(op, dict) and str(op.get("gate", "")).lower() in {"cx", "cz", "swap"}) + return { + "num_ops": len(ops), + "two_qubit_ops": two_qubit, + "depth_est": len(ops), + "backend": self._qcloud_backend_name(), + "execution_mode": "qcloud_sdk_estimate", + } + except (ImportError, RuntimeError, ValueError, TypeError, OSError) as exc: + fallback = self._fallback_allowed("estimate") + self._record_security_event(action="estimate", host="qcloud-sdk", exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + try: + body = self._cloud_post("/estimate", {"ir": ir_packet, "backend": self.config.backend_name}) + if body is not None: + validate_estimate_response(body) + return body + except (HTTPError, URLError, SocketTimeout, TimeoutError, json.JSONDecodeError, TypeError, ValueError) as exc: + fallback = self._fallback_allowed("estimate") + self._record_security_event(action="estimate", host=self._cloud_host(), exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + ops = ir_packet.get("operations", []) if isinstance(ir_packet, dict) else [] + two_qubit = sum(1 for op in ops if isinstance(op, dict) and str(op.get("gate", "")).lower() in {"cx", "cz", "swap"}) + depth_est = len(ops) + return {"num_ops": len(ops), "two_qubit_ops": two_qubit, "depth_est": depth_est, "backend": self.config.backend_name, "execution_mode": "local_fallback"} + + def run(self, compiled_program: CompiledProgram, *, shots: int = 1024, trace_id: str | None = None, + wait_for_result: bool = False) -> RunResult: + if self._qcloud_sdk_enabled() or compiled_program.metadata.get("execution_mode") == "qcloud_sdk_prepared": + try: + return self._qcloud_run(compiled_program, shots=shots, wait_for_result=wait_for_result) + except (ImportError, RuntimeError, ValueError, TypeError, OSError) as exc: + fallback = self._fallback_allowed("run") + self._record_security_event(action="run", host="qcloud-sdk", exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + try: + body = self._cloud_post("/run", {"compiled": {"model_id": compiled_program.model_id, "backend_name": compiled_program.backend_name, "operations": compiled_program.operations, "metadata": compiled_program.metadata}, "shots": shots}) + if body is not None: + validate_run_response(body, limits=ValidationLimits(num_qubits=self.config.num_qubits)) + return RunResult( + model_id=str(body.get("model_id", compiled_program.model_id)), + backend_name=str(body.get("backend_name", self.config.backend_name)), + counts=dict(body.get("counts", {})), + shots=int(body.get("shots", shots)), + latency_ms=float(body.get("latency_ms", 0.0)), + metadata={**dict(body.get("metadata", {"cloud": True})), "execution_mode": "cloud"}, + ) + except (HTTPError, URLError, SocketTimeout, TimeoutError, json.JSONDecodeError, TypeError, ValueError) as exc: + fallback = self._fallback_allowed("run") + self._record_security_event(action="run", host=self._cloud_host(), exc=exc, trace_id=trace_id, fallback=fallback) + if not fallback: + raise + + start = time.perf_counter() + p0 = 0.5 + min(0.2, 0.01 * len(compiled_program.operations)) + c0 = int(shots * p0) + c1 = shots - c0 + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return RunResult(model_id=compiled_program.model_id, backend_name=compiled_program.backend_name, counts={"0": c0, "1": c1}, shots=shots, latency_ms=elapsed_ms, metadata={"simulated": True, "execution_mode": "local_fallback"}) diff --git a/systems/quantum_db/data_plane/quantum/backend.py b/systems/quantum_db/data_plane/quantum/backend.py new file mode 100644 index 0000000..bad9b6b --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/backend.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class BackendConfig: + backend_name: str = "qpanda3-sim" + num_qubits: int = 16 + num_couplers: int = 0 + coupling_map: list[tuple[int, int]] = field(default_factory=list) + basis_gates: list[str] = field(default_factory=lambda: ["u3", "cz"]) + chip_name: str = "default" + calibration_epoch: str = "default" + api_base: str = "" + api_token: str = "" + qcloud_backend_name: str = "" + enforce_https: bool = True + allowed_hosts: list[str] = field(default_factory=list) + timeout_sec: int = 20 + allow_local_fallback: bool = False + no_fallback_actions: tuple[str, ...] = ("compile", "run") + + +@dataclass +class PreflightReport: + ok: bool + fatal_errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + route_edges: list[tuple[int, int]] = field(default_factory=list) + + +@dataclass +class CompiledProgram: + model_id: str + backend_name: str + operations: list[dict[str, Any]] + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class RunResult: + model_id: str + backend_name: str + counts: dict[str, int] + shots: int + latency_ms: float + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/systems/quantum_db/data_plane/quantum/circuit.py b/systems/quantum_db/data_plane/quantum/circuit.py new file mode 100644 index 0000000..7e83432 --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/circuit.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Any + +from data_plane.quantum.backend import CompiledProgram + + +SUPPORTED_GATES: frozenset[str] = frozenset({"h", "x", "y", "z", "rx", "ry", "rz", "u3", "cx", "cz", "measure", "swap"}) + + +def translate_ir_to_program(ir_packet: dict[str, Any], *, backend_name: str) -> CompiledProgram: + model_id = str(ir_packet.get("model_id", "")).strip() + if not model_id: + raise ValueError("IR missing model_id") + + raw_ops = ir_packet.get("operations") + operations: list[dict[str, Any]] = [] + if isinstance(raw_ops, list): + for idx, op in enumerate(raw_ops): + if not isinstance(op, dict): + raise TypeError(f"operation at {idx} must be a mapping") + gate = str(op.get("gate", "")).strip().lower() + if gate not in SUPPORTED_GATES: + raise ValueError(f"unsupported gate: {gate!r} at op#{idx}") + operations.append(dict(op)) + metadata: dict[str, Any] = {"ir_version": ir_packet.get("ir_version", "v1")} + # Keep the bounded, non-secret hardware-probe contract visible in the result + # so experiment output can be interpreted without inspecting raw provider data. + for key in ("probe_contract", "marked_addresses", "iterations", "expected_marked_state", "ideal_marked_probability", + "operation_count", "two_qubit_gate_count"): + if key in ir_packet: + metadata[key] = ir_packet[key] + return CompiledProgram(model_id=model_id, backend_name=backend_name, operations=operations, metadata=metadata) diff --git a/systems/quantum_db/data_plane/quantum/executor.py b/systems/quantum_db/data_plane/quantum/executor.py new file mode 100644 index 0000000..d7e1483 --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/executor.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from data_plane.quantum.adapter import Qpanda3Adapter +from data_plane.quantum.backend import BackendConfig + +_ROUTE_NAMES: tuple[str, ...] = ("quantum.execute", "quantum") + + +class QuantumExecutor: + __slots__ = ("bus", "monitor", "started", "adapter") + + def __init__(self, bus: Any | None = None, monitor: Any | None = None, backend_config: BackendConfig | None = None, **_: Any) -> None: + self.bus = bus + self.monitor = monitor + self.started = False + self.adapter = Qpanda3Adapter(config=backend_config) + + def start(self) -> None: + self.started = True + register = getattr(self.bus, "register", None) + if callable(register): + for name in _ROUTE_NAMES: + register(name, self._on_bus_command, lane="command") + + def stop(self) -> None: + unregister = getattr(self.bus, "unregister", None) + if callable(unregister): + for name in _ROUTE_NAMES: + unregister(name, lane="command") + self.started = False + + def execute(self, command: Mapping[str, Any], metadata: Mapping[str, Any] | None = None) -> Any: + if not isinstance(command, Mapping): + raise TypeError("Quantum command must be a mapping") + action = str(command.get("action", "")).strip().lower() + payload = command.get("payload", command.get("params", {})) + if not isinstance(payload, dict): + payload = {} + + trace_id = "" + if isinstance(metadata, Mapping): + trace_id = str(metadata.get("trace_id", metadata.get("request_id", ""))) + + if action == "preflight": + result = self.adapter.preflight(dict(payload.get("ir", {})), trace_id=trace_id or None) + return {"status": "ok", "action": action, "execution_mode": "cloud_or_local_policy", "result": result} + if action == "compile": + result = self.adapter.compile(dict(payload.get("ir", {})), trace_id=trace_id or None) + return {"status": "ok", "action": action, "execution_mode": result.metadata.get("execution_mode", "unknown"), "result": result} + if action == "estimate": + result = self.adapter.estimate(dict(payload.get("ir", {})), trace_id=trace_id or None) + return {"status": "ok", "action": action, "execution_mode": result.get("execution_mode", "cloud"), "result": result} + if action == "run": + compiled = payload.get("compiled") + if compiled is None: + compiled = self.adapter.compile(dict(payload.get("ir", {})), trace_id=trace_id or None) + shots = int(payload.get("shots", 1024)) + result = self.adapter.run(compiled, shots=shots, trace_id=trace_id or None, + wait_for_result=bool(payload.get("wait_for_result", False))) + return {"status": "ok", "action": action, "execution_mode": result.metadata.get("execution_mode", "unknown"), "result": result} + raise ValueError(f"unsupported quantum action: {action}") + + def _on_bus_command(self, payload: Any, metadata: Mapping[str, Any] | None = None) -> Any: + if isinstance(payload, Mapping): + return self.execute(payload, metadata) + raise TypeError("Quantum bus payload must be a mapping command") + + __call__ = execute diff --git a/systems/quantum_db/data_plane/quantum/fallback.py b/systems/quantum_db/data_plane/quantum/fallback.py new file mode 100644 index 0000000..0bed33e --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/fallback.py @@ -0,0 +1,33 @@ +"""Explicit, non-executing fallback decisions for quantum adapter failures.""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class FallbackDecision: + action: str + fallback_backend: str | None + permitted: bool + reason: str + retryable: bool + + +class QuantumFallbackPolicy: + """Classifies failures without silently running a different backend.""" + + _RETRYABLE = frozenset({"TimeoutError", "ConnectionError", "OSError", "URLError", "HTTPError"}) + + def decide(self, *, action: str, error: BaseException, allow_local_fallback: bool, + no_fallback_actions: tuple[str, ...] = ()) -> FallbackDecision: + normalized_action = action.strip().lower() or "unknown" + error_name = type(error).__name__ + retryable = error_name in self._RETRYABLE + permitted = bool(allow_local_fallback and normalized_action not in set(no_fallback_actions)) + return FallbackDecision( + action=normalized_action, + fallback_backend="local_simulator" if permitted else None, + permitted=permitted, + reason=f"{error_name}: {error}", + retryable=retryable, + ) diff --git a/systems/quantum_db/data_plane/quantum/validation.py b/systems/quantum_db/data_plane/quantum/validation.py new file mode 100644 index 0000000..6416a53 --- /dev/null +++ b/systems/quantum_db/data_plane/quantum/validation.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_OPERATIONS = 20_000 +MAX_COUNTS_KEYS = 16_384 +MAX_METADATA_DEPTH = 6 +MAX_METADATA_KEY_LENGTH = 128 +MAX_WARNINGS = 2_000 +MAX_ROUTE_EDGES = 20_000 +MAX_ERRORS = 2_000 +MIN_SHOTS = 1 +MAX_SHOTS = 10_000_000 +MIN_LATENCY_MS = 0.0 +MAX_LATENCY_MS = 3_600_000.0 + + +class QuantumResponseValidationError(ValueError): + """Raised when cloud response schema/content is invalid.""" + + +@dataclass(frozen=True) +class ValidationLimits: + num_qubits: int + + +def _require_type(value: Any, expected: type | tuple[type, ...], field_name: str) -> None: + if not isinstance(value, expected): + raise QuantumResponseValidationError(f"{field_name} has invalid type: expected {expected}, got {type(value)}") + + +def _validate_str_list(value: Any, field_name: str, *, max_len: int) -> None: + _require_type(value, list, field_name) + if len(value) > max_len: + raise QuantumResponseValidationError(f"{field_name} exceeds limit {max_len}") + for idx, item in enumerate(value): + if not isinstance(item, str): + raise QuantumResponseValidationError(f"{field_name}[{idx}] must be str") + + +def _validate_metadata(value: Any, field_name: str = "metadata", *, depth: int = 0) -> None: + if depth > MAX_METADATA_DEPTH: + raise QuantumResponseValidationError(f"{field_name} depth exceeds limit {MAX_METADATA_DEPTH}") + if isinstance(value, dict): + if len(value) > MAX_COUNTS_KEYS: + raise QuantumResponseValidationError(f"{field_name} has too many keys") + for k, v in value.items(): + if not isinstance(k, str): + raise QuantumResponseValidationError(f"{field_name} key must be str") + if len(k) > MAX_METADATA_KEY_LENGTH: + raise QuantumResponseValidationError(f"{field_name} key too long") + _validate_metadata(v, field_name=f"{field_name}.{k}", depth=depth + 1) + elif isinstance(value, list): + if len(value) > MAX_OPERATIONS: + raise QuantumResponseValidationError(f"{field_name} list too large") + for idx, item in enumerate(value): + _validate_metadata(item, field_name=f"{field_name}[{idx}]", depth=depth + 1) + elif value is None or isinstance(value, (str, bool, int, float)): + return + else: + raise QuantumResponseValidationError(f"{field_name} contains unsupported type {type(value)}") + + +def _validate_qubit_list(qubits: Any, field_name: str, limits: ValidationLimits) -> None: + _require_type(qubits, list, field_name) + if len(qubits) == 0 or len(qubits) > 8: + raise QuantumResponseValidationError(f"{field_name} length out of range") + for idx, q in enumerate(qubits): + if not isinstance(q, int): + raise QuantumResponseValidationError(f"{field_name}[{idx}] must be int") + if q < 0 or q >= limits.num_qubits: + raise QuantumResponseValidationError(f"{field_name}[{idx}] out of range [0, {limits.num_qubits})") + + +def _validate_operations(value: Any, limits: ValidationLimits) -> None: + _require_type(value, list, "operations") + if len(value) > MAX_OPERATIONS: + raise QuantumResponseValidationError(f"operations exceeds limit {MAX_OPERATIONS}") + for idx, op in enumerate(value): + if not isinstance(op, dict): + raise QuantumResponseValidationError(f"operations[{idx}] must be object") + gate = op.get("gate") + qubits = op.get("qubits") + if gate is None or qubits is None: + raise QuantumResponseValidationError(f"operations[{idx}] missing required fields") + if not isinstance(gate, str): + raise QuantumResponseValidationError(f"operations[{idx}].gate must be str") + _validate_qubit_list(qubits, f"operations[{idx}].qubits", limits) + + +def validate_preflight_response(body: dict[str, Any], *, limits: ValidationLimits) -> None: + allowed = {"ok", "fatal_errors", "warnings", "route_edges"} + for key in body: + if key not in allowed: + raise QuantumResponseValidationError(f"preflight unexpected field: {key}") + if "ok" in body: + _require_type(body["ok"], bool, "ok") + if "fatal_errors" in body: + _validate_str_list(body["fatal_errors"], "fatal_errors", max_len=MAX_ERRORS) + if "warnings" in body: + _validate_str_list(body["warnings"], "warnings", max_len=MAX_WARNINGS) + if "route_edges" in body: + edges = body["route_edges"] + _require_type(edges, list, "route_edges") + if len(edges) > MAX_ROUTE_EDGES: + raise QuantumResponseValidationError("route_edges exceeds limit") + for idx, edge in enumerate(edges): + _require_type(edge, (list, tuple), f"route_edges[{idx}]") + if len(edge) != 2: + raise QuantumResponseValidationError(f"route_edges[{idx}] must have length 2") + _validate_qubit_list(list(edge), f"route_edges[{idx}]", limits) + + +def validate_compile_response(body: dict[str, Any], *, limits: ValidationLimits) -> None: + required = {"operations"} + optional = {"model_id", "backend_name", "metadata"} + for key in body: + if key not in required | optional: + raise QuantumResponseValidationError(f"compile unexpected field: {key}") + for key in required: + if key not in body: + raise QuantumResponseValidationError(f"compile missing required field: {key}") + _validate_operations(body["operations"], limits) + if "model_id" in body: + _require_type(body["model_id"], str, "model_id") + if "backend_name" in body: + _require_type(body["backend_name"], str, "backend_name") + if "metadata" in body: + _validate_metadata(body["metadata"]) + + +def validate_estimate_response(body: dict[str, Any]) -> None: + required = {"num_ops", "two_qubit_ops", "depth_est", "backend"} + for key in required: + if key not in body: + raise QuantumResponseValidationError(f"estimate missing required field: {key}") + for field in ("num_ops", "two_qubit_ops", "depth_est"): + val = body[field] + if not isinstance(val, int) or val < 0 or val > MAX_OPERATIONS: + raise QuantumResponseValidationError(f"{field} out of range") + _require_type(body["backend"], str, "backend") + + +def validate_run_response(body: dict[str, Any], *, limits: ValidationLimits) -> None: + required = {"counts", "shots", "latency_ms"} + optional = {"model_id", "backend_name", "metadata"} + for key in body: + if key not in required | optional: + raise QuantumResponseValidationError(f"run unexpected field: {key}") + for key in required: + if key not in body: + raise QuantumResponseValidationError(f"run missing required field: {key}") + + counts = body["counts"] + _require_type(counts, dict, "counts") + if len(counts) > MAX_COUNTS_KEYS: + raise QuantumResponseValidationError(f"counts exceeds limit {MAX_COUNTS_KEYS}") + for bitstring, count in counts.items(): + if not isinstance(bitstring, str): + raise QuantumResponseValidationError("counts key must be str") + if not isinstance(count, int) or count < 0: + raise QuantumResponseValidationError("counts value must be non-negative int") + + shots = body["shots"] + if not isinstance(shots, int) or shots < MIN_SHOTS or shots > MAX_SHOTS: + raise QuantumResponseValidationError(f"shots out of range [{MIN_SHOTS}, {MAX_SHOTS}]") + + latency = body["latency_ms"] + if not isinstance(latency, (int, float)) or latency < MIN_LATENCY_MS or latency > MAX_LATENCY_MS: + raise QuantumResponseValidationError(f"latency_ms out of range [{MIN_LATENCY_MS}, {MAX_LATENCY_MS}]") + + if "model_id" in body: + _require_type(body["model_id"], str, "model_id") + if "backend_name" in body: + _require_type(body["backend_name"], str, "backend_name") + if "metadata" in body: + _validate_metadata(body["metadata"]) diff --git a/systems/quantum_db/data_plane/tn/cache.py b/systems/quantum_db/data_plane/tn/cache.py new file mode 100644 index 0000000..7537318 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/cache.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any + + +@dataclass +class CacheStats: + hits: int = 0 + misses: int = 0 + evictions: int = 0 + + +class TNCache: + """TN 侧缓存：热点页与中间结果复用。""" + + __slots__ = ("capacity", "_store", "stats") + + def __init__(self, capacity: int = 128) -> None: + if capacity <= 0: + raise ValueError("capacity must be positive") + self.capacity = capacity + self._store: OrderedDict[str, Any] = OrderedDict() + self.stats = CacheStats() + + def has(self, key: str) -> bool: + return key in self._store + + def get(self, key: str) -> Any | None: + if key not in self._store: + self.stats.misses += 1 + return None + self._store.move_to_end(key) + self.stats.hits += 1 + return self._store[key] + + def put(self, key: str, value: Any) -> None: + if key in self._store: + self._store.move_to_end(key) + self._store[key] = value + if len(self._store) > self.capacity: + self._store.popitem(last=False) + self.stats.evictions += 1 + + def pop(self, key: str) -> Any | None: + return self._store.pop(key, None) + + def clear(self) -> None: + self._store.clear() + + def snapshot(self) -> dict[str, Any]: + total = self.stats.hits + self.stats.misses + hit_rate = self.stats.hits / total if total else 0.0 + return { + "capacity": self.capacity, + "size": len(self._store), + "hits": self.stats.hits, + "misses": self.stats.misses, + "evictions": self.stats.evictions, + "hit_rate": hit_rate, + } + + +# 统一缓存入口：三层缓存能力从 cache 模块导出，避免外部直接依赖 multi_cache 模块路径。 +from data_plane.tn.multi_cache import ( # noqa: E402 + ArtifactEntry, + MaterializedArtifactCache, + QueryCanonicalCache, + QueryCanonicalEntry, + TNMultiLayerCache, + TNPlanCache, + TNPlanEntry, +) diff --git a/systems/quantum_db/data_plane/tn/compute.py b/systems/quantum_db/data_plane/tn/compute.py new file mode 100644 index 0000000..16af4d7 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/compute.py @@ -0,0 +1,508 @@ +from __future__ import annotations + +import cmath +import math +import random +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any, Protocol + +from data_plane.tn.page import TNPage, TNPageMetadata +from data_plane.tn.contraction import ExecutionOrderBundle, PhysicalTopologyGraph, TNContractionPlanner + + +@dataclass +class EncodingResult: + encoded: list[list[complex]] + kept_indices: list[int] + ratio: float + + +class ALUType(str, Enum): + WLOC = "wloc" + WBRICK = "wbrick" + WTREE = "wtree" + WSTAR = "wstar" + WSYM = "wsym" + + +@dataclass(slots=True) +class ALUPlan: + input_dim: int + expected_gate_type: str + parallel_layers: int + depth_estimate: int + error_sensitivity_estimate: float + alu_type: ALUType + + +@dataclass(slots=True) +class ALUResourceEstimate: + num_atoms: int + num_1q_gates: int + num_2q_gates: int + num_k_body_ops: int + depth_estimate: int + parallel_layers: int + error_risk_score: float + + +class ALUExecutor(Protocol): + alu_type: ALUType + + def build_plan(self, samples: Sequence[Any], config: dict[str, Any]) -> ALUPlan: ... + + def apply(self, page: TNPage, plan: ALUPlan) -> TNPage: ... + + def estimate_resources(self, plan: ALUPlan) -> ALUResourceEstimate: ... + + +class RuleBasedALUBase: + alu_type: ALUType + gate_type: str + + def build_plan(self, samples: Sequence[Any], config: dict[str, Any]) -> ALUPlan: + num_sites = int(config.get("num_sites", 1)) + phys_dim = int(config.get("phys_dim", 2)) + input_dim = max(1, num_sites * phys_dim) + layers = self._estimate_layers(num_sites) + depth = max(1, layers * self._depth_factor()) + err = min(1.0, 0.08 + 0.02 * layers + 0.005 * phys_dim) + return ALUPlan( + input_dim=input_dim, + expected_gate_type=self.gate_type, + parallel_layers=layers, + depth_estimate=depth, + error_sensitivity_estimate=err, + alu_type=self.alu_type, + ) + + def apply(self, page: TNPage, plan: ALUPlan) -> TNPage: + page.artifacts["alu_selected"] = plan.alu_type.value + page.artifacts["alu_plan"] = { + "input_dim": plan.input_dim, + "expected_gate_type": plan.expected_gate_type, + "parallel_layers": plan.parallel_layers, + "depth_estimate": plan.depth_estimate, + "error_sensitivity_estimate": plan.error_sensitivity_estimate, + } + page.artifacts["alu_resources"] = asdict(self.estimate_resources(plan)) + return page + + def estimate_resources(self, plan: ALUPlan) -> ALUResourceEstimate: + atoms = max(1, int(math.sqrt(plan.input_dim))) + num_1q = atoms * 2 + num_2q = atoms * max(plan.parallel_layers - 1, 0) + num_k = 0 if plan.expected_gate_type != "k-body" else max(1, atoms // 2) + return ALUResourceEstimate( + num_atoms=atoms, + num_1q_gates=num_1q, + num_2q_gates=num_2q, + num_k_body_ops=num_k, + depth_estimate=plan.depth_estimate, + parallel_layers=plan.parallel_layers, + error_risk_score=plan.error_sensitivity_estimate, + ) + + def _estimate_layers(self, num_sites: int) -> int: + return max(1, math.ceil(math.log2(max(2, num_sites)))) + + def _depth_factor(self) -> int: + return 2 + + +class WlocALUStrategy(RuleBasedALUBase): + alu_type = ALUType.WLOC + gate_type = "1q" + + +class WbrickALUStrategy(RuleBasedALUBase): + alu_type = ALUType.WBRICK + gate_type = "2q" + + +class WtreeALUStrategy(RuleBasedALUBase): + alu_type = ALUType.WTREE + gate_type = "k-body" + + def _depth_factor(self) -> int: + return 3 + + +class WstarALUStrategy(RuleBasedALUBase): + alu_type = ALUType.WSTAR + gate_type = "2q" + + +class WsymALUStrategy(RuleBasedALUBase): + alu_type = ALUType.WSYM + gate_type = "k-body" + + +class TNCompute: + """TN 计算原语与 sample-to-MPS 构造流程（纯 Python，无外部依赖）。""" + + __slots__ = ("seed", "_rng", "_alu_strategies") + + def __init__(self, seed: int | None = None) -> None: + self.seed = seed + self._rng = random.Random(seed) + self._alu_strategies: dict[ALUType, ALUExecutor] = { + ALUType.WLOC: WlocALUStrategy(), + ALUType.WBRICK: WbrickALUStrategy(), + ALUType.WTREE: WtreeALUStrategy(), + ALUType.WSTAR: WstarALUStrategy(), + ALUType.WSYM: WsymALUStrategy(), + } + + def encode_fft_topk(self, samples: Sequence[Any], keep_ratio: float = 0.25) -> EncodingResult: + vectors = [self._flatten_numeric(sample) for sample in samples] + if not vectors: + raise ValueError("samples must not be empty") + if not 0 < keep_ratio <= 1: + raise ValueError("keep_ratio must be in (0, 1]") + + feature_len = len(vectors[0]) + if feature_len == 0: + raise ValueError("sample vector must not be empty") + if any(len(v) != feature_len for v in vectors): + raise ValueError("all samples must have consistent flattened size") + + # 仅计算前 K 个低频分量，降低参数量并保持多尺度信息。 + k = max(1, int(math.ceil(feature_len * keep_ratio)), int(math.ceil(math.log2(feature_len + 1)))) + spectrum = [self._dft_prefix(v, k) for v in vectors] + + amp_mean = [0.0] * k + for coeffs in spectrum: + for i, c in enumerate(coeffs): + amp_mean[i] += abs(c) + amp_mean = [x / len(spectrum) for x in amp_mean] + + keep = k + ranked = sorted(range(k), key=lambda i: amp_mean[i], reverse=True) + kept_indices = sorted(ranked[:keep]) + encoded = [[coeffs[i] for i in kept_indices] for coeffs in spectrum] + return EncodingResult(encoded=encoded, kept_indices=kept_indices, ratio=float(keep_ratio)) + + def build_data_graph(self, features: Sequence[Sequence[complex]], corr_threshold: float = 0.1) -> list[tuple[int, int, float]]: + matrix = [[float(v.real) for v in row] for row in features] + if not matrix or len(matrix[0]) < 2: + return [] + + n = len(matrix) + f = len(matrix[0]) + columns = [[matrix[r][c] for r in range(n)] for c in range(f)] + edges: list[tuple[int, int, float]] = [] + for i in range(f): + for j in range(i + 1, f): + weight = abs(self._pearson(columns[i], columns[j])) + if math.isfinite(weight) and weight >= corr_threshold: + edges.append((i, j, weight)) + return edges + + def fit_mps_from_samples( + self, + samples: Sequence[Any], + *, + page_id: str, + num_sites: int, + phys_dim: int = 8, + chi_max: int = 16, + trunc_eps: float = 1e-4, + keep_ratio: float = 0.25, + alu_type: ALUType | str = ALUType.WLOC, + ) -> TNPage: + """从样本直接构造初始 MPS（避免显式指数张量）。""" + enc = self.encode_fft_topk(samples, keep_ratio=keep_ratio) + projected = self._project_to_sites([[v.real for v in row] for row in enc.encoded], num_sites=num_sites) + discrete = self._discretize(projected, bins=phys_dim) + + singles = self._single_probs(discrete, phys_dim) + bonds = self._bond_dims(discrete, phys_dim=phys_dim, chi_max=chi_max, trunc_eps=trunc_eps) + tensors = self._build_chain_tensors(singles, bonds) + + metadata = TNPageMetadata( + page_id=page_id, + layout="mps", + physical_dims=tuple(phys_dim for _ in range(num_sites)), + bond_dims=tuple(bonds), + encoding="fft_topk", + extra={ + "keep_ratio": keep_ratio, + "fft_kept": len(enc.kept_indices), + "chi_max": chi_max, + "trunc_eps": trunc_eps, + }, + ) + page = TNPage(metadata=metadata, tensors=tensors) + graph = self.build_data_graph(enc.encoded) + page.update_stats(samples=float(len(samples)), locality=self.measure_locality(graph)) + page.artifacts["fft_kept_indices"] = list(enc.kept_indices) + page.artifacts["graph_data_edges"] = graph + return self._execute_alu_pipeline( + page=page, + samples=samples, + config={"num_sites": num_sites, "phys_dim": phys_dim, "chi_max": chi_max, "trunc_eps": trunc_eps}, + alu_type=alu_type, + ) + + def slice(self, page: TNPage, *, start: int = 0, stop: int | None = None) -> list[Any]: + return page.tensors[start:stop] + + def sample(self, page: TNPage, num_samples: int = 1) -> list[int]: + if num_samples <= 0: + raise ValueError("num_samples must be positive") + if not page.metadata.physical_dims: + return [] + draws: list[int] = [] + for _ in range(num_samples): + for site in page.tensors: + probs = [max(float(v[0]), 0.0) for v in site[0]] if site and site[0] else [] + if not probs: + draws.append(self._rng.randrange(page.metadata.physical_dims[0])) + continue + total = sum(probs) + if total <= 0: + draws.append(self._rng.randrange(len(probs))) + continue + r = self._rng.random() * total + s = 0.0 + idx = 0 + for i, p in enumerate(probs): + s += p + if s >= r: + idx = i + break + draws.append(idx) + return draws + + def reconstruct(self, page: TNPage, *, mode: str = "shape") -> Any: + if mode == "shape": + return { + "num_tensors": len(page.tensors), + "physical_dims": page.metadata.physical_dims, + "bond_dims": page.metadata.bond_dims, + } + if mode == "summary": + return page.summary() + raise ValueError(f"unsupported reconstruct mode: {mode}") + + def plan_contraction_order( + self, + *, + topology: PhysicalTopologyGraph, + partition: dict[str, str], + mode: str = "main", + window_size: int = 4, + ) -> ExecutionOrderBundle: + planner = TNContractionPlanner() + patches, adjacency = planner.build_patch_topology(topology, partition) + if mode == "main": + return planner.plan_main_order(patches, adjacency) + if mode == "compat": + return planner.plan_compat_order(patches, window_size=window_size) + raise ValueError(f"unsupported contraction mode: {mode}") + + def _execute_alu_pipeline( + self, + *, + page: TNPage, + samples: Sequence[Any], + config: dict[str, Any], + alu_type: ALUType | str, + ) -> TNPage: + selected = ALUType(alu_type) if isinstance(alu_type, str) else alu_type + strategy = self._alu_strategies[selected] + plan = strategy.build_plan(samples=samples, config=config) + return strategy.apply(page=page, plan=plan) + + + def estimate_probability(self, page: TNPage, sample: Sequence[float]) -> float: + flattened = self._flatten_numeric(sample) + if not flattened: + return 0.0 + phys = page.metadata.physical_dims[0] if page.metadata.physical_dims else 1 + probs = 1.0 + for i, tensor in enumerate(page.tensors): + value = flattened[min(i, len(flattened) - 1)] + idx = int(abs(value) * phys) % phys + # tensor[0][idx][0] 约等价于边缘概率近似 + p = float(tensor[0][idx][0]) if tensor and tensor[0] and tensor[0][idx] else 1.0 / phys + probs *= max(p, 1e-9) + return probs + + def approximate_aggregate(self, samples: Sequence[Any], *, agg: str = "mean") -> float: + vectors = [self._flatten_numeric(s) for s in samples] + if not vectors: + raise ValueError("samples must not be empty") + merged = [v for row in vectors for v in row] + if agg == "mean": + return sum(merged) / len(merged) + if agg == "sum": + return sum(merged) + if agg == "var": + mean = sum(merged) / len(merged) + return sum((x - mean) ** 2 for x in merged) / len(merged) + raise ValueError(f"unsupported aggregate: {agg}") + + def estimate_selectivity(self, samples: Sequence[Any], predicate: callable) -> float: + total = 0 + hit = 0 + for s in samples: + total += 1 + if predicate(s): + hit += 1 + return 0.0 if total == 0 else hit / total + + def similarity(self, a: Sequence[float], b: Sequence[float]) -> float: + va = self._flatten_numeric(a) + vb = self._flatten_numeric(b) + n = min(len(va), len(vb)) + if n == 0: + return 0.0 + dot = sum(va[i] * vb[i] for i in range(n)) + na = math.sqrt(sum(va[i] * va[i] for i in range(n))) + nb = math.sqrt(sum(vb[i] * vb[i] for i in range(n))) + if na == 0.0 or nb == 0.0: + return 0.0 + return dot / (na * nb) + + def measure_locality(self, adjacency: Sequence[tuple[int, int, float]], radius: int = 1) -> float: + if radius <= 0: + raise ValueError("radius must be positive") + total = 0.0 + local = 0.0 + for i, j, weight in adjacency: + w = float(weight) + total += w + if abs(i - j) <= radius: + local += w + return 1.0 if total == 0 else local / total + + def _flatten_numeric(self, value: Any) -> list[float]: + if isinstance(value, (int, float)): + return [float(value)] + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + out: list[float] = [] + for item in value: + out.extend(self._flatten_numeric(item)) + return out + raise TypeError(f"unsupported sample item type: {type(value)!r}") + + def _dft_prefix(self, values: Sequence[float], k: int) -> list[complex]: + n = len(values) + out: list[complex] = [] + for freq in range(k): + coeff = 0j + for t, x in enumerate(values): + coeff += x * cmath.exp(-2j * math.pi * freq * t / n) + out.append(coeff) + return out + + def _pearson(self, x: Sequence[float], y: Sequence[float]) -> float: + n = len(x) + if n == 0: + return 0.0 + mx = sum(x) / n + my = sum(y) / n + num = sum((a - mx) * (b - my) for a, b in zip(x, y, strict=False)) + denx = math.sqrt(sum((a - mx) ** 2 for a in x)) + deny = math.sqrt(sum((b - my) ** 2 for b in y)) + if denx == 0.0 or deny == 0.0: + return 0.0 + return num / (denx * deny) + + def _project_to_sites(self, features: Sequence[Sequence[float]], *, num_sites: int) -> list[list[float]]: + if num_sites <= 0: + raise ValueError("num_sites must be positive") + n = len(features) + f = len(features[0]) + bounds = [round(i * f / num_sites) for i in range(num_sites + 1)] + projected: list[list[float]] = [[0.0] * num_sites for _ in range(n)] + for s in range(num_sites): + l, r = bounds[s], bounds[s + 1] + if l >= f: + l = f - 1 + if r <= l: + r = min(l + 1, f) + for i in range(n): + chunk = features[i][l:r] + if not chunk: + chunk = [features[i][min(l, f - 1)]] + projected[i][s] = sum(chunk) / len(chunk) + return projected + + def _discretize(self, values: Sequence[Sequence[float]], *, bins: int) -> list[list[int]]: + n = len(values) + m = len(values[0]) + out = [[0] * m for _ in range(n)] + for col in range(m): + column = sorted(row[col] for row in values) + thresholds = [column[min(len(column) - 1, int((q / bins) * (len(column) - 1)))] for q in range(1, bins)] + for i in range(n): + val = values[i][col] + idx = 0 + while idx < len(thresholds) and val > thresholds[idx]: + idx += 1 + out[i][col] = idx + return out + + def _single_probs(self, discrete: Sequence[Sequence[int]], phys_dim: int) -> list[list[float]]: + n = len(discrete) + sites = len(discrete[0]) + probs: list[list[float]] = [] + for s in range(sites): + hist = [1e-6] * phys_dim + for row in discrete: + hist[row[s]] += 1.0 + total = n + phys_dim * 1e-6 + probs.append([h / total for h in hist]) + return probs + + def _bond_dims(self, discrete: Sequence[Sequence[int]], *, phys_dim: int, chi_max: int, trunc_eps: float) -> list[int]: + sites = len(discrete[0]) + bonds: list[int] = [] + for s in range(sites - 1): + joint = [[1e-9] * phys_dim for _ in range(phys_dim)] + for row in discrete: + joint[row[s]][row[s + 1]] += 1.0 + total = sum(sum(r) for r in joint) + pxy = [[v / total for v in row] for row in joint] + px = [sum(row) for row in pxy] + py = [sum(pxy[i][j] for i in range(phys_dim)) for j in range(phys_dim)] + + mi = 0.0 + for i in range(phys_dim): + for j in range(phys_dim): + p = pxy[i][j] + q = px[i] * py[j] + if p > 0 and q > 0: + mi += p * math.log(p / q + 1e-12) + entropy_x = -sum(p * math.log(p + 1e-12) for p in px) + entropy_y = -sum(p * math.log(p + 1e-12) for p in py) + entropy_norm = max(0.0, min(1.0, ((entropy_x + entropy_y) / 2.0) / math.log(max(2, phys_dim)))) + mi_norm = max(0.0, min(1.0, mi / math.log(max(2, phys_dim)))) + rank_pressure = math.sqrt(mi_norm) * (entropy_norm ** 1.5) + raw = 1 + int(round((chi_max - 1) * rank_pressure + entropy_norm * trunc_eps)) + bonds.append(max(1, min(chi_max, raw))) + return bonds + + def _build_chain_tensors(self, singles: Sequence[Sequence[float]], bonds: Sequence[int]) -> list[list[list[list[float]]]]: + n = len(singles) + if n == 1: + return [[[list(singles[0])]]] + + tensors: list[list[list[list[float]]]] = [] + left_bond = 1 + for i, prob in enumerate(singles): + right_bond = 1 if i == n - 1 else bonds[i] + tensor: list[list[list[float]]] = [] + for _ in range(left_bond): + physical: list[list[float]] = [] + for p in prob: + physical.append([p / right_bond] * right_bond) + tensor.append(physical) + tensors.append(tensor) + left_bond = right_bond + return tensors diff --git a/systems/quantum_db/data_plane/tn/contraction.py b/systems/quantum_db/data_plane/tn/contraction.py new file mode 100644 index 0000000..a2c89ad --- /dev/null +++ b/systems/quantum_db/data_plane/tn/contraction.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class PhysicalTopologyGraph: + nodes: list[dict[str, Any]] + edges: list[dict[str, Any]] + coord_map: dict[str, tuple[int, int]] + quality_map: dict[str, float] + + +@dataclass +class Patch: + patch_id: str + member_qubits: list[str] + internal_edges: list[tuple[str, str]] + boundary_qubits: list[str] + interface_edges: list[tuple[str, str]] + patch_center: tuple[float, float] + patch_quality_score: float + hole_score: float + local_order_id: str + + +@dataclass +class PatchAdjacency: + src_patch: str + dst_patch: str + boundary_width: int + routing_cost: float + bridge_risk: float + merge_priority: float + + +@dataclass +class LocalOrderPhase: + id: str + nodes: list[str] + boundary: list[str] + + +@dataclass +class LocalOrder: + patch_id: str + leaf_phase: LocalOrderPhase + chain_phase: LocalOrderPhase + core_phase: LocalOrderPhase + boundary_phase: LocalOrderPhase + + +@dataclass +class MergeLevel: + level: int + merge_pairs: list[tuple[str, str]] + result_patch_id: str + consumed_interfaces: int + residual_boundary: int + estimated_depth_increment: float + + +@dataclass +class MergePlan: + merge_levels: list[MergeLevel] = field(default_factory=list) + + +@dataclass +class ExecutionOrderBundle: + local_orders: list[LocalOrder] + merge_plan: MergePlan + global_trace: list[str] + strategy: str + + def to_dict(self) -> dict[str, Any]: + return { + "excutionOrderBundle": { + "local_orders": [ + { + "patch_id": lo.patch_id, + "localOrder": { + "leaf_phase": {"id": lo.leaf_phase.id, "nodes": lo.leaf_phase.nodes, "boundary": lo.leaf_phase.boundary}, + "chain_phase": {"id": lo.chain_phase.id, "nodes": lo.chain_phase.nodes, "boundary": lo.chain_phase.boundary}, + "core_phase": {"id": lo.core_phase.id, "nodes": lo.core_phase.nodes, "boundary": lo.core_phase.boundary}, + "boundary_phase": {"id": lo.boundary_phase.id, "nodes": lo.boundary_phase.nodes, "boundary": lo.boundary_phase.boundary}, + }, + } + for lo in self.local_orders + ], + "merge_plan": { + "merge_levels": [ + { + "level": lv.level, + "merge_pairs": lv.merge_pairs, + "result_patch_id": lv.result_patch_id, + "consumed_interfaces": lv.consumed_interfaces, + "residual_boundary": lv.residual_boundary, + "estimated_depth_increment": lv.estimated_depth_increment, + } + for lv in self.merge_plan.merge_levels + ] + }, + "global_trace": list(self.global_trace), + "strategy": self.strategy, + } + } + + +class TNContractionPlanner: + """Patch 拓扑感知收缩规划：主/兼容两种 order。""" + + __slots__ = () + + def build_patch_topology(self, topology: PhysicalTopologyGraph, partition: dict[str, str]) -> tuple[list[Patch], list[PatchAdjacency]]: + patch_members: dict[str, list[str]] = {} + for q in topology.coord_map: + patch_members.setdefault(partition.get(q, "p0"), []).append(q) + + edge_pairs: list[tuple[str, str, float]] = [] + for edge in topology.edges: + a, b = edge.get("qubits", (None, None)) + if a is None or b is None: + continue + edge_pairs.append((str(a), str(b), float(edge.get("quality", 1.0)))) + + patches: list[Patch] = [] + for pid, members in sorted(patch_members.items()): + mset = set(members) + internal: list[tuple[str, str]] = [] + interfaces: list[tuple[str, str]] = [] + boundary: set[str] = set() + qsum = 0.0 + qcnt = 0 + for a, b, q in edge_pairs: + if a in mset and b in mset: + internal.append((a, b)) + qsum += q + qcnt += 1 + elif a in mset or b in mset: + interfaces.append((a, b)) + boundary.add(a if a in mset else b) + center = self._center(members, topology.coord_map) + possible_internal = max(1, len(members) * 2) + hole_score = max(0.0, 1.0 - (len(internal) / possible_internal)) + patches.append( + Patch( + patch_id=pid, + member_qubits=sorted(members), + internal_edges=internal, + boundary_qubits=sorted(boundary), + interface_edges=interfaces, + patch_center=center, + patch_quality_score=(qsum / qcnt) if qcnt else 0.0, + hole_score=hole_score, + local_order_id=f"{pid}-local", + ) + ) + + adjacency: list[PatchAdjacency] = [] + pindex = {p.patch_id: p for p in patches} + patch_ids = sorted(pindex) + for i, src in enumerate(patch_ids): + for dst in patch_ids[i + 1 :]: + w = self._interface_width(pindex[src], pindex[dst]) + if w <= 0: + continue + cost = 1.0 / max(1, w) + risk = (pindex[src].hole_score + pindex[dst].hole_score) / 2.0 + priority = w * (2.0 - risk) + adjacency.append(PatchAdjacency(src, dst, w, cost, risk, priority)) + adjacency.sort(key=lambda x: x.merge_priority, reverse=True) + return patches, adjacency + + def plan_main_order(self, patches: list[Patch], adjacency: list[PatchAdjacency]) -> ExecutionOrderBundle: + local_orders = [self._build_local_order_patch_inward(p) for p in patches] + merge_plan = self._build_hierarchical_merge(patches, adjacency) + trace: list[str] = [] + for lo in local_orders: + trace.extend([lo.leaf_phase.id, lo.chain_phase.id, lo.core_phase.id, lo.boundary_phase.id]) + for level in merge_plan.merge_levels: + trace.append(f"merge-L{level.level}:{level.result_patch_id}") + return ExecutionOrderBundle(local_orders=local_orders, merge_plan=merge_plan, global_trace=trace, strategy="patch_inward_hierarchical") + + def plan_compat_order(self, patches: list[Patch], window_size: int = 4) -> ExecutionOrderBundle: + local_orders = [self._build_local_order_snake_window(p, window_size=window_size) for p in patches] + merge_plan = self._build_balanced_merge([p.patch_id for p in patches]) + trace: list[str] = [] + for lo in local_orders: + trace.extend([lo.leaf_phase.id, lo.chain_phase.id, lo.core_phase.id, lo.boundary_phase.id]) + for level in merge_plan.merge_levels: + trace.append(f"merge-L{level.level}:{level.result_patch_id}") + return ExecutionOrderBundle(local_orders=local_orders, merge_plan=merge_plan, global_trace=trace, strategy="snake_window_balanced") + + def _build_local_order_patch_inward(self, patch: Patch) -> LocalOrder: + leaf_nodes = [q for q in patch.member_qubits if q not in patch.boundary_qubits] + chain_nodes = list(patch.boundary_qubits) + core_nodes = patch.member_qubits[: max(1, len(patch.member_qubits) // 3)] + return LocalOrder( + patch_id=patch.patch_id, + leaf_phase=LocalOrderPhase(id=f"{patch.patch_id}.leaf_phase", nodes=leaf_nodes, boundary=patch.boundary_qubits), + chain_phase=LocalOrderPhase(id=f"{patch.patch_id}.chain_phase", nodes=chain_nodes, boundary=patch.boundary_qubits), + core_phase=LocalOrderPhase(id=f"{patch.patch_id}.core_phase", nodes=core_nodes, boundary=patch.boundary_qubits), + boundary_phase=LocalOrderPhase(id=f"{patch.patch_id}.boundary_phase", nodes=patch.boundary_qubits, boundary=patch.boundary_qubits), + ) + + def _build_local_order_snake_window(self, patch: Patch, *, window_size: int) -> LocalOrder: + snake = list(patch.member_qubits) + windows: list[str] = [] + for i in range(0, len(snake), max(1, window_size)): + windows.extend(snake[i : i + window_size]) + core = windows[: max(1, len(windows) // 2)] + return LocalOrder( + patch_id=patch.patch_id, + leaf_phase=LocalOrderPhase(id=f"{patch.patch_id}.leaf_phase", nodes=snake, boundary=patch.boundary_qubits), + chain_phase=LocalOrderPhase(id=f"{patch.patch_id}.chain_phase", nodes=windows, boundary=patch.boundary_qubits), + core_phase=LocalOrderPhase(id=f"{patch.patch_id}.core_phase", nodes=core, boundary=patch.boundary_qubits), + boundary_phase=LocalOrderPhase(id=f"{patch.patch_id}.boundary_phase", nodes=patch.boundary_qubits, boundary=patch.boundary_qubits), + ) + + def _build_hierarchical_merge(self, patches: list[Patch], adjacency: list[PatchAdjacency]) -> MergePlan: + active = {p.patch_id for p in patches} + levels: list[MergeLevel] = [] + lvl = 1 + for adj in adjacency: + if adj.src_patch not in active or adj.dst_patch not in active: + continue + merged = f"{adj.src_patch}+{adj.dst_patch}" + active.remove(adj.src_patch) + active.remove(adj.dst_patch) + active.add(merged) + levels.append(MergeLevel(level=lvl, merge_pairs=[(adj.src_patch, adj.dst_patch)], result_patch_id=merged, consumed_interfaces=adj.boundary_width, residual_boundary=max(0, adj.boundary_width - 1), estimated_depth_increment=adj.routing_cost + adj.bridge_risk)) + lvl += 1 + if len(active) > 1: + remain = sorted(active) + levels.append(MergeLevel(level=lvl, merge_pairs=[(remain[0], remain[1])], result_patch_id=f"{remain[0]}+{remain[1]}", consumed_interfaces=1, residual_boundary=1, estimated_depth_increment=1.0)) + if not levels: + levels.append(MergeLevel(level=0, merge_pairs=[], result_patch_id=patches[0].patch_id if patches else "none", consumed_interfaces=0, residual_boundary=0, estimated_depth_increment=0.0)) + return MergePlan(merge_levels=levels) + + def _build_balanced_merge(self, patch_ids: list[str]) -> MergePlan: + current = list(sorted(patch_ids)) + levels: list[MergeLevel] = [] + level = 1 + while len(current) > 1: + nxt: list[str] = [] + pairs: list[tuple[str, str]] = [] + depth = 0.0 + for i in range(0, len(current), 2): + if i + 1 >= len(current): + nxt.append(current[i]) + continue + a, b = current[i], current[i + 1] + pairs.append((a, b)) + mid = f"{a}+{b}" + nxt.append(mid) + depth += math.log2(3) + levels.append(MergeLevel(level=level, merge_pairs=pairs, result_patch_id="|".join(nxt), consumed_interfaces=len(pairs), residual_boundary=max(0, len(nxt) - 1), estimated_depth_increment=depth)) + current = nxt + level += 1 + return MergePlan(merge_levels=levels) + + def _center(self, members: list[str], coord_map: dict[str, tuple[int, int]]) -> tuple[float, float]: + xs: list[int] = [] + ys: list[int] = [] + for q in members: + x, y = coord_map.get(q, (0, 0)) + xs.append(x) + ys.append(y) + if not xs: + return (0.0, 0.0) + return (sum(xs) / len(xs), sum(ys) / len(ys)) + + def _interface_width(self, src: Patch, dst: Patch) -> int: + src_set = set(src.member_qubits) + dst_set = set(dst.member_qubits) + width = 0 + for a, b in src.interface_edges: + if (a in src_set and b in dst_set) or (b in src_set and a in dst_set): + width += 1 + return width diff --git a/systems/quantum_db/data_plane/tn/executor.py b/systems/quantum_db/data_plane/tn/executor.py new file mode 100644 index 0000000..9f84db8 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/executor.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from data_plane.tn.cache import TNMultiLayerCache + +_ROUTE_NAMES: tuple[str, ...] = ("tn.execute", "tn") +_TARGETS: frozenset[str] = frozenset(("page", "compute", "cache")) + + +def _first_text(source: Mapping[str, Any], *keys: str) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str): + text = value.strip() + if text: + return text + return None + + +def _split_target_action(action: str | None) -> tuple[str | None, str]: + if not action: + return None, "" + text = action.strip() + if not text: + return None, "" + for sep in (".", ":", "/", "|"): + if sep not in text: + continue + target, sub_action = text.split(sep, 1) + normalized = target.strip().casefold() + if normalized in _TARGETS: + return normalized, sub_action.strip() + return None, text + + +def _pick_payload(command: Mapping[str, Any]) -> Any: + if "payload" in command: + return command["payload"] + if "params" in command: + return command["params"] + if "data" in command: + return command["data"] + return None + + +def _invoke_adapter( + adapter: Any, + action: str, + payload: Any, + metadata: Mapping[str, Any] | None, +) -> Any: + if callable(adapter): + try: + return adapter(action, payload, metadata) + except TypeError: + try: + return adapter(action, payload) + except TypeError: + return adapter(payload) + + method = getattr(adapter, action, None) + if callable(method): + try: + return method(payload, metadata) + except TypeError: + try: + return method(payload) + except TypeError: + return method() + + for fallback in ("execute", "run", "handle"): + method = getattr(adapter, fallback, None) + if not callable(method): + continue + try: + return method(action, payload, metadata) + except TypeError: + try: + return method(action, payload) + except TypeError: + try: + return method(payload, metadata) + except TypeError: + try: + return method(payload) + except TypeError: + return method() + + raise TypeError(f"Unsupported TN adapter type: {type(adapter)!r}") + + +class _EchoAdapter: + __slots__ = ("name",) + + def __init__(self, name: str) -> None: + self.name = name + + def execute( + self, + action: str, + payload: Any, + metadata: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + result: dict[str, Any] = { + "target": self.name, + "action": action, + "payload": payload, + } + if metadata: + result["metadata"] = dict(metadata) + return result + + +class TNExecutor: + __slots__ = ("bus", "monitor", "started", "_modules") + + def __init__( + self, + bus: Any | None = None, + monitor: Any | None = None, + *, + page: Any | None = None, + compute: Any | None = None, + cache: Any | None = None, + **_: Any, + ) -> None: + self.bus = bus + self.monitor = monitor + self.started = False + self._modules: dict[str, Any] = { + "page": page if page is not None else _EchoAdapter("page"), + "compute": compute if compute is not None else _EchoAdapter("compute"), + "cache": cache if cache is not None else TNMultiLayerCache(), + } + + def start(self) -> None: + self.started = True + register = getattr(self.bus, "register", None) + if callable(register): + for name in _ROUTE_NAMES: + register(name, self._on_bus_command, lane="command") + + def stop(self) -> None: + unregister = getattr(self.bus, "unregister", None) + if callable(unregister): + for name in _ROUTE_NAMES: + unregister(name, lane="command") + self.started = False + + def bind(self, target: str, module: Any) -> None: + normalized = target.strip().casefold() + if normalized not in _TARGETS: + raise ValueError(f"Unsupported TN target: {target!r}") + self._modules[normalized] = module + + def execute( + self, + command: Mapping[str, Any], + metadata: Mapping[str, Any] | None = None, + ) -> Any: + if not isinstance(command, Mapping): + raise TypeError("TN command must be a mapping") + + action_text = _first_text(command, "action", "op", "name", "command") + target = _first_text(command, "target", "module", "component", "domain") + implied_target, action = _split_target_action(action_text) + if target is None: + target = implied_target + + if target is None: + raise ValueError("TN command missing target/module") + normalized_target = target.strip().casefold() + adapter = self._modules.get(normalized_target) + if adapter is None: + raise KeyError(f"Unsupported TN target: {target!r}") + + payload = _pick_payload(command) + result = _invoke_adapter(adapter, action, payload, metadata) + self._report_success(normalized_target) + return result + + def _on_bus_command( + self, + payload: Any, + metadata: Mapping[str, Any] | None = None, + ) -> Any: + if isinstance(payload, Mapping): + return self.execute(payload, metadata=metadata) + raise TypeError("TN bus payload must be a mapping command") + + def _report_success(self, target: str) -> None: + monitor = self.monitor + if monitor is None: + return + report_metric = getattr(monitor, "report_metric", None) + if callable(report_metric): + report_metric("tn_executor", "performance", f"{target}_calls", 1.0) + + __call__ = execute diff --git a/systems/quantum_db/data_plane/tn/exproter.py b/systems/quantum_db/data_plane/tn/exproter.py new file mode 100644 index 0000000..58ef554 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/exproter.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import math +from typing import Any + +from data_plane.tn.page import TNPage + + +@dataclass +class QuantumExportPacket: + """导出到量子层的标准包。""" + + model_id: str + layout: str + encoding: str + tensors: list[Any] = field(default_factory=list) + constraints: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + ir_version: str = "v1" + + +class TNExporter: + """将 TN 结果导出为量子层可消费对象，不直接调用量子后端。""" + + __slots__ = ("bus",) + + def __init__(self, bus: Any | None = None) -> None: + self.bus = bus + + def export( + self, + page: TNPage, + *, + target_backend: str, + max_two_qubit_depth: int | None = None, + allowed_encodings: list[str] | None = None, + graph_hardware: dict[str, Any] | None = None, + ) -> QuantumExportPacket: + constraints: dict[str, Any] = {"target_backend": target_backend} + if max_two_qubit_depth is not None: + constraints["max_two_qubit_depth"] = max_two_qubit_depth + if allowed_encodings is not None: + constraints["allowed_encodings"] = list(allowed_encodings) + + metadata = { + "graph_data_ref": page.metadata.graph_data_ref, + "graph_hardware_ref": page.metadata.graph_hardware_ref, + "mapping_ref": page.metadata.mapping_ref, + "physical_dims": tuple(page.metadata.physical_dims), + "bond_dims": tuple(page.metadata.bond_dims), + "tn_mode": page.metadata.extra.get("tn_mode", ""), + "extra": dict(page.metadata.extra), + "stats": dict(page.stats), + "artifacts": dict(page.artifacts), + "logical_to_physical": dict(page.artifacts.get("logical_to_physical", {})), + } + if graph_hardware is not None: + metadata["graph_hardware"] = graph_hardware + + packet = QuantumExportPacket( + model_id=page.page_id, + layout=page.metadata.layout, + encoding=page.metadata.encoding, + tensors=list(page.tensors), + constraints=constraints, + metadata=metadata, + ) + self.validate(packet) + return packet + + def export_via_bus(self, page: TNPage, *, target_backend: str, metadata: dict[str, Any] | None = None) -> Any: + if self.bus is None: + return self.export(page, target_backend=target_backend) + payload = { + "target": "exporter", + "action": "export", + "payload": { + "page_id": page.page_id, + "target_backend": target_backend, + }, + } + sender = getattr(self.bus, "send", None) + if callable(sender): + return sender("tn.execute", payload, lane="command", metadata=metadata) + return self.export(page, target_backend=target_backend) + + def validate(self, packet: QuantumExportPacket) -> None: + if not packet.model_id.strip(): + raise ValueError("model_id must not be empty") + if packet.layout not in {"mps", "tt", "tree"}: + raise ValueError(f"unsupported layout: {packet.layout}") + if not packet.constraints.get("target_backend"): + raise ValueError("target_backend is required") + + allowed = packet.constraints.get("allowed_encodings") + if allowed and packet.encoding not in set(allowed): + raise ValueError(f"encoding {packet.encoding!r} not in allowed_encodings") + + def validate_ir_contract(self, ir: dict[str, Any]) -> None: + if not str(ir.get("model_id", "")).strip(): + raise ValueError("IR missing model_id") + if not isinstance(ir.get("operations", []), list): + raise TypeError("IR operations must be a list") + if "target_backend" not in ir: + raise ValueError("IR missing target_backend") + + def _effective_scalar_count(self, packet: QuantumExportPacket) -> int: + physical_dims = packet.metadata.get("physical_dims", ()) + bond_dims = packet.metadata.get("bond_dims", ()) + count = sum(int(dim) for dim in physical_dims) + sum(int(dim) for dim in bond_dims) + return max(1, count) + + def to_quantum_ir(self, packet: QuantumExportPacket) -> dict[str, Any]: + """预留量子层通行接口：返回硬件无关 IR。""" + ops: list[dict[str, Any]] = [] + stats = packet.metadata.get("stats", {}) + physical_dims = packet.metadata.get("physical_dims", ()) + num_sites = max(1, len(packet.tensors)) + if physical_dims: + max_phys_dim = max(int(dim) for dim in physical_dims) + else: + max_phys_dim = 2 + qubits_per_site = max(1, int(math.ceil(math.log2(max(2, max_phys_dim))))) + effective_scalars = self._effective_scalar_count(packet) + index_qubits = max(1, int(math.ceil(math.log2(max(2, effective_scalars))))) + value_precision_qubits = int(packet.metadata.get("extra", {}).get("native_mapping", {}).get("value_precision_qubits", 8)) + site_qubits = max(1, int(math.ceil(math.log2(max(2, num_sites))))) + phys_qubits = qubits_per_site + bond_dims = tuple(int(dim) for dim in packet.metadata.get("bond_dims", ())) + max_bond_dim = max(bond_dims, default=1) + bond_qubits = max(1, int(math.ceil(math.log2(max(2, max_bond_dim))))) + tn_mode = str(packet.metadata.get("tn_mode", packet.metadata.get("extra", {}).get("tn_mode", "")) or "") + if not tn_mode: + tn_mode = "native_mapping" if packet.encoding == "native_mapping" else ("tree" if packet.layout == "tree" else "mps_linear") + native_register_qubits = index_qubits + value_precision_qubits + mps_streaming_qubits = site_qubits + phys_qubits + 2 * bond_qubits + value_precision_qubits + tree_frontier_qubits = site_qubits + materialized_table_qubits = effective_scalars if tn_mode == "native_mapping" else 0 + core_load_info = [] + physical_dims_tuple = tuple(int(dim) for dim in physical_dims) + for core_idx in range(num_sites): + physical_dim = physical_dims_tuple[core_idx] if core_idx < len(physical_dims_tuple) else max_phys_dim + left_bond_dim = 1 if core_idx == 0 else (bond_dims[core_idx - 1] if core_idx - 1 < len(bond_dims) else 1) + right_bond_dim = 1 if core_idx >= len(bond_dims) else bond_dims[core_idx] + core_phys_qubits = max(1, int(math.ceil(math.log2(max(2, physical_dim))))) + core_left_bond_qubits = 0 if core_idx == 0 else max(1, int(math.ceil(math.log2(max(2, left_bond_dim))))) + core_right_bond_qubits = 0 if core_idx == num_sites - 1 else max(1, int(math.ceil(math.log2(max(2, right_bond_dim))))) + core_information_qubits = core_phys_qubits + core_left_bond_qubits + core_right_bond_qubits + value_precision_qubits + core_payload_scalars = physical_dim + (0 if core_idx == 0 else left_bond_dim) + (0 if core_idx == num_sites - 1 else right_bond_dim) + core_load_info.append( + { + "core_index": core_idx, + "physical_dim": physical_dim, + "left_bond_dim": left_bond_dim, + "right_bond_dim": right_bond_dim, + "physical_qubits": core_phys_qubits, + "left_bond_qubits": core_left_bond_qubits, + "right_bond_qubits": core_right_bond_qubits, + "value_precision_qubits": value_precision_qubits, + "information_qubits": core_information_qubits, + "payload_scalars": core_payload_scalars, + "dense_scalar_count": left_bond_dim * physical_dim * right_bond_dim, + "local_marginal_scalars": physical_dim, + "left_bond_rank_scalars": 0 if core_idx == 0 else left_bond_dim, + "right_bond_rank_scalars": 0 if core_idx == num_sites - 1 else right_bond_dim, + } + ) + mps_all_core_qubits = sum(core["information_qubits"] for core in core_load_info) + core_start_indices: list[int] = [] + cursor = 0 + for core in core_load_info: + core_start_indices.append(cursor) + cursor += int(core["information_qubits"]) + max_core_information_qubits = max((int(core["information_qubits"]) for core in core_load_info), default=1) + max_core_payload_scalars = max((int(core["payload_scalars"]) for core in core_load_info), default=1) + total_core_payload_scalars = sum(int(core["payload_scalars"]) for core in core_load_info) + core_address_auxiliary_qubits = site_qubits * num_sites + if tn_mode == "native_mapping": + num_qubits = native_register_qubits + materialized_table_qubits + elif tn_mode == "tree": + num_qubits = mps_all_core_qubits + core_address_auxiliary_qubits + tree_frontier_qubits + else: + num_qubits = mps_all_core_qubits + core_address_auxiliary_qubits + if tn_mode == "native_mapping": + information_qubits = materialized_table_qubits + logical_auxiliary_qubits = native_register_qubits + elif tn_mode == "tree": + information_qubits = mps_all_core_qubits + logical_auxiliary_qubits = core_address_auxiliary_qubits + tree_frontier_qubits + else: + information_qubits = mps_all_core_qubits + logical_auxiliary_qubits = core_address_auxiliary_qubits + qubit_load_info = { + "num_sites": num_sites, + "num_cores": num_sites, + "max_phys_dim": max_phys_dim, + "qubits_per_site": qubits_per_site, + "data_loading_qubits": num_qubits, + "definition": ( + "mps_linear streams one tensor with site, physical, left/right bond, and value registers; " + "tree adds frontier/level control qubits for parallel merging; " + "native_mapping materializes the full compressed table plus address/value registers" + ), + "includes_ancilla_or_routing_qubits": False, + "effective_scalar_count": effective_scalars, + "index_qubits": index_qubits, + "site_qubits": site_qubits, + "phys_qubits": phys_qubits, + "bond_qubits": bond_qubits, + "value_precision_qubits": value_precision_qubits, + "mps_streaming_qubits": mps_streaming_qubits, + "tree_frontier_qubits": tree_frontier_qubits, + "mps_all_core_qubits": mps_all_core_qubits, + "core_address_auxiliary_qubits": core_address_auxiliary_qubits, + "tree_all_core_qubits": mps_all_core_qubits + core_address_auxiliary_qubits + tree_frontier_qubits, + "native_register_qubits": native_register_qubits, + "materialized_table_qubits": materialized_table_qubits, + "information_qubits": information_qubits, + "logical_auxiliary_qubits": logical_auxiliary_qubits, + "core_load_info": core_load_info, + "core_start_indices": core_start_indices, + "core_load_summary": { + "dense_scalar_count": sum(core["dense_scalar_count"] for core in core_load_info), + "local_marginal_scalars": sum(core["local_marginal_scalars"] for core in core_load_info), + "retained_bond_rank_scalars": sum(bond_dims), + "all_core_information_qubits": mps_all_core_qubits, + "max_core_information_qubits": max_core_information_qubits, + "max_core_payload_scalars": max_core_payload_scalars, + "total_core_payload_scalars": total_core_payload_scalars, + }, + } + + # prepare layer: initialize every logical qubit needed by the loader. + for i in range(num_qubits): + ops.append({"gate": "u3", "qubits": [i], "params": [0.0, 0.0, 0.0]}) + + site_roots = list(range(num_qubits)) + if tn_mode == "native_mapping": + # Native direct mapping materializes every compressed-table entry and + # touches a shared address/value register for each parameter. Reusing + # the register makes the loader depth scale with table cardinality. + shared_register = 0 + for _ in range(effective_scalars): + ops.append({"gate": "u3", "qubits": [shared_register], "params": [0.0, 0.0, 0.0]}) + elif tn_mode == "tree": + for step in range(max_core_payload_scalars): + for core, root in zip(core_load_info, core_start_indices, strict=False): + if step < int(core["payload_scalars"]): + ops.append({"gate": "u3", "qubits": [root], "params": [0.0, 0.0, 0.0]}) + active = list(core_start_indices) + while len(active) > 1: + next_active: list[int] = [] + for i in range(0, len(active), 2): + if i + 1 < len(active): + ops.append({"gate": "cz", "qubits": [active[i], active[i + 1]]}) + next_active.append(active[i]) + active = next_active + elif tn_mode == "mps_linear": + for step in range(max_core_payload_scalars): + for core, root in zip(core_load_info, core_start_indices, strict=False): + if step < int(core["payload_scalars"]): + ops.append({"gate": "u3", "qubits": [root], "params": [0.0, 0.0, 0.0]}) + for i in range(max(0, len(core_start_indices) - 1)): + ops.append({"gate": "cz", "qubits": [core_start_indices[i], core_start_indices[i + 1]]}) + else: + raise ValueError(f"unsupported tn_mode for quantum IR: {tn_mode}") + + for i in range(num_qubits): + ops.append({"gate": "measure", "qubits": [i], "cbit": i}) + + ir = { + "ir_version": packet.ir_version, + "model_id": packet.model_id, + "layout": packet.layout, + "encoding": packet.encoding, + "tn_mode": tn_mode, + "target_backend": packet.constraints.get("target_backend"), + "constraints": packet.constraints, + "logical_to_physical": packet.metadata.get("logical_to_physical", {}), + "operations": ops, + "qubit_load_info": qubit_load_info, + "metadata": packet.metadata, + } + self.validate_ir_contract(ir) + return ir diff --git a/systems/quantum_db/data_plane/tn/modes.py b/systems/quantum_db/data_plane/tn/modes.py new file mode 100644 index 0000000..02b7429 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/modes.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class TNMode(str, Enum): + MPS_LINEAR = "mps_linear" + TREE = "tree" + NATIVE = "native_mapping" + + +@dataclass +class BuildStats: + mode: TNMode + qubits: int + depth: int + fidelity: float + notes: dict[str, Any] + + +def estimate_qubits_depth(mode: TNMode, num_sites: int, phys_dim: int, bond_dims: tuple[int, ...]) -> tuple[int, int]: + effective_scalars = max(1, num_sites * phys_dim + sum(int(dim) for dim in bond_dims)) + index_qubits = max(1, int(math.ceil(math.log2(max(2, effective_scalars))))) + value_precision_qubits = 8 + site_qubits = max(1, int(math.ceil(math.log2(max(2, num_sites))))) + phys_qubits = max(1, int(math.ceil(math.log2(max(2, phys_dim))))) + max_bond = max((int(b) for b in bond_dims), default=1) + bond_qubits = max(1, int(math.ceil(math.log2(max(2, max_bond))))) + all_core_qubits = 0 + max_core_qubits = 1 + max_core_payload_scalars = 1 + total_core_payload_scalars = 0 + for core_idx in range(num_sites): + left_bond = 1 if core_idx == 0 else (int(bond_dims[core_idx - 1]) if core_idx - 1 < len(bond_dims) else 1) + right_bond = 1 if core_idx >= len(bond_dims) else int(bond_dims[core_idx]) + left_bond_qubits = 0 if core_idx == 0 else max(1, int(math.ceil(math.log2(max(2, left_bond))))) + right_bond_qubits = 0 if core_idx == num_sites - 1 else max(1, int(math.ceil(math.log2(max(2, right_bond))))) + core_qubits = phys_qubits + left_bond_qubits + right_bond_qubits + value_precision_qubits + core_payload_scalars = phys_dim + (0 if core_idx == 0 else left_bond) + (0 if core_idx == num_sites - 1 else right_bond) + all_core_qubits += core_qubits + max_core_qubits = max(max_core_qubits, core_qubits) + max_core_payload_scalars = max(max_core_payload_scalars, core_payload_scalars) + total_core_payload_scalars += core_payload_scalars + core_address_auxiliary_qubits = site_qubits * num_sites + if mode == TNMode.NATIVE: + qubits = index_qubits + value_precision_qubits + effective_scalars + depth = effective_scalars + 1 + elif mode == TNMode.MPS_LINEAR: + qubits = all_core_qubits + core_address_auxiliary_qubits + depth = max_core_payload_scalars + max(0, num_sites - 1) + else: + qubits = all_core_qubits + core_address_auxiliary_qubits + site_qubits + depth = max_core_payload_scalars + int(math.ceil(math.log2(max(2, num_sites)))) + return qubits, max(1, depth) diff --git a/systems/quantum_db/data_plane/tn/multi_cache.py b/systems/quantum_db/data_plane/tn/multi_cache.py new file mode 100644 index 0000000..e8640c9 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/multi_cache.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import hashlib +import json +import time +from dataclasses import dataclass, field +from typing import Any + +from data_plane.tn.cache import TNCache +from data_plane.tn.scoring import ScoreWeights, unified_score + + +@dataclass +class QueryCanonicalEntry: + raw_query: str + normalized_query: str + fingerprint: str + parser_version: str = "v1" + created_at_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +@dataclass +class TNPlanEntry: + plan_id: str + query_fingerprint: str + tn_graph: dict[str, Any] + rewrite_result: dict[str, Any] + contraction_order: list[int] + slicing_strategy: dict[str, Any] + physical_layout_choice: dict[str, Any] + execution_path: list[str] + quality_annotation: dict[str, Any] + score: float = 0.0 + score_breakdown: dict[str, float] = field(default_factory=dict) + created_at_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +@dataclass +class ArtifactEntry: + artifact_type: str + artifact_id: str + plan_id: str + payload: Any + created_at_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +class QueryCanonicalCache: + __slots__ = ("_cache",) + + def __init__(self, capacity: int = 256) -> None: + self._cache = TNCache(capacity=capacity) + + def normalize_query(self, query: str) -> str: + return " ".join(query.strip().lower().split()) + + def fingerprint(self, normalized_query: str, parser_version: str = "v1") -> str: + digest = hashlib.sha256(f"{parser_version}:{normalized_query}".encode("utf-8")).hexdigest() + return digest[:24] + + def put(self, query: str, *, parser_version: str = "v1") -> QueryCanonicalEntry: + normalized = self.normalize_query(query) + fp = self.fingerprint(normalized, parser_version=parser_version) + entry = QueryCanonicalEntry( + raw_query=query, + normalized_query=normalized, + fingerprint=fp, + parser_version=parser_version, + ) + self._cache.put(fp, entry) + return entry + + def get(self, query: str, *, parser_version: str = "v1") -> QueryCanonicalEntry | None: + normalized = self.normalize_query(query) + fp = self.fingerprint(normalized, parser_version=parser_version) + value = self._cache.get(fp) + return value if isinstance(value, QueryCanonicalEntry) else None + + def get_by_fingerprint(self, fingerprint: str) -> QueryCanonicalEntry | None: + value = self._cache.get(fingerprint) + return value if isinstance(value, QueryCanonicalEntry) else None + + def snapshot(self) -> dict[str, Any]: + return self._cache.snapshot() + + +class TNPlanCache: + __slots__ = ("_cache",) + + def __init__(self, capacity: int = 256) -> None: + self._cache = TNCache(capacity=capacity) + + def _plan_fingerprint(self, query_fingerprint: str, plan_payload: dict[str, Any]) -> str: + canonical = json.dumps(plan_payload, sort_keys=True, ensure_ascii=False) + digest = hashlib.sha256(f"{query_fingerprint}:{canonical}".encode("utf-8")).hexdigest() + return digest[:24] + + def put( + self, + *, + query_fingerprint: str, + tn_graph: dict[str, Any], + rewrite_result: dict[str, Any], + contraction_order: list[int], + slicing_strategy: dict[str, Any], + physical_layout_choice: dict[str, Any], + execution_path: list[str], + quality_annotation: dict[str, Any], + weights: ScoreWeights | None = None, + ) -> TNPlanEntry: + score, breakdown = unified_score( + query_utility=float(quality_annotation.get("query_utility", 0.0)), + approximation_error=float(quality_annotation.get("approximation_error", 1.0)), + reconstruction_cost=float(quality_annotation.get("reconstruction_cost", 1.0)), + fidelity_loss=float(quality_annotation.get("fidelity_loss", 1.0)), + materialization_latency=float(quality_annotation.get("materialization_latency", 1.0)), + weights=weights, + ) + payload = { + "tn_graph": tn_graph, + "rewrite_result": rewrite_result, + "contraction_order": contraction_order, + "slicing_strategy": slicing_strategy, + "physical_layout_choice": physical_layout_choice, + "execution_path": execution_path, + } + plan_id = self._plan_fingerprint(query_fingerprint, payload) + entry = TNPlanEntry( + plan_id=plan_id, + query_fingerprint=query_fingerprint, + tn_graph=tn_graph, + rewrite_result=rewrite_result, + contraction_order=list(contraction_order), + slicing_strategy=dict(slicing_strategy), + physical_layout_choice=dict(physical_layout_choice), + execution_path=list(execution_path), + quality_annotation=dict(quality_annotation), + score=score, + score_breakdown=breakdown, + ) + self._cache.put(plan_id, entry) + return entry + + def get(self, plan_id: str) -> TNPlanEntry | None: + value = self._cache.get(plan_id) + return value if isinstance(value, TNPlanEntry) else None + + def choose_best(self, plan_ids: list[str]) -> TNPlanEntry | None: + best: TNPlanEntry | None = None + for plan_id in plan_ids: + entry = self.get(plan_id) + if entry is None: + continue + if best is None or entry.score > best.score: + best = entry + return best + + def snapshot(self) -> dict[str, Any]: + return self._cache.snapshot() + + +class MaterializedArtifactCache: + __slots__ = ("_stores",) + + ARTIFACT_TYPES: tuple[str, ...] = ( + "tn_page", + "decompressed_tensor_block", + "boundary_tensor", + "partial_contraction", + "quantum_circuit_template", + "calibrated_hardware_mapping", + "measurement_result_cache", + "compiled_qpanda_program", + "execution_trace", + "measurement_histogram", + ) + + def __init__(self, *, per_type_capacity: int = 128) -> None: + self._stores: dict[str, TNCache] = { + name: TNCache(capacity=per_type_capacity) for name in self.ARTIFACT_TYPES + } + + def put(self, *, artifact_type: str, artifact_id: str, plan_id: str, payload: Any) -> ArtifactEntry: + if artifact_type not in self._stores: + raise ValueError(f"unsupported artifact type: {artifact_type}") + entry = ArtifactEntry( + artifact_type=artifact_type, + artifact_id=artifact_id, + plan_id=plan_id, + payload=payload, + ) + self._stores[artifact_type].put(artifact_id, entry) + return entry + + def get(self, *, artifact_type: str, artifact_id: str) -> ArtifactEntry | None: + if artifact_type not in self._stores: + raise ValueError(f"unsupported artifact type: {artifact_type}") + value = self._stores[artifact_type].get(artifact_id) + return value if isinstance(value, ArtifactEntry) else None + + def snapshot(self) -> dict[str, Any]: + return {name: store.snapshot() for name, store in self._stores.items()} + + +class TNMultiLayerCache: + """三层缓存：query canonical / plan / materialized artifact。""" + + __slots__ = ("query", "plan", "artifact") + + def __init__(self) -> None: + self.query = QueryCanonicalCache() + self.plan = TNPlanCache() + self.artifact = MaterializedArtifactCache() + + + def execute(self, action: str, payload: Any, metadata: dict[str, Any] | None = None) -> Any: + data = payload if isinstance(payload, dict) else {} + if action == "query.put": + return self.query.put(str(data.get("query", "")), parser_version=str(data.get("parser_version", "v1"))) + if action == "query.get": + return self.query.get(str(data.get("query", "")), parser_version=str(data.get("parser_version", "v1"))) + if action == "plan.put": + return self.plan.put( + query_fingerprint=str(data.get("query_fingerprint", "")), + tn_graph=dict(data.get("tn_graph", {})), + rewrite_result=dict(data.get("rewrite_result", {})), + contraction_order=list(data.get("contraction_order", [])), + slicing_strategy=dict(data.get("slicing_strategy", {})), + physical_layout_choice=dict(data.get("physical_layout_choice", {})), + execution_path=list(data.get("execution_path", [])), + quality_annotation=dict(data.get("quality_annotation", {})), + ) + if action == "plan.get": + return self.plan.get(str(data.get("plan_id", ""))) + if action == "artifact.put": + return self.artifact.put( + artifact_type=str(data.get("artifact_type", "")), + artifact_id=str(data.get("artifact_id", "")), + plan_id=str(data.get("plan_id", "")), + payload=data.get("payload"), + ) + if action == "artifact.get": + return self.artifact.get( + artifact_type=str(data.get("artifact_type", "")), + artifact_id=str(data.get("artifact_id", "")), + ) + if action == "summary": + return self.summary() + raise ValueError(f"unsupported multi-cache action: {action}") + + def summary(self) -> dict[str, Any]: + return { + "query": self.query.snapshot(), + "plan": self.plan.snapshot(), + "artifact": self.artifact.snapshot(), + } diff --git a/systems/quantum_db/data_plane/tn/page.py b/systems/quantum_db/data_plane/tn/page.py new file mode 100644 index 0000000..4bc4d9a --- /dev/null +++ b/systems/quantum_db/data_plane/tn/page.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class TNPageMetadata: + """TN Page 元数据。""" + + page_id: str + version: str = "v1" + layout: str = "mps" + physical_dims: tuple[int, ...] = () + bond_dims: tuple[int, ...] = () + encoding: str = "fft_topk" + graph_data_ref: str | None = None + graph_hardware_ref: str | None = None + mapping_ref: str | None = None + tags: tuple[str, ...] = () + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class TNPage: + """TN page 数据结构：只承载内容与元信息，不负责 IO。""" + + metadata: TNPageMetadata + tensors: list[Any] = field(default_factory=list) + stats: dict[str, float] = field(default_factory=dict) + artifacts: dict[str, Any] = field(default_factory=dict) + + @property + def page_id(self) -> str: + return self.metadata.page_id + + def validate(self) -> None: + if not self.metadata.page_id.strip(): + raise ValueError("page_id must not be empty") + if self.metadata.layout not in {"mps", "tt", "tree"}: + raise ValueError(f"unsupported layout: {self.metadata.layout}") + if self.metadata.physical_dims and any(dim <= 0 for dim in self.metadata.physical_dims): + raise ValueError("physical_dims must be positive") + if self.metadata.bond_dims and any(dim <= 0 for dim in self.metadata.bond_dims): + raise ValueError("bond_dims must be positive") + if self.metadata.bond_dims and self.metadata.layout == "mps": + expected = max(len(self.metadata.physical_dims) - 1, 0) + if len(self.metadata.bond_dims) not in {0, expected}: + raise ValueError("for MPS, bond_dims length should be len(physical_dims)-1") + + def update_stats(self, **metrics: float) -> None: + for key, value in metrics.items(): + self.stats[key] = float(value) + + def summary(self) -> dict[str, Any]: + return { + "page_id": self.metadata.page_id, + "version": self.metadata.version, + "layout": self.metadata.layout, + "num_tensors": len(self.tensors), + "physical_dims": self.metadata.physical_dims, + "bond_dims": self.metadata.bond_dims, + "encoding": self.metadata.encoding, + "stats": dict(self.stats), + "artifacts": tuple(self.artifacts.keys()), + } diff --git a/systems/quantum_db/data_plane/tn/page_manager.py b/systems/quantum_db/data_plane/tn/page_manager.py new file mode 100644 index 0000000..6b5b1ed --- /dev/null +++ b/systems/quantum_db/data_plane/tn/page_manager.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any + +from data_plane.tn.cache import TNMultiLayerCache +from data_plane.tn.page import TNPage + + +@dataclass +class PageVersionInfo: + version: int + updated_at_ms: int + + +class TNPageManager: + """管理 TN Page 的装载、写回与生命周期，不做张量计算。""" + + __slots__ = ("cache", "_storage", "_versions") + + def __init__(self, cache: TNMultiLayerCache | None = None) -> None: + self.cache = cache if cache is not None else TNMultiLayerCache() + self._storage: dict[str, TNPage] = {} + self._versions: dict[str, dict[str, int]] = {} + + def _record_version(self, page_id: str) -> PageVersionInfo: + now_ms = int(time.time() * 1000) + current = self._versions.get(page_id) + version = 1 if current is None else current["version"] + 1 + self._versions[page_id] = {"version": version, "updated_at_ms": now_ms} + return PageVersionInfo(version=version, updated_at_ms=now_ms) + + def create(self, page: TNPage) -> TNPage: + page.validate() + version_info = self._record_version(page.page_id) + self._storage[page.page_id] = page + self.cache.execute( + "artifact.put", + { + "artifact_type": "tn_page", + "artifact_id": page.page_id, + "plan_id": f"page_v{version_info.version}", + "payload": page, + }, + ) + return page + + def upsert(self, page: TNPage) -> TNPage: + return self.create(page) + + def load(self, page_id: str) -> TNPage: + cached = self.cache.execute("artifact.get", {"artifact_type": "tn_page", "artifact_id": page_id}) + if cached is not None and isinstance(getattr(cached, "payload", None), TNPage): + return cached.payload + + page = self._storage.get(page_id) + if page is None: + raise KeyError(f"unknown TN page: {page_id}") + self.cache.execute( + "artifact.put", + { + "artifact_type": "tn_page", + "artifact_id": page_id, + "plan_id": "page_reload", + "payload": page, + }, + ) + return page + + def write_back( + self, + page_id: str, + *, + stats: dict[str, float] | None = None, + tensors: list[Any] | None = None, + artifacts: dict[str, Any] | None = None, + ) -> TNPage: + page = self.load(page_id) + if stats: + page.stats.update({k: float(v) for k, v in stats.items()}) + if tensors is not None: + page.tensors = list(tensors) + if artifacts: + page.artifacts.update(artifacts) + + version_info = self._record_version(page_id) + self.cache.execute( + "artifact.put", + { + "artifact_type": "tn_page", + "artifact_id": page_id, + "plan_id": f"page_v{version_info.version}", + "payload": page, + }, + ) + return page + + def release(self, page_id: str) -> None: + # 释放 cache（通过 tombstone 覆盖），保留存储与版本信息 + if page_id not in self._storage: + return + self.cache.execute( + "artifact.put", + { + "artifact_type": "tn_page", + "artifact_id": page_id, + "plan_id": "page_released", + "payload": None, + }, + ) + + def delete(self, page_id: str, *, expected_version: int, delete_before_ms: int) -> None: + version_meta = self._versions.get(page_id) + if version_meta is None: + return + if version_meta["version"] != expected_version: + raise ValueError(f"version mismatch for {page_id}") + if version_meta["updated_at_ms"] > delete_before_ms: + raise ValueError(f"delete rejected by timestamp for {page_id}") + + self._storage.pop(page_id, None) + self._versions.pop(page_id, None) + self.cache.execute( + "artifact.put", + { + "artifact_type": "tn_page", + "artifact_id": page_id, + "plan_id": "page_deleted", + "payload": None, + }, + ) + + def version_of(self, page_id: str) -> PageVersionInfo: + version_meta = self._versions.get(page_id) + if version_meta is None: + raise KeyError(f"unknown TN page version: {page_id}") + return PageVersionInfo( + version=int(version_meta["version"]), + updated_at_ms=int(version_meta["updated_at_ms"]), + ) + + def list_pages(self) -> list[str]: + return sorted(self._storage.keys()) + + def describe(self, page_id: str) -> dict[str, Any]: + page = self.load(page_id) + version_info = self.version_of(page_id) + data = page.summary() + data["version"] = version_info.version + data["updated_at_ms"] = version_info.updated_at_ms + return data diff --git a/systems/quantum_db/data_plane/tn/scoring.py b/systems/quantum_db/data_plane/tn/scoring.py new file mode 100644 index 0000000..4942d8b --- /dev/null +++ b/systems/quantum_db/data_plane/tn/scoring.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ScoreWeights: + query_utility: float = 0.25 + approximation_error: float = 0.2 + reconstruction_cost: float = 0.15 + fidelity_loss: float = 0.2 + materialization_latency: float = 0.2 + + def normalized(self) -> "ScoreWeights": + total = ( + self.query_utility + + self.approximation_error + + self.reconstruction_cost + + self.fidelity_loss + + self.materialization_latency + ) + if total <= 0: + raise ValueError("score weight total must be positive") + return ScoreWeights( + query_utility=self.query_utility / total, + approximation_error=self.approximation_error / total, + reconstruction_cost=self.reconstruction_cost / total, + fidelity_loss=self.fidelity_loss / total, + materialization_latency=self.materialization_latency / total, + ) + + +@dataclass(frozen=True) +class MetricBounds: + low: float + high: float + + def normalize(self, value: float) -> float: + if self.high <= self.low: + return 0.0 + scaled = (value - self.low) / (self.high - self.low) + if scaled < 0.0: + return 0.0 + if scaled > 1.0: + return 1.0 + return scaled + + +DEFAULT_BOUNDS: dict[str, MetricBounds] = { + "query_utility": MetricBounds(0.0, 1.0), + "approximation_error": MetricBounds(0.0, 1.0), + "reconstruction_cost": MetricBounds(0.0, 1.0), + "fidelity_loss": MetricBounds(0.0, 1.0), + "materialization_latency": MetricBounds(0.0, 1.0), +} + + +def unified_score( + *, + query_utility: float, + approximation_error: float, + reconstruction_cost: float, + fidelity_loss: float, + materialization_latency: float, + weights: ScoreWeights | None = None, + bounds: dict[str, MetricBounds] | None = None, +) -> tuple[float, dict[str, float]]: + """统一多标准打分（utility 最大化，其余成本最小化）。""" + + normalized_weights = (weights or ScoreWeights()).normalized() + metric_bounds = DEFAULT_BOUNDS if bounds is None else bounds + + u = metric_bounds["query_utility"].normalize(query_utility) + e = metric_bounds["approximation_error"].normalize(approximation_error) + r = metric_bounds["reconstruction_cost"].normalize(reconstruction_cost) + f = metric_bounds["fidelity_loss"].normalize(fidelity_loss) + t = metric_bounds["materialization_latency"].normalize(materialization_latency) + + score = ( + normalized_weights.query_utility * u + - normalized_weights.approximation_error * e + - normalized_weights.reconstruction_cost * r + - normalized_weights.fidelity_loss * f + - normalized_weights.materialization_latency * t + ) + breakdown = { + "query_utility_norm": u, + "approximation_error_norm": e, + "reconstruction_cost_norm": r, + "fidelity_loss_norm": f, + "materialization_latency_norm": t, + } + return score, breakdown diff --git a/systems/quantum_db/data_plane/tn/workloads.py b/systems/quantum_db/data_plane/tn/workloads.py new file mode 100644 index 0000000..6a356a4 --- /dev/null +++ b/systems/quantum_db/data_plane/tn/workloads.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import math +import random +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Literal + +from data_plane.tn.compute import TNCompute +from data_plane.tn.page import TNPage +from data_plane.tn.page_manager import TNPageManager +from data_plane.tn.modes import TNMode, estimate_qubits_depth + + +@dataclass +class WorkloadConfig: + num_sites: int = 16 + phys_dim: int = 8 + chi_max: int = 16 + keep_ratio: float = 0.25 + alu_components: tuple[str, ...] = ( + "adder", + "multiplier", + "divider", + "fma", + "comparator", + ) + alu_mode: Literal["auto", "manual", "hybrid"] = "auto" + primary_alu: str = "none" + selection_objective: str = "fidelity_storage_depth" + hardware_profile: dict[str, Any] | None = None + evaluation_weights: dict[str, float] | None = None + tn_mode: Literal["mps_linear", "tree", "native_mapping"] = "mps_linear" + + @property + def candidate_alus(self) -> tuple[str, ...]: + """Backward-compatible alias for older experiment scripts/configs.""" + return self.alu_components + + +class TNWorkloadService: + """解耦后的 TN workload 功能集；语义层通过此类调用。""" + + __slots__ = ("compute", "page_manager", "config") + + def __init__(self, *, config: WorkloadConfig | None = None, seed: int | None = None) -> None: + self.compute = TNCompute(seed=seed) + self.page_manager = TNPageManager() + self.config = config or WorkloadConfig() + + def create_page(self, *, dataset: str, samples: list[list[float]]) -> TNPage: + selection = self._select_alu(samples) + page = self.compute.fit_mps_from_samples( + samples, + page_id=f"experiment:{dataset}", + num_sites=self.config.num_sites, + phys_dim=self.config.phys_dim, + chi_max=self.config.chi_max, + keep_ratio=self.config.keep_ratio, + ) + # three-mode support: mps linear / tree organization / native mapping + tn_mode = TNMode(self.config.tn_mode) + page.metadata.layout = "tree" if tn_mode is TNMode.TREE else "mps" + page.metadata.extra["tn_mode"] = tn_mode.value + if tn_mode is TNMode.NATIVE: + page.metadata.encoding = "native_mapping" + page.metadata.extra["native_mapping"] = {"enabled": True, "columns": len(samples[0]) if samples else 0} + qubits, depth = estimate_qubits_depth(tn_mode, self.config.num_sites, self.config.phys_dim, page.metadata.bond_dims) + page.stats["qubits_required"] = float(qubits) + page.stats["depth_estimate"] = float(depth) + resource_estimate = self._estimate_resources(page) + page.metadata.extra["alu_selection"] = selection + page.metadata.extra["resource_estimate"] = resource_estimate + page.artifacts["alu_selection_log"] = self._build_selection_log(dataset=dataset, selection=selection) + page.stats.update({f"resource_{k}": float(v) for k, v in resource_estimate.items()}) + self.page_manager.upsert(page) + self.page_manager.release(page.page_id) + return self.page_manager.load(page.page_id) + + def _select_alu(self, samples: list[list[float]]) -> dict[str, Any]: + allowed = ("wloc", "wbrick", "wtree", "wstar", "wsym") + candidates = [a for a in self.config.candidate_alus if a in allowed] + if not candidates: + candidates = ["wloc"] + + width = len(samples[0]) if samples and samples[0] else 0 + dependency = self.cross_column_dependency(samples) + prior_scores = { + "wloc": 1.0 if dependency >= 0.35 else 0.55, + "wbrick": 0.7, + "wtree": 0.95 if width >= 16 else 0.6, + "wstar": 0.85 if width <= 6 else 0.45, + "wsym": 0.9 if dependency <= 0.2 else 0.5, + } + + hardware = { + "supports_k_body": False, + "max_parallel_pairs": 2, + "ancilla_reuse": True, + "blockade_radius_class": "medium", + } + if self.config.hardware_profile: + hardware.update(self.config.hardware_profile) + + weights = {"fidelity": 0.4, "storage": 0.2, "depth": 0.2, "error": 0.2} + if self.config.evaluation_weights: + weights.update(self.config.evaluation_weights) + + scored: list[dict[str, Any]] = [] + for alu in candidates: + fidelity = prior_scores[alu] + storage = 1.0 - (0.1 if alu == "wtree" else 0.35 if alu == "wstar" else 0.2) + depth = 0.9 if alu in {"wloc", "wbrick"} else 0.6 + error = max(0.0, 1.0 - fidelity) + score = fidelity * weights["fidelity"] + storage * weights["storage"] + depth * weights["depth"] + (1 - error) * weights["error"] + scored.append({"alu": alu, "score": round(score, 6), "fidelity": fidelity, "storage": storage, "depth": depth, "error": error}) + + scored.sort(key=lambda x: x["score"], reverse=True) + selected = scored[0]["alu"] + if self.config.alu_mode == "manual": + selected = self.config.primary_alu + elif self.config.alu_mode == "hybrid" and self.config.primary_alu != "none": + selected = self.config.primary_alu if any(s["alu"] == self.config.primary_alu for s in scored[:2]) else selected + + reason_bits = [] + if dependency >= 0.35: + reason_bits.append("局域相关") + if width >= 16: + reason_bits.append("树形") + if dependency <= 0.2: + reason_bits.append("对称") + reason_bits.append("约束") + + return { + "mode": self.config.alu_mode, + "objective": self.config.selection_objective, + "selected_alu": selected, + "primary_alu": self.config.primary_alu, + "candidate_scores": scored, + "hardware_profile": hardware, + "selection_reason": "结构先验匹配：" + "/".join(reason_bits), + } + + def _build_selection_log(self, *, dataset: str, selection: dict[str, Any]) -> dict[str, Any]: + return { + "ts": time.time(), + "dataset": dataset, + "selected_alu": selection["selected_alu"], + "mode": selection["mode"], + "objective": selection["objective"], + "selection_reason": selection["selection_reason"], + "candidate_scores": selection["candidate_scores"], + } + + def _estimate_resources(self, page: TNPage) -> dict[str, int | float]: + num_tensors = len(page.tensors) + total_params = 0 + for t in page.tensors: + for left in t: + for phys in left: + total_params += len(phys) + estimated_depth = max(1, int(math.ceil(math.log2(max(2, num_tensors))))) + estimated_peak_pairs = min( + int((self.config.hardware_profile or {}).get("max_parallel_pairs", 2)), + max(1, num_tensors // 2), + ) + return { + "num_tensors": num_tensors, + "total_params": total_params, + "estimated_depth": estimated_depth, + "estimated_peak_parallel_pairs": estimated_peak_pairs, + } + + def distribution_sampling(self, page: TNPage, *, num_samples: int = 64) -> list[int]: + return self.compute.sample(page, num_samples=num_samples) + + def approximate_aggregate(self, samples: list[list[float]]) -> dict[str, float]: + mean_est = self.compute.approximate_aggregate(samples, agg="mean") + var_est = self.compute.approximate_aggregate(samples, agg="var") + mean_gt = sum(sum(r) for r in samples) / max(1, sum(len(r) for r in samples)) + var_gt = sum((v - mean_gt) ** 2 for r in samples for v in r) / max(1, sum(len(r) for r in samples)) + return {"estimated_mean": mean_est, "ground_truth_mean": mean_gt, "estimated_var": var_est, "ground_truth_var": var_gt} + + def likelihood_estimation(self, page: TNPage, sample: list[float]) -> float: + return self.compute.estimate_probability(page, sample) + + def multi_attribute_selectivity(self, samples: list[list[float]], predicate: Callable[[list[float]], bool]) -> dict[str, float]: + est = self.compute.estimate_selectivity(samples, predicate) + gt = sum(1 for row in samples if predicate(row)) / max(1, len(samples)) + return {"estimated": est, "ground_truth": gt} + + def sampling(self, page: TNPage, *, num_samples: int = 16) -> list[int]: + return self.compute.sample(page, num_samples=num_samples) + + def similarity(self, a: list[float], b: list[float]) -> float: + return self.compute.similarity(a, b) + + def compression_ratio(self, page: TNPage, samples: list[list[float]]) -> float: + raw_values = sum(len(s) for s in samples) + raw_bytes = raw_values * 8 + params = 0 + for t in page.tensors: + for left in t: + for phys in left: + params += len(phys) + return raw_bytes / max(params * 8, 1) + + def reconstruction_error_l1(self, samples: list[list[float]], generated: list[int]) -> float: + if not samples or not generated: + return 1.0 + bins = max(generated) + 1 + h_true = [0.0] * bins + h_gen = [0.0] * bins + col0 = [row[0] for row in samples] + lo, hi = min(col0), max(col0) + scale = (hi - lo) if hi > lo else 1.0 + for v in col0: + idx = min(bins - 1, max(0, int(((v - lo) / scale) * (bins - 1)))) + h_true[idx] += 1.0 + for g in generated: + h_gen[min(bins - 1, max(0, g))] += 1.0 + st = sum(h_true) or 1.0 + sg = sum(h_gen) or 1.0 + return sum(abs((h_true[i] / st) - (h_gen[i] / sg)) for i in range(bins)) + + def fidelity(self, samples: list[list[float]], generated: list[int]) -> float: + return self.marginal_fidelity(samples, generated) + + def cross_column_dependency(self, samples: list[list[float]]) -> float: + if not samples or len(samples[0]) < 2: + return 0.0 + cols = len(samples[0]) + matrix = [[row[c] for row in samples] for c in range(cols)] + score = 0.0 + pairs = 0 + for i in range(cols): + for j in range(i + 1, cols): + score += abs(self._pearson(matrix[i], matrix[j])) + pairs += 1 + return 0.0 if pairs == 0 else score / pairs + + def marginal_fidelity(self, samples: list[list[float]], generated: list[int]) -> float: + # 单列边缘分布 fidelity proxy + err = self.reconstruction_error_l1(samples, generated) + return max(0.0, 1.0 - min(1.0, err / 2.0)) + + def joint_fidelity(self, samples: list[list[float]], generated: list[int], *, phys_dim: int) -> float: + if not samples or len(samples[0]) < 2 or len(generated) < 2: + return 0.0 + # 用(首列, 次列)联合分布与采样相邻对联合分布比较 + joint_true = [[1e-9] * phys_dim for _ in range(phys_dim)] + c0 = [r[0] for r in samples] + c1 = [r[1] for r in samples] + lo0, hi0 = min(c0), max(c0) + lo1, hi1 = min(c1), max(c1) + s0 = (hi0 - lo0) if hi0 > lo0 else 1.0 + s1 = (hi1 - lo1) if hi1 > lo1 else 1.0 + for a, b in zip(c0, c1): + i = min(phys_dim - 1, max(0, int(((a - lo0) / s0) * (phys_dim - 1)))) + j = min(phys_dim - 1, max(0, int(((b - lo1) / s1) * (phys_dim - 1)))) + joint_true[i][j] += 1.0 + + joint_gen = [[1e-9] * phys_dim for _ in range(phys_dim)] + for k in range(0, len(generated) - 1, 2): + i = min(phys_dim - 1, max(0, generated[k])) + j = min(phys_dim - 1, max(0, generated[k + 1])) + joint_gen[i][j] += 1.0 + + st = sum(sum(r) for r in joint_true) + sg = sum(sum(r) for r in joint_gen) + l1 = 0.0 + for i in range(phys_dim): + for j in range(phys_dim): + l1 += abs(joint_true[i][j] / st - joint_gen[i][j] / sg) + return max(0.0, 1.0 - min(1.0, l1 / 2.0)) + + def _pearson(self, x: list[float], y: list[float]) -> float: + n = min(len(x), len(y)) + if n == 0: + return 0.0 + mx = sum(x[:n]) / n + my = sum(y[:n]) / n + num = sum((x[i] - mx) * (y[i] - my) for i in range(n)) + denx = math.sqrt(sum((x[i] - mx) ** 2 for i in range(n))) + deny = math.sqrt(sum((y[i] - my) ** 2 for i in range(n))) + if denx == 0.0 or deny == 0.0: + return 0.0 + return num / (denx * deny) + + +def q_error(estimate: float, truth: float) -> float: + e = max(abs(estimate), 1e-12) + t = max(abs(truth), 1e-12) + return max(e / t, t / e) + + +def suggest_workload_config( + samples: list[list[float]], + *, + max_num_sites: int = 32, + max_phys_dim: int = 16, + max_chi: int = 32, +) -> WorkloadConfig: + if not samples: + return WorkloadConfig() + + width = min(len(r) for r in samples) + width = max(2, width) + n = len(samples) + + # num_sites: 与特征维同阶，但受上限限制 + num_sites = max(4, min(max_num_sites, width)) + + # phys_dim: 随样本规模缓慢增大，避免小数据过拟合 + phys_dim = max(4, min(max_phys_dim, int(2 + math.log2(max(4, n))))) + + # chi_max: 由有效复杂度决定（特征宽度 + 样本量） + complexity = math.log2(max(4, n)) + math.log2(max(4, width)) + chi_max = max(4, min(max_chi, int(2 * complexity))) + + keep_ratio = 0.25 if width >= 8 else 0.5 + return WorkloadConfig(num_sites=num_sites, phys_dim=phys_dim, chi_max=chi_max, keep_ratio=keep_ratio) diff --git a/systems/quantum_db/docs/REPOSITORY_LAYOUT.md b/systems/quantum_db/docs/REPOSITORY_LAYOUT.md new file mode 100644 index 0000000..6a32124 --- /dev/null +++ b/systems/quantum_db/docs/REPOSITORY_LAYOUT.md @@ -0,0 +1,17 @@ +# Repository layout and Git policy + +`storage/` is the maintained implementation. The historical `storage-main 2/` copy is not maintained and must not be +included in new commits. + +| Path | Purpose | Policy | +|---|---|---| +| `qdb/` | Sidecar, jobs, routing, candidate contract, large-scale benchmark | Commit | +| `pg_extension/` | PostgreSQL 16 PGXS extension and SQL migrations | Commit source and SQL; ignore binaries | +| `qute/` | Query policies and simulator backends | Commit | +| `data_plane/`, `control_plane/`, `interface/`, `protocol/`, `infra/` | Runtime integration layers | Commit | +| `scripts/`, `tests/`, `config/`, `docs/` | Reproducible commands, tests, examples, documentation | Commit | +| `outputs/`, `results/`, `artifacts/` | Generated state and reports | Ignore; curate summaries only | +| `train-test-data/` | Large research corpus | Keep outside normal Git or use Git LFS | + +Before a public push, remove already-tracked caches, generated binaries, duplicate copies, and large corpora from the +Git index while preserving local files. Review that index cleanup as a separate commit. diff --git a/systems/quantum_db/docs/architecture_plan.md b/systems/quantum_db/docs/architecture_plan.md new file mode 100644 index 0000000..cc2aa0f --- /dev/null +++ b/systems/quantum_db/docs/architecture_plan.md @@ -0,0 +1,27 @@ +# Qute first-round architecture + +The classical dataset is authoritative. A `QueryRequest` becomes a logical plan and four candidate policies: +C0 classical exact, C1 manual hybrid, C2 static quantum-if-feasible, and C3 adaptive minimum estimated cost +under quality/resource constraints. Every policy returns one `QueryResult` and one `ExecutionTrace`. + +C3 supports five explicit strategies: `quantum-first`, `classical-first`, `min-estimated-latency`, +`min-cost-under-quality-constraint`, and `fallback-safe`. Every strategy applies feasibility guards; a quantum +preference never bypasses qubit, depth, shots, latency, or quality constraints. + +Quantum operators (`QEXPECT`, `QMEASURE`, `QSIMILARITY`, `QESTIMATE`) are submitted only through the +`QuantumBackend` interface. Ideal, noisy, and optional OriginQ implementations share the same task/result +types. Quantum estimates are validated against the quality contract; they are never labeled as exact SQL +results. Failed or rejected plans produce a structured `FallbackDecision` and execute the classical oracle. + +Artifacts are versioned by authoritative data source/version, encoding, circuit, observable, backend, noise +model, transpilation settings, and shots. A data-version change makes older artifacts for that source stale +without invalidating unrelated datasets. Benchmark runs persist configuration, software +and backend metadata, raw JSON traces, aggregate CSV, and an end-to-end timing report. + +Compatible tasks are grouped by state reference/probabilities, circuit skeleton, backend, and shots while +allowing different observables. Batch outputs record wait time, execution time, and amortized preparation and +compile time. `TNPageTaskAdapter` is the only supported bridge from a versioned TN-Page through `TNExporter` +into `QuantumTask`; mismatched data versions are rejected before backend execution. + +The PostgreSQL extension and sidecar remain optional ingress layers. They must translate into this core model; +they are not the source of SQL truth or benchmark semantics. diff --git a/systems/quantum_db/docs/completion_audit.md b/systems/quantum_db/docs/completion_audit.md new file mode 100644 index 0000000..3d33864 --- /dev/null +++ b/systems/quantum_db/docs/completion_audit.md @@ -0,0 +1,47 @@ +# Qute completion audit + +This audit maps the requested framework requirements to authoritative local evidence. “Verified” means the +behavior is covered by executable tests or generated benchmark artifacts; it does not imply hardware speedup. + +| Requirement | Status | Evidence | +|---|---|---| +| Classical authoritative source and exact oracle | Verified | `classical_executor.py`; C0 and cross-policy agreement tests. | +| Quantum/hybrid results never silently presented as exact SQL | Verified | `QuteEngine` result types, validation and fallback tests. | +| Quality/resource contract and feasibility gates | Verified | `contracts.py`, `feasibility.py`, depth/qubit/quality tests and sweeps. | +| Complete `T_e2e` decomposition | Verified | Timing trace test, raw JSON traces, CSV and Markdown breakdown. | +| Stable task/result/plan/trace/fallback/artifact models | Verified | `qute/api`; JSON serialization tests. | +| QEXPECT/QMEASURE/QSIMILARITY/QESTIMATE | Verified offline | Ideal-backend operator tests. | +| C0/C1/C2/C3 configurable execution | Verified | Policy tests and cold workload policy matrix. | +| Five explainable optimizer strategies | Verified | Strategy tests and benchmark strategy workload. | +| Ideal simulator without credentials | Verified | Offline unit and smoke benchmark. | +| Noisy simulator and injected degradation | Verified | Noisy feasibility test and 125-run noisy benchmark. | +| Optional OriginQ/pyQPanda adapter | Local fallback and SDK preflight verified | An async candidate job emits a `U3 + CZ` Grover-probe IR and successfully compiles/runs through the adapter's local fallback. The pyQPanda3 0.4.0 environment imports after installing `libidn2`, `openssl@3`, and `zstd`; SDK preflight discovers the configured Wukong topology. | +| Safe backend/depth/qubit/quality fallback | Verified | Failure-injection tests and structured fallback traces. | +| Versioned cache lifecycle and source-scoped invalidation | Verified | Artifact transition and automatic invalidation tests. | +| Compatible-task batching and amortization metrics | Verified | Batch scheduler test and `batch_metrics.json`. | +| TN-Page formal adapter and version binding | Verified offline | `TNPageTaskAdapter` and mismatch rejection test. | +| Native/TN ablation with guarded narrative | Verified offline | Metrics, plot, and `tn_ablation_assessment.json`. | +| Cold/warm/failure workloads | Verified | Benchmark runner and raw traces. | +| Reproducible config, seed, versions, backend/noise metadata | Verified | Benchmark `config.json` and `dataset_metadata.json`. | +| Raw traces, aggregate metrics, calibration and reports | Verified | JSONL, CSV, summary, calibration, report and eight SVG plots. | +| Structured execution logging | Verified | `StructuredLogger` and benchmark `execution.log.jsonl`. | +| PostgreSQL PGXS build/load regression | Verified on PostgreSQL 16.14 | Official source SHA-256 verified; isolated build succeeded; `QuantumDBScan` executed exact filters/aggregates; PGXS `installcheck` passed. Deployment requires `shared_preload_libraries`. | +| PostgreSQL extension upgrade path | Verified on PostgreSQL 16.14 | Temporary cluster created `0.1.0`, upgraded to `0.1.1`, and completed candidate, fallback, and job-audit checks. | +| PostgreSQL read-only and sidecar SQL APIs | Verified locally | All four write operations rejected; explain/submit/status/result completed over Unix socket with unwrapped results. Unqualified SQL relation names resolve a registered `schema.table` snapshot only when the sidecar has exactly one matching suffix; ambiguous names remain fallback-only. | +| Durable cancellation semantics | Verified locally | SQLite conditional state transitions ensure a late provider failure or completion cannot replace `cancelled`; unit tests cover both service and store races. | +| Candidate RID pushdown with exact PostgreSQL recheck | Verified on PostgreSQL 16.14 | Candidate filtering requires a complete classical-snapshot *superset*, matching snapshot/checksum, recheck, and the `bool/int2/int4/int8` equality scope. A 600-row paged upload produced two candidates and `{1,3}`; checksum mismatch used native fallback and returned `{1,3}`. Text/range/type-sensitive predicates remain advisory-only. | +| Routed TN/Grover candidate probes | Verified locally; advisory-only | Async routes build a real `TNCompute` MPS page and/or execute an ideal-simulator Grover-style `QMEASURE` probe. `run_qpanda_probe=true` uses the pyQPanda3-compatible IR adapter in local fallback mode. Tests assert that probes cannot mutate candidate keys; no Wukong credit is used by default. | +| Integrated hybrid TN + Qute validation | Verified locally and through PostgreSQL 16 | `/*+ QDB_HYBRID */` forces a bounded TN-page and ideal Grover telemetry on the same immutable snapshot. Optional `run_qute_validation=true` runs QuteEngine C0 over the parsed predicates and records its exact count separately. The temporary PostgreSQL 16 smoke created a registered eight-row table, submitted a hybrid SQL job, reported all of `tn_page`, `ideal_qmeasure`, `qute_engine`, and `postgres_extension` as `used`, and `QuantumDBScan` rechecked the sole candidate to exact result `{0}`. | +| Wukong real-device preflight/execution | Verified for the bounded hardware probe | An explicitly authorized `WK_C180` preflight succeeded. A 256-shot, one-iteration `grover-oracle-diffuser-v1` run over a registered SQLite snapshot returned `JobStatus.FINISHED`, binary counts summing to 256, and 88 observations of the predeclared `0x0` target (34.4%, 2.75× the 12.5% uniform baseline; Wilson 95% interval 28.8%–40.4%). A second 256-shot two-iteration run returned 77 target observations (30.1%, 2.41× uniform); its additional CZ depth did not improve the observed rate. The new guarded PostgreSQL hybrid script is ready for one full-stack submission but has not been run automatically. | + +## Verification commands + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v +python3 -m qute.benchmark.runner --config qute/benchmark/configs/smoke.json --output /tmp/qute-smoke +python3 -m qute.benchmark.runner --config qute/benchmark/configs/noisy.json --output /tmp/qute-noisy +``` + +Local completion is therefore proven for the backend-neutral Qute framework and PostgreSQL 16 integration. +Wukong Grover-amplification acceptance still requires a valid credential and an explicitly authorized hardware run +whose marked-state counts can be independently checked; it is not inferred from local simulation or a finished job status. diff --git a/systems/quantum_db/docs/current_state.md b/systems/quantum_db/docs/current_state.md new file mode 100644 index 0000000..9060476 --- /dev/null +++ b/systems/quantum_db/docs/current_state.md @@ -0,0 +1,45 @@ +# Qute current-state audit + +## Existing assets + +- `Qute-main` contains a real-device low-selectivity Grover experiment, bit-order probing, `U3 + CZ` + circuit construction, QCloud polling, CSV datasets, and merged JSON reporting. It is an experiment pipeline, + not a general query engine. +- `storage/data_plane/quantum` contains a pyQPanda3/QCloud adapter, circuit IR validation, local fallback, + and Wukong topology configuration. SDK availability is optional. +- `storage/data_plane/tn` contains TN page construction, contraction, caching, workload metrics, and export to + quantum IR. Existing experiment scripts already produce several hardware/resource reports. +- `storage/control_plane` contains a scheduler, bus, semantic parser, and basic IR builder. Several public + interfaces were placeholders; routing had no quality/resource contract. +- `storage/qdb` and `storage/pg_extension` provide a local JSON-RPC sidecar and an initial PostgreSQL + integration. PostgreSQL remains authoritative and registered datasets are read-only. + +## Current query and backend boundaries + +- Query inputs are currently split between free-form semantic requests, ad-hoc dictionaries, experiment CLI + arguments, and PostgreSQL SQL text. +- Quantum execution is represented by gate-operation dictionaries accepted by `QuantumExecutor`. +- Supported hardware integration is pyQPanda3 QCloud when installed and configured; otherwise the legacy + adapter can simulate counts locally. +- Existing state preparation is TN export or the specialized Grover builder. Neither provides a stable + operator-level `QuantumTask` contract. + +## Reusable versus replaceable code + +Reusable: quantum response validation, hardware profiles, TN workload service/exporter, scheduler primitives, +QCloud security checks, Qute datasets, and experiment result artifacts. + +To replace or isolate: TODO-only protocol interfaces, implicit local random fallback presented as a backend, +experiment-specific task dictionaries, duplicated `storage-main 2`, and metrics that omit planning, +validation, queue, or fallback time. + +## Missing before this delivery + +- Stable request/result/quality/resource/trace models. +- A backend-neutral ideal and noisy simulator. +- Configurable C0/C1/C2/C3 execution policies. +- Explicit feasibility and fallback decisions. +- Version-bound quantum artifact lifecycle. +- Reproducible cold, warm, and failure workloads with raw end-to-end traces. + +`storage-main 2` is retained as historical data but is not an implementation source. diff --git a/systems/quantum_db/docs/experiment_fields.md b/systems/quantum_db/docs/experiment_fields.md new file mode 100644 index 0000000..1ee010c --- /dev/null +++ b/systems/quantum_db/docs/experiment_fields.md @@ -0,0 +1,44 @@ +# Qute experiment field dictionary + +| Field | Unit/type | Meaning | +|---|---|---| +| `query_id` | string | Stable workload query identifier. | +| `dataset_size` | rows | Authoritative classical input cardinality. | +| `policy` | C0–C3 | Classical oracle, manual hybrid, static, or adaptive policy. | +| `optimization_strategy` | enum | C3 quantum-first, classical-first, latency, quality-constrained cost, or fallback-safe strategy. | +| `selected_path` | enum | Final classical, TN, quantum, or hybrid path. | +| `candidate_path_costs` | JSON/ms | Explainable predicted costs considered by the dispatcher. | +| `plan_ms` | ms | Logical plan, resource estimation, cache lookup, and dispatch. | +| `prepare_ms` | ms | Encoding/state or reusable artifact preparation. | +| `compile_ms` | ms | Circuit construction/transpilation; amortized on cache reuse. | +| `submit_ms` | ms | Backend submission overhead. | +| `queue_ms` | ms | Backend queue delay. | +| `execute_ms` | ms | Classical, simulator, or device execution. | +| `measure_ms` | ms | Shot acquisition and measurement decoding. | +| `postprocess_ms` | ms | Classical reconstruction/aggregation. | +| `validation_ms` | ms | Quality-contract and oracle comparison. | +| `fallback_ms` | ms | Extra cost after rejecting or failing the selected path. | +| `e2e_ms` | ms | Sum of all timing components above. | +| `error_or_confidence` | ratio | Relative observed/estimated error; trace also stores confidence interval and level. | +| `shots` | count | Measurements requested from the quantum backend. | +| `logical_qubits` | count | Logical state/circuit qubits. | +| `physical_qubits` | count | Estimated backend qubits after overhead. | +| `depth` | layers | Estimated circuit depth. | +| `two_qubit_gate_count` | count | Estimated entangling gate count. | +| `cache_hit` | boolean | Whether the exact versioned artifact key was reused. | +| `fallback_triggered` | boolean | Whether the intended quantum/hybrid path was replaced by the oracle. | +| `fallback_reason` | string | Structured, human-readable rejection/failure reason. | +| `fallback_rate` | ratio | Per-run fallback indicator; aggregate reports average it over a workload. | +| `cache_reuse_rate` | ratio | Per-run cache-hit indicator; aggregate reports average it over a workload. | +| `estimated_fidelity` | ratio | Backend/encoding fidelity estimate when available. | +| `reconstruction_error` | ratio | Relative L1 error introduced by the encoding artifact. | +| `query_error` | ratio | Observed query-result error against the C0 oracle. | + +Raw traces additionally retain quality/resource contracts, backend and noise metadata, artifact/version keys, +structured errors, final result type, and exactness. Reported speed claims must use `e2e_ms`; individual timing +components are diagnostic only. + +`batch_metrics.json` records `batch_size`, `batch_wait_time_ms`, `batch_execution_time_ms`, +`amortized_compile_time_ms`, and `amortized_prepare_time_ms`. `summary.json` reports repeated-run means, +standard deviations, and 95% confidence half-widths. `calibration.json` preserves estimated versus observed +end-to-end latency for subsequent cost-model calibration. diff --git a/systems/quantum_db/docs/first_round_status.md b/systems/quantum_db/docs/first_round_status.md new file mode 100644 index 0000000..2531057 --- /dev/null +++ b/systems/quantum_db/docs/first_round_status.md @@ -0,0 +1,50 @@ +# First-round delivery status + +## 1. Completed + +- Repository audit and architecture freeze. +- Stable request, task, result, quality, resource, plan, trace, fallback, timing, and artifact models. +- Ideal, noisy, and optional OriginQ backends behind one interface. +- C0 classical oracle, C1 manual hybrid, C2 static feasibility policy, and C3 adaptive cost policy. +- Exact classical, classical TN approximation, quantum/hybrid validation, and safe fallback paths. +- Versioned artifact cache with reuse, stale, invalidation, and rebuild transitions. +- Cold, warm reusable-state, degradation, quality/depth/shots sweeps, and native/TN encoding ablation. +- Raw JSON traces, CSV metrics, aggregate JSON, Markdown timing report, and eight SVG experiment plots. +- Compatible-task batching metrics, repeated-run confidence statistics, calibration data, and a version-checked + TN-Page-to-QuantumTask adapter. + +## 2. Main files + +- `qute/api`: stable contracts and query models. +- `qute/quantum/backend`: backend abstraction and adapters. +- `qute/optimizer` and `qute/execution`: feasibility, costs, dispatch, execution, validation, and fallback. +- `qute/artifacts`: version-bound artifact lifecycle. +- `qute/benchmark`: workloads, runner, reporting, configuration, and plots. +- `tests/test_qute_framework.py`: contract, backend, policy, fallback, cache, and benchmark tests. + +## 3. New interfaces + +`QueryRequest`, `QueryResult`, `QuantumTask`, `QuantumResult`, `QualityContract`, `ResourceEstimate`, +`ExecutionPlan`, `ExecutionTrace`, `FallbackDecision`, `ArtifactMetadata`, `QuantumBackend`, and `QuteEngine`. + +## 4. Verification + +Run `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v` and +`python3 -m qute.benchmark.runner --config qute/benchmark/configs/smoke.json --output outputs/qute_smoke`. +The offline suite requires no cloud token or network access. + +## 5. Remaining risks + +- Simulator resource/time formulas are explicit research-model estimates, not calibrated hardware predictions. +- OriginQ execution still requires validation against the locally installed pyQPanda3 version and credentials. +- Classical TN approximation is a replaceable surrogate; existing TN-Page modules require a formal adapter into + `QuantumTask` before comparative claims are made. +- PostgreSQL PGXS compilation remains unverified on this machine because PostgreSQL server headers are absent. +- SVG plots are dependency-free baseline figures; publication styling and repeated-run confidence bands remain. + +See `completion_audit.md` for the requirement-by-requirement evidence and external verification boundary. + +## 6. Recommended next stage + +Fit calibrated cost coefficients from repeated simulator runs, run noisy-simulator sweeps, and validate +OriginQ/Wukong in an isolated credentialed environment. diff --git a/systems/quantum_db/docs/postgres_validation.md b/systems/quantum_db/docs/postgres_validation.md new file mode 100644 index 0000000..da763fd --- /dev/null +++ b/systems/quantum_db/docs/postgres_validation.md @@ -0,0 +1,63 @@ +# PostgreSQL 16.14 validation record + +Validation date: 2026-07-08. PostgreSQL 16.14 source was downloaded from the official distribution site and +verified using its published SHA-256 file. It was configured and installed only under `/tmp/pgsql16` with ICU, +readline, and zlib disabled. + +Verified outcomes: + +- PGXS C compilation completed without warnings after final changes. +- `shared_preload_libraries=quantum_db` installed the planner hook in a fresh backend session. +- `EXPLAIN (COSTS OFF)` selected `Custom Scan (QuantumDBScan)` for a registered table. +- Equality filtering and `COUNT(*)` returned the classical PostgreSQL result. +- INSERT, UPDATE, DELETE, and TRUNCATE were all rejected for a registered dataset; data remained unchanged. +- `make installcheck` passed the `quantum_db` pg_regress test. +- With the Python sidecar running, `qdb_explain`, `qdb_submit`, `qdb_status`, and `qdb_result` succeeded across + the Unix socket. SQL wrappers returned the unwrapped business result and preserved `requires_recheck=true`. + +The temporary PostgreSQL server and sidecar were stopped after verification. No system PostgreSQL installation +or cloud credential was modified. + +## Candidate-RID end-to-end validation + +On 2026-07-14, PostgreSQL 16.14 was rebuilt in `/tmp`, the extension was compiled with that isolated PGXS toolchain, +and a temporary preloaded cluster was used with a local `0600` Unix-socket sidecar. Registering a 600-row `orders` +table synchronized its immutable snapshot through a two-pass streaming scan and three 256-row-or-smaller pages, marked it `artifact_status=ready`, +and recorded a versioned `quantum_index` artifact checksum. `EXPLAIN ANALYZE` for the supported `int4` equality +filter reported: + +```text +Custom Scan (QuantumDBScan) + Quantum DB candidate source: sidecar snapshot + Quantum DB candidate keys: 2 +``` + +PostgreSQL then returned `{1,3}` after original predicate recheck. After an intentional checksum mismatch, the same +plan reported `Quantum DB candidate source: PostgreSQL fallback`, removed 598 rows through native filtering, and +still returned `{1,3}`. This verifies that an unavailable or stale sidecar cannot produce an incorrect SQL result. + +The end-to-end smoke also replaces the sidecar dataset record with the same rows but a different artifact checksum. +The extension rejects that candidate contract, reports `Quantum DB candidate source: PostgreSQL fallback`, and +returns `{1,3}`. Candidate filtering is therefore conditioned on complete classical-snapshot-superset provenance, +matching snapshot version/checksum, `requires_recheck=true`, and the explicitly supported built-in bool/integer +equality semantic scope. Text/collated, range, numeric, temporal, and expression predicates are deliberately +advisory-only and fall back to PostgreSQL. + +Registration also rejected a table with a `NULL` key and a 1024-byte textual key. This is intentional: the Custom +Scan stores candidate key output in a bounded 1024-byte hash entry, so accepting unrepresentable keys could allow a +candidate set to omit a real tuple before PostgreSQL recheck. + +The smoke also submitted a sidecar job through `qdb_submit` and read it in a subsequent SQL command. The matching +`qdb.jobs` row contained `status=succeeded` and the normalized SQL request, confirming SQL-side job audit mirroring. + +The same temporary-cluster script created extension `0.1.0`, executed `ALTER EXTENSION quantum_db UPDATE TO +'0.1.1'`, verified the installed version, and then completed both the normal candidate and checksum-mismatch +fallback checks. This validates the local upgrade path. + +## pyQPanda3 preflight + +The published pyQPanda3 0.4.0 CPython 3.11 macOS arm64 wheel was installed into the isolated +`/tmp/qdb-pyqpanda-env` environment. Import stopped before SDK initialization because the wheel links to +`/opt/homebrew/opt/libidn2/lib/libidn2.0.dylib`, which is not present on this machine. No credential was found, +and no Wukong request was made. The application-level adapter converts an SDK import/runtime failure into its +configured local fallback; real-device acceptance remains an environment-gated test. diff --git a/systems/quantum_db/infra/clock.py b/systems/quantum_db/infra/clock.py new file mode 100644 index 0000000..cfa7cdb --- /dev/null +++ b/systems/quantum_db/infra/clock.py @@ -0,0 +1,4 @@ +# TODO: +# - 提供统一时间接口 +# - 便于测试与重放 +# - 避免直接依赖系统时间 \ No newline at end of file diff --git a/systems/quantum_db/infra/config.py b/systems/quantum_db/infra/config.py new file mode 100644 index 0000000..d32a994 --- /dev/null +++ b/systems/quantum_db/infra/config.py @@ -0,0 +1,4 @@ +# TODO: +# - 统一配置读取与访问 +# - 避免配置散落在各模块 +# - 支持后续环境切换 \ No newline at end of file diff --git a/systems/quantum_db/infra/ids.py b/systems/quantum_db/infra/ids.py new file mode 100644 index 0000000..e27c148 --- /dev/null +++ b/systems/quantum_db/infra/ids.py @@ -0,0 +1,4 @@ +# TODO: +# - 统一生成 task/page/job/event 等 ID +# - 保持 ID 规则集中管理 +# - 避免各模块各自生成 \ No newline at end of file diff --git a/systems/quantum_db/infra/logger.py b/systems/quantum_db/infra/logger.py new file mode 100644 index 0000000..50ae50d --- /dev/null +++ b/systems/quantum_db/infra/logger.py @@ -0,0 +1,4 @@ +# TODO: +# - 提供统一日志入口 +# - 支持结构化日志 +# - 避免业务代码直接拼接日志格式 \ No newline at end of file diff --git a/systems/quantum_db/interface/bus.py b/systems/quantum_db/interface/bus.py new file mode 100644 index 0000000..0989d24 --- /dev/null +++ b/systems/quantum_db/interface/bus.py @@ -0,0 +1,18 @@ +"""Transport-neutral contracts for the control/data bus.""" +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any, Protocol + +PayloadHandler = Callable[[Any, Mapping[str, Any] | None], Any] + + +class BusInterface(Protocol): + def start(self) -> None: ... + def stop(self) -> None: ... + def register(self, name: str, handler: PayloadHandler, *, lane: str = "command") -> None: ... + def unregister(self, name: str, *, lane: str = "command") -> None: ... + def send(self, name: str, payload: Any, *, lane: str = "command", metadata: Mapping[str, Any] | None = None) -> Any: ... + def subscribe(self, topic: str, handler: PayloadHandler, *, lane: str = "data") -> int: ... + def unsubscribe(self, topic: str, subscription_id: int, *, lane: str = "data") -> bool: ... + def publish(self, topic: str, payload: Any, *, lane: str = "data", metadata: Mapping[str, Any] | None = None) -> int: ... diff --git a/systems/quantum_db/interface/cdk_service.py b/systems/quantum_db/interface/cdk_service.py new file mode 100644 index 0000000..3494c02 --- /dev/null +++ b/systems/quantum_db/interface/cdk_service.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from data_plane.config_loader import load_runtime_config +from data_plane.quantum.backend import BackendConfig +from data_plane.quantum.executor import QuantumExecutor +from data_plane.tn.exproter import TNExporter +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig + + +@dataclass +class TNBuildRequest: + dataset: str + samples: list[list[float]] + tn_mode: str + shots: int | None = None + + +class CdkTNQuantumService: + """API-style facade for TN build -> cloud quantum service execution.""" + + def __init__(self, *, config_path: str | None = None) -> None: + self.runtime = load_runtime_config(config_path) + qcfg = dict(self.runtime.get("quantum", {})) + hwcfg = self._load_hardware_profile(qcfg.get("hardware_profile")) + raw_api_base = str(qcfg.get("api_base", "")).strip() + parsed_api_base = urlparse(raw_api_base) + api_base_is_url = bool(parsed_api_base.scheme and parsed_api_base.hostname) + api_token = str(qcfg.get("api_token", hwcfg.get("API_KEY", ""))).strip() + if raw_api_base and not api_base_is_url and not api_token: + api_token = raw_api_base + raw_api_base = "" + qcloud_backend = str(qcfg.get("qcloud_backend_name", "")).strip() + if not qcloud_backend: + chip_name = str(qcfg.get("chip_name", hwcfg.get("chip_name", ""))).lower() + qcloud_backend = "WK_C180" if "wukong" in chip_name else "full_amplitude" + backend = BackendConfig( + backend_name=str(qcfg.get("backend_name", "qpanda3-sim")), + api_base=raw_api_base if api_base_is_url else "", + api_token=api_token, + qcloud_backend_name=qcloud_backend, + enforce_https=bool(qcfg.get("enforce_https", True)), + allowed_hosts=list(qcfg.get("allowed_hosts", [])), + timeout_sec=int(qcfg.get("timeout_sec", 20)), + allow_local_fallback=bool(qcfg.get("allow_local_fallback", True)), + no_fallback_actions=tuple(qcfg.get("no_fallback_actions", ("compile", "run"))), + num_qubits=int(qcfg.get("num_qubits", hwcfg.get("num_qubits", 16))), + num_couplers=int(qcfg.get("num_couplers", hwcfg.get("num_couplers", 0))), + coupling_map=[tuple(edge) for edge in qcfg.get("coupling_map", hwcfg.get("coupling_map", []))], + basis_gates=list(qcfg.get("basis_gates", hwcfg.get("basis_gates", ["u3", "cz"]))), + chip_name=str(qcfg.get("chip_name", hwcfg.get("chip_name", "default"))), + calibration_epoch=str(qcfg.get("calibration_epoch", hwcfg.get("calibration_epoch", "default"))), + ) + self.quantum = QuantumExecutor(backend_config=backend) + self.exporter = TNExporter() + + def _load_hardware_profile(self, profile_path: Any) -> dict[str, Any]: + if not profile_path: + return {} + raw = Path(str(profile_path)).read_text(encoding="utf-8") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise TypeError("hardware profile must be a JSON object") + return payload + + def _workload_config(self, tn_mode: str) -> WorkloadConfig: + return WorkloadConfig( + num_sites=int(self.runtime["num_sites"]), + phys_dim=int(self.runtime["phys_dim"]), + chi_max=int(self.runtime["chi_max"]), + keep_ratio=float(self.runtime["keep_ratio"]), + tn_mode=tn_mode, + ) + + def _analyze_ir_cost(self, ir: dict[str, Any]) -> dict[str, Any]: + ops = ir.get("operations", []) + if not isinstance(ops, list): + ops = [] + + gate_counts: dict[str, int] = {} + touched_qubits: set[int] = set() + qubit_available_at: dict[int, int] = {} + total_depth = 0 + prepare_depth = 0 + prepare_ops = 0 + + for raw_op in ops: + if not isinstance(raw_op, dict): + continue + gate = str(raw_op.get("gate", "")).strip().lower() + if not gate: + continue + qubits = [int(q) for q in raw_op.get("qubits", [])] + gate_counts[gate] = gate_counts.get(gate, 0) + 1 + touched_qubits.update(qubits) + + start_layer = max((qubit_available_at.get(q, 0) for q in qubits), default=0) + 1 + for q in qubits: + qubit_available_at[q] = start_layer + total_depth = max(total_depth, start_layer) + + if gate != "measure": + prepare_ops += 1 + prepare_depth = max(prepare_depth, start_layer) + + two_qubit_gates = sum(gate_counts.get(g, 0) for g in ("cx", "cz", "swap")) + one_qubit_gates = sum(count for gate, count in gate_counts.items() if gate not in {"cx", "cz", "swap", "measure"}) + return { + "ir_num_qubits": (max(touched_qubits) + 1) if touched_qubits else 0, + "qubit_load_info": dict(ir.get("qubit_load_info", {})), + "ir_num_operations": sum(gate_counts.values()), + "gate_counts": gate_counts, + "one_qubit_gates": one_qubit_gates, + "two_qubit_gates": two_qubit_gates, + "measurement_ops": gate_counts.get("measure", 0), + "gate_depth": total_depth, + "prepare": { + "num_operations": prepare_ops, + "gate_counts": {k: v for k, v in gate_counts.items() if k != "measure"}, + "gate_depth": prepare_depth, + "qubits": (max(touched_qubits) + 1) if touched_qubits else 0, + }, + } + + def _estimate_hardware_cost(self, ir: dict[str, Any]) -> dict[str, Any]: + backend = self.quantum.adapter.config + coupling = [tuple(sorted((int(a), int(b)))) for a, b in backend.coupling_map] + graph: dict[int, set[int]] = {i: set() for i in range(backend.num_qubits)} + for a, b in coupling: + graph.setdefault(a, set()).add(b) + graph.setdefault(b, set()).add(a) + + ops = ir.get("operations", []) + if not isinstance(ops, list): + ops = [] + + data_qubits = int(ir.get("qubit_load_info", {}).get("data_loading_qubits", 0)) + if str(ir.get("tn_mode", "")).lower() == "native_mapping": + logical_to_physical = {q: q for q in range(min(data_qubits, backend.num_qubits))} + else: + logical_to_physical = self._build_topology_aware_layout(ir, graph, backend.num_qubits) + if not logical_to_physical: + logical_to_physical = {q: q for q in range(min(data_qubits, backend.num_qubits))} + physical_qubits_used: set[int] = set(logical_to_physical.values()) + couplers_used: set[tuple[int, int]] = set() + routed_ops: list[dict[str, Any]] = [] + route_failures: list[dict[str, Any]] = [] + original_two_qubit_ops = 0 + swap_count = 0 + + for op in ops: + if not isinstance(op, dict): + continue + gate = str(op.get("gate", "")).lower() + qubits = op.get("qubits", []) + if not isinstance(qubits, list): + qubits = [] + + if gate in {"cz", "cx", "swap"} and len(qubits) == 2: + original_two_qubit_ops += 1 + p0 = logical_to_physical.get(int(qubits[0]), int(qubits[0])) + p1 = logical_to_physical.get(int(qubits[1]), int(qubits[1])) + path = self._shortest_path(graph, p0, p1) + if not path: + route_failures.append({"gate": gate, "physical_qubits": [p0, p1]}) + continue + + physical_qubits_used.update(path) + for a, b in zip(path, path[1:]): + couplers_used.add(tuple(sorted((a, b)))) + + if len(path) > 2: + # Move one endpoint along the shortest path until adjacent. + for a, b in zip(path[:-2], path[1:-1]): + routed_ops.append({"gate": "swap", "qubits": [a, b]}) + swap_count += 1 + routed_ops.append({"gate": gate, "qubits": [path[-2], path[-1]]}) + else: + physical = [logical_to_physical.get(int(q), int(q)) for q in qubits if isinstance(q, int)] + physical_qubits_used.update(physical) + routed_ops.append({"gate": gate, "qubits": physical}) + + routed_depth = self._operation_depth(routed_ops) + swap_cz_equivalent = 3 * swap_count if "cz" in set(backend.basis_gates) else swap_count + return { + "backend": backend.backend_name, + "chip_name": backend.chip_name, + "available_qubits": backend.num_qubits, + "available_couplers": backend.num_couplers, + "basis_gates": list(backend.basis_gates), + "initial_layout": "topology_aware_roots", + "logical_to_physical_size": len(logical_to_physical), + "physical_qubits_used": len(physical_qubits_used), + "couplers_used": len(couplers_used), + "original_two_qubit_ops": original_two_qubit_ops, + "routing_swap_count": swap_count, + "routed_two_qubit_ops": original_two_qubit_ops + swap_count, + "routed_cz_equivalent_ops": original_two_qubit_ops + swap_cz_equivalent, + "routed_depth": routed_depth, + "route_failures": route_failures, + "cloud_compiler_used": False, + } + + def _build_topology_aware_layout( + self, + ir: dict[str, Any], + graph: dict[int, set[int]], + num_physical_qubits: int, + ) -> dict[int, int]: + load = ir.get("qubit_load_info", {}) + data_qubits = int(load.get("data_loading_qubits", 0)) + num_sites = int(load.get("num_sites", 0)) + qubits_per_site = max(1, int(load.get("qubits_per_site", 1))) + if data_qubits <= 0 or data_qubits > num_physical_qubits: + return {} + + ops = ir.get("operations", []) + twoq_edges: list[tuple[int, int]] = [] + if isinstance(ops, list): + for op in ops: + if not isinstance(op, dict): + continue + if str(op.get("gate", "")).lower() in {"cz", "cx", "swap"}: + qubits = op.get("qubits", []) + if isinstance(qubits, list) and len(qubits) == 2: + twoq_edges.append((int(qubits[0]), int(qubits[1]))) + + logical_roots = [i * qubits_per_site for i in range(num_sites)] + root_set = set(logical_roots) + physical_order = sorted(graph, key=lambda q: (-len(graph.get(q, ())), q)) + if not physical_order: + return {} + + layout: dict[int, int] = {} + used: set[int] = set() + if twoq_edges: + adjacency: dict[int, set[int]] = {} + for a, b in twoq_edges: + adjacency.setdefault(a, set()).add(b) + adjacency.setdefault(b, set()).add(a) + start_logical = max(adjacency, key=lambda q: len(adjacency[q])) + start_physical = physical_order[0] + layout[start_logical] = start_physical + used.add(start_physical) + queue: deque[int] = deque([start_logical]) + while queue: + logical = queue.popleft() + for nxt in sorted(adjacency.get(logical, ())): + if nxt in layout: + continue + candidate = self._nearest_unused_physical(graph, layout[logical], used) + if candidate is None: + return {} + layout[nxt] = candidate + used.add(candidate) + queue.append(nxt) + + for root in logical_roots: + if root in layout: + continue + anchor = next(iter(layout.values()), physical_order[0]) + candidate = self._nearest_unused_physical(graph, anchor, used) + if candidate is None: + return {} + layout[root] = candidate + used.add(candidate) + + for site in range(num_sites): + root_logical = site * qubits_per_site + root_physical = layout[root_logical] + for offset in range(qubits_per_site): + logical = root_logical + offset + if logical >= data_qubits or logical in layout: + continue + candidate = self._nearest_unused_physical(graph, root_physical, used) + if candidate is None: + return {} + layout[logical] = candidate + used.add(candidate) + + for logical in range(data_qubits): + if logical in layout: + continue + candidate = self._nearest_unused_physical(graph, physical_order[0], used) + if candidate is None: + return {} + layout[logical] = candidate + used.add(candidate) + return layout + + def _nearest_unused_physical(self, graph: dict[int, set[int]], start: int, used: set[int]) -> int | None: + if start not in used: + return start + seen = {start} + queue: deque[int] = deque([start]) + while queue: + node = queue.popleft() + for nxt in sorted(graph.get(node, ())): + if nxt in seen: + continue + if nxt not in used: + return nxt + seen.add(nxt) + queue.append(nxt) + return None + + def _shortest_path(self, graph: dict[int, set[int]], start: int, goal: int) -> list[int]: + if start == goal: + return [start] + seen = {start} + queue: deque[tuple[int, list[int]]] = deque([(start, [start])]) + while queue: + node, path = queue.popleft() + for nxt in graph.get(node, set()): + if nxt in seen: + continue + next_path = [*path, nxt] + if nxt == goal: + return next_path + seen.add(nxt) + queue.append((nxt, next_path)) + return [] + + def _operation_depth(self, ops: list[dict[str, Any]]) -> int: + qubit_available_at: dict[int, int] = {} + depth = 0 + for op in ops: + qubits = [int(q) for q in op.get("qubits", [])] + layer = max((qubit_available_at.get(q, 0) for q in qubits), default=0) + 1 + for q in qubits: + qubit_available_at[q] = layer + depth = max(depth, layer) + return depth + + def invoke(self, request: TNBuildRequest) -> dict[str, Any]: + workload = TNWorkloadService(config=self._workload_config(request.tn_mode)) + page = workload.create_page(dataset=request.dataset, samples=request.samples) + target_backend = str(self.runtime.get("quantum", {}).get("backend_name", "qpanda3-sim")) + packet = self.exporter.export(page, target_backend=target_backend, allowed_encodings=["fft_topk", "native_mapping"]) + ir = self.exporter.to_quantum_ir(packet) + ir_cost = self._analyze_ir_cost(ir) + hardware_cost = self._estimate_hardware_cost(ir) + preflight_resp = self.quantum.execute({"action": "preflight", "payload": {"ir": ir}}) + preflight = preflight_resp["result"] + compiled_resp = self.quantum.execute({"action": "compile", "payload": {"ir": ir}}) + compiled = compiled_resp["result"] + estimate_resp = self.quantum.execute({"action": "estimate", "payload": {"ir": ir}}) + estimate = estimate_resp["result"] + shots = int(request.shots or self.runtime.get("quantum", {}).get("shots", 512)) + run_resp = self.quantum.execute({"action": "run", "payload": {"compiled": compiled, "shots": shots}}) + run_result = run_resp["result"] + backend_config = self.quantum.adapter.config + cloud_execution_mode = str(run_result.metadata.get("execution_mode", run_resp.get("execution_mode", "unknown"))) + return { + "page_id": page.page_id, + "tn_mode": page.metadata.extra.get("tn_mode", request.tn_mode), + "qubits_required": int(page.stats.get("qubits_required", 0)), + "depth_estimate": int(page.stats.get("depth_estimate", estimate.get("depth_est", 0))), + "ir_cost": ir_cost, + "qubit_load_info": dict(ir.get("qubit_load_info", {})), + "prepare_qubits": int(ir_cost["prepare"]["qubits"]), + "prepare_gate_depth": int(ir_cost["prepare"]["gate_depth"]), + "prepare_gate_counts": dict(ir_cost["prepare"]["gate_counts"]), + "hardware_cost": hardware_cost, + "preflight": { + "ok": bool(preflight.ok), + "warnings": list(preflight.warnings), + "route_edges": list(preflight.route_edges), + }, + "load_read_fidelity": float( + workload.fidelity( + request.samples, + workload.sampling(page, num_samples=int(self.runtime.get("sample_num_for_fidelity", 64))), + ) + ), + "quantum_counts": run_result.counts, + "quantum_shots": run_result.shots, + "quantum_backend": run_result.backend_name, + "quantum_metadata": dict(run_result.metadata), + "cloud_api_enabled": bool(backend_config.api_base or (backend_config.api_token and cloud_execution_mode == "qcloud_sdk")), + "cloud_execution_mode": cloud_execution_mode, + } diff --git a/systems/quantum_db/interface/monitor.py b/systems/quantum_db/interface/monitor.py new file mode 100644 index 0000000..5d61cb0 --- /dev/null +++ b/systems/quantum_db/interface/monitor.py @@ -0,0 +1,11 @@ +"""Metrics and health boundary independent of a monitoring backend.""" +from __future__ import annotations + +from typing import Any, Protocol + + +class MonitorInterface(Protocol): + def report_metric(self, component: str, category: str, name: str, value: float, *, timestamp: float | None = None) -> None: ... + def report_status(self, component: str, state: str, *, detail: str = "", timestamp: float | None = None) -> None: ... + def snapshot(self) -> dict[str, Any]: ... + def feedback_for_scheduler(self) -> Any: ... diff --git a/systems/quantum_db/interface/quantum.py b/systems/quantum_db/interface/quantum.py new file mode 100644 index 0000000..d5feb05 --- /dev/null +++ b/systems/quantum_db/interface/quantum.py @@ -0,0 +1,11 @@ +"""Backend-neutral quantum execution contract.""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol + + +class QuantumExecutorInterface(Protocol): + def start(self) -> None: ... + def stop(self) -> None: ... + def execute(self, command: Mapping[str, Any], metadata: Mapping[str, Any] | None = None) -> Any: ... diff --git a/systems/quantum_db/interface/scheduler.py b/systems/quantum_db/interface/scheduler.py new file mode 100644 index 0000000..34c614c --- /dev/null +++ b/systems/quantum_db/interface/scheduler.py @@ -0,0 +1,13 @@ +"""Minimal scheduling interface; callers do not depend on queue/DAG details.""" +from __future__ import annotations + +from typing import Any, Protocol + + +class SchedulerInterface(Protocol): + def submit(self, task_id: str, *, priority: int = 0, dependencies: tuple[str, ...] | list[str] = (), + payload: Any = None, metadata: dict[str, Any] | None = None) -> Any: ... + def dispatch(self, limit: int = 1) -> list[Any]: ... + def finish(self, task_id: str, *, success: bool = True) -> tuple[str, ...]: ... + def cancel(self, task_id: str) -> bool: ... + def state_of(self, task_id: str) -> str: ... diff --git a/systems/quantum_db/interface/semantic.py b/systems/quantum_db/interface/semantic.py new file mode 100644 index 0000000..2730922 --- /dev/null +++ b/systems/quantum_db/interface/semantic.py @@ -0,0 +1,11 @@ +"""Semantic layer boundary used by SQL/intent callers.""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol + + +class SemanticInterface(Protocol): + def parse(self, source: str) -> Any: ... + def build_ir(self, request: Mapping[str, Any]) -> Any: ... + def explain(self, request: Mapping[str, Any]) -> Mapping[str, Any]: ... diff --git a/systems/quantum_db/interface/tn.py b/systems/quantum_db/interface/tn.py new file mode 100644 index 0000000..bf8edc5 --- /dev/null +++ b/systems/quantum_db/interface/tn.py @@ -0,0 +1,12 @@ +"""Tensor-network execution contract exposed to the control plane.""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol + + +class TNExecutorInterface(Protocol): + def start(self) -> None: ... + def stop(self) -> None: ... + def bind(self, target: str, module: Any) -> None: ... + def execute(self, command: Mapping[str, Any], metadata: Mapping[str, Any] | None = None) -> Any: ... diff --git a/systems/quantum_db/pg_extension/Makefile b/systems/quantum_db/pg_extension/Makefile new file mode 100644 index 0000000..4c58013 --- /dev/null +++ b/systems/quantum_db/pg_extension/Makefile @@ -0,0 +1,9 @@ +EXTENSION = quantum_db +MODULE_big = quantum_db +OBJS = quantum_db.o +DATA = sql/quantum_db--0.1.0.sql sql/quantum_db--0.1.1.sql sql/quantum_db--0.1.0--0.1.1.sql +REGRESS = quantum_db + +PG_CONFIG ?= pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/systems/quantum_db/pg_extension/README.md b/systems/quantum_db/pg_extension/README.md new file mode 100644 index 0000000..ddb9ae2 --- /dev/null +++ b/systems/quantum_db/pg_extension/README.md @@ -0,0 +1,36 @@ +# Quantum DB PostgreSQL extension + +Build against PostgreSQL 16 server headers: + +```bash +make PG_CONFIG=/path/to/pg_config +make install PG_CONFIG=/path/to/pg_config +``` + +Add the following to `postgresql.conf` and restart PostgreSQL before using the extension: + +```conf +shared_preload_libraries = 'quantum_db' +quantum_db.socket_path = '/run/quantum-db/qdb.sock' +quantum_db.timeout_ms = 30000 +``` + +`shared_preload_libraries` is required because planner hooks are process-local; `CREATE EXTENSION` alone loads +the module only in the session executing that command. After restart, run `CREATE EXTENSION quantum_db` in each +database where metadata and SQL functions are needed. + +The current extension version is `0.1.1`. Existing `0.1.0` installations must first install the matching shared +library and SQL files, restart PostgreSQL, then run: + +```sql +ALTER EXTENSION quantum_db UPDATE TO '0.1.1'; +``` + +The upgrade adds the artifact checksum gate and SQL job-audit mirroring. A backend still on the old table layout +falls back to native PostgreSQL scanning until the upgrade completes. + +Run regression tests against an existing temporary server with: + +```bash +PGHOST=/path/to/socket PGPORT=55432 make installcheck PG_CONFIG=/path/to/pg_config +``` diff --git a/systems/quantum_db/pg_extension/expected/quantum_db.out b/systems/quantum_db/pg_extension/expected/quantum_db.out new file mode 100644 index 0000000..892ad61 --- /dev/null +++ b/systems/quantum_db/pg_extension/expected/quantum_db.out @@ -0,0 +1,23 @@ +CREATE EXTENSION quantum_db; +CREATE TABLE qdb_test(id bigint PRIMARY KEY, value integer, category text); +INSERT INTO qdb_test VALUES (1, 10, 'a'), (2, 20, 'b'); +SELECT qdb_register_dataset('qdb_test', 'id', ARRAY['value','category']) IS NOT NULL AS registered; + registered +------------ + t +(1 row) + +SELECT count(*) FROM qdb_test WHERE value = 10; + count +------- + 1 +(1 row) + +SELECT qdb_unregister_dataset('qdb_test'); + qdb_unregister_dataset +------------------------ + t +(1 row) + +DROP TABLE qdb_test; +DROP EXTENSION quantum_db CASCADE; diff --git a/systems/quantum_db/pg_extension/quantum_db.c b/systems/quantum_db/pg_extension/quantum_db.c new file mode 100644 index 0000000..d6f242c --- /dev/null +++ b/systems/quantum_db/pg_extension/quantum_db.c @@ -0,0 +1,654 @@ +#include "postgres.h" + +#include +#include +#include +#include +#include + +#include "access/relscan.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/tableam.h" +#include "commands/explain.h" +#include "executor/executor.h" +#include "executor/nodeCustom.h" +#include "executor/spi.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "nodes/extensible.h" +#include "nodes/primnodes.h" +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "optimizer/planner.h" +#include "optimizer/restrictinfo.h" +#include "catalog/namespace.h" +#include "catalog/pg_type_d.h" +#include "access/table.h" +#include "utils/builtins.h" +#include "utils/guc.h" +#include "utils/hsearch.h" +#include "utils/json.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" + +PG_MODULE_MAGIC; + +static char *qdb_socket_path = NULL; +static bool qdb_enable = true; +static int qdb_timeout_ms = 30000; +static set_rel_pathlist_hook_type previous_set_rel_pathlist_hook = NULL; + +typedef struct QdbScanState +{ + CustomScanState css; + TableScanDesc scan; + HTAB *candidate_keys; + AttrNumber key_attnum; + bool candidate_filter_active; +} QdbScanState; + +#define QDB_MAX_KEY_TEXT 1024 + +typedef struct QdbCandidateKey +{ + char key[QDB_MAX_KEY_TEXT]; +} QdbCandidateKey; + +static Plan *qdb_plan_custom_path(PlannerInfo *, RelOptInfo *, CustomPath *, List *, List *, List *); +static Node *qdb_create_scan_state(CustomScan *); +static void qdb_begin(CustomScanState *, EState *, int); +static TupleTableSlot *qdb_exec(CustomScanState *); +static void qdb_end(CustomScanState *); +static void qdb_rescan(CustomScanState *); +static void qdb_explain_scan(CustomScanState *, List *, ExplainState *); +static TupleTableSlot *qdb_next(ScanState *); +static bool qdb_recheck(ScanState *, TupleTableSlot *); +static bool qdb_dataset_metadata(Oid, char **, char **, char **); +static char *qdb_candidate_params(CustomScanState *); +static bool qdb_load_candidate_keys(QdbScanState *, const char *); +static void qdb_try_load_candidates(QdbScanState *); +static char *qdb_call_sidecar(const char *, const char *); +static const char *qdb_scalar_type_name(Oid); + +static CustomPathMethods qdb_path_methods = { + .CustomName = "QuantumDBScan", + .PlanCustomPath = qdb_plan_custom_path, +}; + +static CustomScanMethods qdb_scan_methods = { + .CustomName = "QuantumDBScan", + .CreateCustomScanState = qdb_create_scan_state, +}; + +static CustomExecMethods qdb_exec_methods = { + .CustomName = "QuantumDBScan", + .BeginCustomScan = qdb_begin, + .ExecCustomScan = qdb_exec, + .EndCustomScan = qdb_end, + .ReScanCustomScan = qdb_rescan, + .ExplainCustomScan = qdb_explain_scan, +}; + +void _PG_init(void); +void _PG_fini(void); +PG_FUNCTION_INFO_V1(qdb_rpc); + +static bool +qdb_relation_registered(Oid relid) +{ + char *key_column = NULL; + char *snapshot_version = NULL; + char *artifact_checksum = NULL; + + return qdb_dataset_metadata(relid, &key_column, &snapshot_version, &artifact_checksum); +} + +static bool +qdb_dataset_metadata(Oid relid, char **key_column, char **snapshot_version, + char **artifact_checksum) +{ + Oid nsp = get_namespace_oid("qdb", true); + Oid table; + Relation rel; + TableScanDesc scan; + HeapTuple tuple; + bool found = false; + + if (!OidIsValid(nsp)) + return false; + table = get_relname_relid("datasets", nsp); + if (!OidIsValid(table)) + return false; + rel = table_open(table, AccessShareLock); + scan = table_beginscan_catalog(rel, 0, NULL); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + bool isnull; + Datum value = heap_getattr(tuple, 2, RelationGetDescr(rel), &isnull); + if (!isnull && DatumGetObjectId(value) == relid) + { + bool key_isnull; + bool snapshot_isnull; + bool checksum_isnull = true; + Datum key = heap_getattr(tuple, 4, RelationGetDescr(rel), &key_isnull); + Datum snapshot = heap_getattr(tuple, 7, RelationGetDescr(rel), &snapshot_isnull); + Datum checksum = (Datum) 0; + /* Existing databases may still have the 0.1.0 schema while the newer + * shared library is already installed. Treat them as fallback-only until + * ALTER EXTENSION UPDATE installs artifact_checksum. */ + if (RelationGetDescr(rel)->natts >= 9) + checksum = heap_getattr(tuple, 9, RelationGetDescr(rel), &checksum_isnull); + if (!key_isnull && key_column != NULL) + *key_column = pstrdup(NameStr(*DatumGetName(key))); + if (!snapshot_isnull && snapshot_version != NULL) + *snapshot_version = TextDatumGetCString(snapshot); + if (!checksum_isnull && artifact_checksum != NULL) + *artifact_checksum = TextDatumGetCString(checksum); + found = true; + break; + } + } + table_endscan(scan); + table_close(rel, AccessShareLock); + return found; +} + +static void +qdb_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) +{ + Path *child; + CustomPath *path; + ListCell *cell; + + if (previous_set_rel_pathlist_hook) + previous_set_rel_pathlist_hook(root, rel, rti, rte); + if (!qdb_enable || rte->rtekind != RTE_RELATION || rel->reloptkind != RELOPT_BASEREL || rel->pathlist == NIL) + return; + if (!qdb_relation_registered(rte->relid)) + return; + child = NULL; + foreach(cell, rel->pathlist) + { + Path *candidate = lfirst_node(Path, cell); + + if (IsA(candidate, CustomPath) && + ((CustomPath *) candidate)->methods == &qdb_path_methods) + return; /* already added for this relation */ + if (child == NULL || candidate->total_cost < child->total_cost) + child = candidate; + } + if (child == NULL) + return; + path = makeNode(CustomPath); + path->path.pathtype = T_CustomScan; + path->path.parent = rel; + path->path.pathtarget = rel->reltarget; + path->path.param_info = child->param_info; + path->path.parallel_aware = false; + /* This path itself is never partial/parallel (parallel_workers=0). Retaining the + * child's safety flag prevents add_path() from discarding it as less capable than + * an otherwise identical native path before cost comparison. */ + path->path.parallel_safe = child->parallel_safe; + path->path.parallel_workers = 0; + path->path.rows = child->rows; + path->path.startup_cost = child->startup_cost; + /* Keep the scan equivalent but marginally cheaper so add_path() does not + * discard it as dominated by the native scan. */ + path->path.total_cost = Max(child->startup_cost, child->total_cost - 0.0001); + path->path.pathkeys = child->pathkeys; + path->flags = 0; + path->custom_paths = NIL; + path->methods = &qdb_path_methods; + add_path(rel, &path->path); +} + +static Plan * +qdb_plan_custom_path(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, + List *tlist, List *clauses, List *custom_plans) +{ + CustomScan *scan = makeNode(CustomScan); + RangeTblEntry *rte = root->simple_rte_array[rel->relid]; + char *key_column = NULL; + char *snapshot_version = NULL; + char *artifact_checksum = NULL; + + scan->scan.plan.targetlist = tlist; + scan->scan.plan.qual = extract_actual_clauses(clauses, false); + scan->scan.scanrelid = rel->relid; + scan->custom_plans = NIL; + if (rte != NULL && qdb_dataset_metadata(rte->relid, &key_column, &snapshot_version, + &artifact_checksum) && artifact_checksum != NULL && *artifact_checksum != '\0') + scan->custom_private = list_make3(makeString(key_column), makeString(snapshot_version), + makeString(artifact_checksum)); + scan->methods = &qdb_scan_methods; + return &scan->scan.plan; +} + +static Node * +qdb_create_scan_state(CustomScan *scan) +{ + QdbScanState *state = palloc0(sizeof(QdbScanState)); + + NodeSetTag(&state->css, T_CustomScanState); + state->css.methods = &qdb_exec_methods; + state->css.slotOps = &TTSOpsBufferHeapTuple; + return (Node *) state; +} + +static void +qdb_begin(CustomScanState *node, EState *estate, int eflags) +{ + QdbScanState *state = (QdbScanState *) node; + + if ((eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + { + state->scan = table_beginscan(node->ss.ss_currentRelation, + estate->es_snapshot, 0, NULL); + qdb_try_load_candidates(state); + } +} + +static TupleTableSlot * +qdb_exec(CustomScanState *node) +{ + return ExecScan(&node->ss, qdb_next, qdb_recheck); +} + +static TupleTableSlot * +qdb_next(ScanState *scan_state) +{ + QdbScanState *state = (QdbScanState *) scan_state; + TupleTableSlot *slot = scan_state->ss_ScanTupleSlot; + + while (table_scan_getnextslot(state->scan, + scan_state->ps.state->es_direction, + slot)) + { + if (!state->candidate_filter_active) + return slot; + else + { + bool isnull; + Datum value = slot_getattr(slot, state->key_attnum, &isnull); + Oid output; + bool isvarlena; + char *key; + + if (!isnull) + { + getTypeOutputInfo(TupleDescAttr(slot->tts_tupleDescriptor, + state->key_attnum - 1)->atttypid, + &output, &isvarlena); + key = OidOutputFunctionCall(output, value); + if (strlen(key) < QDB_MAX_KEY_TEXT && + hash_search(state->candidate_keys, key, HASH_FIND, NULL) != NULL) + return slot; + } + ExecClearTuple(slot); + } + } + return NULL; +} + +static bool +qdb_recheck(ScanState *scan_state, TupleTableSlot *slot) +{ + return true; +} + +static void +qdb_end(CustomScanState *node) +{ + QdbScanState *state = (QdbScanState *) node; + + if (state->scan != NULL) + table_endscan(state->scan); +} + +static void +qdb_rescan(CustomScanState *node) +{ + QdbScanState *state = (QdbScanState *) node; + + if (state->scan != NULL) + table_rescan(state->scan, NULL); +} + +static void +qdb_explain_scan(CustomScanState *node, List *ancestors, ExplainState *es) +{ + QdbScanState *state = (QdbScanState *) node; + + ExplainPropertyText("Quantum DB candidate source", + state->candidate_filter_active ? "sidecar snapshot" : "PostgreSQL fallback", + es); + if (state->candidate_filter_active) + ExplainPropertyInteger("Quantum DB candidate keys", NULL, + hash_get_num_entries(state->candidate_keys), es); +} + +static void +qdb_append_predicate(StringInfo params, Expr *expr, Oid relid, Index scanrelid, bool *first) +{ + OpExpr *op; + Node *left; + Node *right; + Var *var; + Const *constant; + char *operator_name; + char *column_name; + Oid output; + bool isvarlena; + char *value; + const char *scalar_type; + + if (!IsA(expr, OpExpr)) + return; + op = (OpExpr *) expr; + if (list_length(op->args) != 2) + return; + left = linitial(op->args); + right = lsecond(op->args); + if (!IsA(left, Var) || !IsA(right, Const)) + return; + var = (Var *) left; + constant = (Const *) right; + if (constant->constisnull || var->varno != scanrelid || var->varlevelsup != 0 || var->varattno <= 0 || + var->vartype != constant->consttype) + return; + operator_name = get_opname(op->opno); + scalar_type = qdb_scalar_type_name(var->vartype); + /* This is deliberately narrower than the advisory parser. Only built-in + * scalar equality has an exact JSON/Python representation contract. */ + if (operator_name == NULL || strcmp(operator_name, "=") != 0 || scalar_type == NULL) + return; + column_name = get_attname(relid, var->varattno, false); + getTypeOutputInfo(constant->consttype, &output, &isvarlena); + value = OidOutputFunctionCall(output, constant->constvalue); + if (!*first) + appendStringInfoChar(params, ','); + appendStringInfoString(params, "{\"column\":"); + escape_json(params, column_name); + appendStringInfoString(params, ",\"operator\":"); + escape_json(params, operator_name); + appendStringInfoString(params, ",\"value\":"); + /* Text preserves PostgreSQL output semantics; the sidecar coerces to the registered value type. */ + escape_json(params, value); + appendStringInfoString(params, ",\"scalar_type\":"); + escape_json(params, scalar_type); + appendStringInfoChar(params, '}'); + *first = false; +} + +static const char * +qdb_scalar_type_name(Oid type) +{ + switch (type) + { + case BOOLOID: return "bool"; + case INT2OID: return "int2"; + case INT4OID: return "int4"; + case INT8OID: return "int8"; + default: return NULL; + } +} + +static char * +qdb_candidate_params(CustomScanState *node) +{ + CustomScan *plan = (CustomScan *) node->ss.ps.plan; + Relation relation = node->ss.ss_currentRelation; + Oid relid = RelationGetRelid(relation); + char *namespace_name; + char *relation_name; + char *snapshot_version; + char *artifact_checksum; + StringInfoData params; + ListCell *cell; + bool first = true; + + if (list_length(plan->custom_private) < 3) + return NULL; + snapshot_version = strVal(lsecond(plan->custom_private)); + artifact_checksum = strVal(lthird(plan->custom_private)); + namespace_name = get_namespace_name(RelationGetNamespace(relation)); + relation_name = RelationGetRelationName(relation); + if (namespace_name == NULL || relation_name == NULL) + return NULL; + initStringInfo(¶ms); + appendStringInfoString(¶ms, "{\"sql\":\"SELECT 1\",\"relation\":"); + escape_json(¶ms, psprintf("%s.%s", namespace_name, relation_name)); + appendStringInfoString(¶ms, ",\"snapshot_version\":"); + escape_json(¶ms, snapshot_version); + appendStringInfoString(¶ms, ",\"artifact_checksum\":"); + escape_json(¶ms, artifact_checksum); + appendStringInfoString(¶ms, ",\"candidate_semantics\":\"pg_builtin_scalar_equality_v1\""); + appendStringInfoString(¶ms, ",\"predicates\":["); + foreach(cell, plan->scan.plan.qual) + qdb_append_predicate(¶ms, (Expr *) lfirst(cell), relid, plan->scan.scanrelid, &first); + if (first) + return NULL; /* Unsupported predicates use the PostgreSQL scan unchanged. */ + appendStringInfoString(¶ms, "]}"); + return params.data; +} + +static bool +qdb_load_candidate_keys(QdbScanState *state, const char *params) +{ + char *response; + char *literal; + char *query; + int status; + uint64 row; + HASHCTL control; + CustomScan *plan = (CustomScan *) state->css.ss.ps.plan; + char *expected_snapshot; + char *expected_checksum; + + if (list_length(plan->custom_private) < 3) + return false; + expected_snapshot = strVal(lsecond(plan->custom_private)); + expected_checksum = strVal(lthird(plan->custom_private)); + response = qdb_call_sidecar("candidates", params); + status = SPI_connect(); + if (status != SPI_OK_CONNECT) + ereport(ERROR, (errmsg("could not initialize Quantum DB candidate parser"))); + literal = quote_literal_cstr(response); + query = psprintf( + "SELECT jsonb_typeof(%1$s::jsonb #> '{result,candidate_keys}'), " + "%1$s::jsonb #>> '{result,candidate_contract,complete}', " + "%1$s::jsonb #>> '{result,candidate_contract,provenance}', " + "%1$s::jsonb #>> '{result,candidate_contract,candidate_set}', " + "%1$s::jsonb #>> '{result,candidate_contract,semantic_scope}', " + "%1$s::jsonb #>> '{result,candidate_contract,snapshot_version}', " + "%1$s::jsonb #>> '{result,candidate_contract,artifact_checksum}', " + "%1$s::jsonb #>> '{result,requires_recheck}'", + literal); + status = SPI_execute(query, true, 0); + if (status != SPI_OK_SELECT || SPI_processed != 1) + { + SPI_finish(); + ereport(ERROR, (errmsg("could not parse Quantum DB candidate response"))); + } + { + char *kind = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + char *complete = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2); + char *provenance = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 3); + char *candidate_set = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 4); + char *semantic_scope = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 5); + char *snapshot = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 6); + char *checksum = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 7); + char *recheck = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 8); + if (kind == NULL || complete == NULL || provenance == NULL || candidate_set == NULL || semantic_scope == NULL || snapshot == NULL || + checksum == NULL || recheck == NULL || strcmp(kind, "array") != 0 || + strcmp(complete, "true") != 0 || strcmp(provenance, "classical_snapshot_filter") != 0 || + strcmp(candidate_set, "complete_superset") != 0 || strcmp(semantic_scope, "pg_builtin_scalar_equality_v1") != 0 || + strcmp(snapshot, expected_snapshot) != 0 || strcmp(checksum, expected_checksum) != 0 || + strcmp(recheck, "true") != 0) + { + SPI_finish(); + return false; + } + } + query = psprintf( + "SELECT jsonb_array_elements_text((%s::jsonb #> '{result,candidate_keys}')) AS candidate_key", + literal); + status = SPI_execute(query, true, 0); + if (status != SPI_OK_SELECT) + { + SPI_finish(); + ereport(ERROR, (errmsg("could not parse Quantum DB candidate response"))); + } + MemSet(&control, 0, sizeof(control)); + control.keysize = QDB_MAX_KEY_TEXT; + control.entrysize = sizeof(QdbCandidateKey); + state->candidate_keys = hash_create("Quantum DB candidate keys", 64, &control, + HASH_ELEM | HASH_STRINGS); + for (row = 0; row < SPI_processed; row++) + { + HeapTuple tuple = SPI_tuptable->vals[row]; + bool isnull; + Datum datum = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + char *key; + if (isnull) + continue; + key = TextDatumGetCString(datum); + if (!isnull && strlen(key) < QDB_MAX_KEY_TEXT) + (void) hash_search(state->candidate_keys, key, HASH_ENTER, NULL); + } + SPI_finish(); + state->candidate_filter_active = true; + return true; +} + +static void +qdb_try_load_candidates(QdbScanState *state) +{ + CustomScan *plan = (CustomScan *) state->css.ss.ps.plan; + char *key_column; + char *params; + + if (list_length(plan->custom_private) < 3) + return; + key_column = strVal(linitial(plan->custom_private)); + state->key_attnum = get_attnum(RelationGetRelid(state->css.ss.ss_currentRelation), key_column); + if (state->key_attnum == InvalidAttrNumber) + return; + params = qdb_candidate_params(&state->css); + if (params == NULL) + return; + PG_TRY(); + { + (void) qdb_load_candidate_keys(state, params); + } + PG_CATCH(); + { + FlushErrorState(); + state->candidate_keys = NULL; + state->candidate_filter_active = false; + } + PG_END_TRY(); +} + +static char * +qdb_call_sidecar(const char *method, const char *params) +{ + int fd; + struct sockaddr_un address; + StringInfoData request; + StringInfoData response; + char buffer[4096]; + ssize_t count; + size_t written = 0; + struct timeval timeout; + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not create Quantum DB socket: %m"))); + timeout.tv_sec = qdb_timeout_ms / 1000; + timeout.tv_usec = (qdb_timeout_ms % 1000) * 1000; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) < 0 || + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)) < 0) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not configure Quantum DB socket timeout: %m"))); + } + MemSet(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + if (strlen(qdb_socket_path) >= sizeof(address.sun_path)) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Quantum DB socket path is too long"))); + } + strlcpy(address.sun_path, qdb_socket_path, sizeof(address.sun_path)); + if (connect(fd, (struct sockaddr *) &address, sizeof(address)) < 0) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not connect to Quantum DB sidecar at %s: %m", qdb_socket_path))); + } + initStringInfo(&request); + appendStringInfo(&request, "{\"version\":\"1.0\",\"id\":\"postgres\",\"method\":"); + escape_json(&request, method); + appendStringInfoString(&request, ",\"params\":"); + appendStringInfoString(&request, params); + appendStringInfoString(&request, "}\n"); + while (written < request.len) + { + count = write(fd, request.data + written, request.len - written); + if (count < 0 && errno == EINTR) + continue; + if (count <= 0) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not write to Quantum DB sidecar: %m"))); + } + written += count; + } + initStringInfo(&response); + while ((count = read(fd, buffer, sizeof(buffer))) > 0) + { + appendBinaryStringInfo(&response, buffer, count); + if (response.len > 1048576) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("Quantum DB response exceeds 1 MiB"))); + } + } + if (count < 0) + { + close(fd); + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not read from Quantum DB sidecar: %m"))); + } + close(fd); + return response.data; +} + +Datum +qdb_rpc(PG_FUNCTION_ARGS) +{ + text *method = PG_GETARG_TEXT_PP(0); + text *params = PG_GETARG_TEXT_PP(1); + char *response = qdb_call_sidecar(text_to_cstring(method), text_to_cstring(params)); + PG_RETURN_TEXT_P(cstring_to_text(response)); +} + +void +_PG_init(void) +{ + DefineCustomStringVariable("quantum_db.socket_path", "Quantum DB sidecar Unix socket", NULL, + &qdb_socket_path, "/tmp/qdb.sock", PGC_SUSET, 0, NULL, NULL, NULL); + DefineCustomBoolVariable("quantum_db.enable", "Enable Quantum DB custom scans", NULL, + &qdb_enable, true, PGC_USERSET, 0, NULL, NULL, NULL); + DefineCustomIntVariable("quantum_db.timeout_ms", "Quantum DB sidecar socket timeout in milliseconds", NULL, + &qdb_timeout_ms, 30000, 1, 3600000, PGC_USERSET, 0, NULL, NULL, NULL); + RegisterCustomScanMethods(&qdb_scan_methods); + previous_set_rel_pathlist_hook = set_rel_pathlist_hook; + set_rel_pathlist_hook = qdb_set_rel_pathlist; +} + +void +_PG_fini(void) +{ + set_rel_pathlist_hook = previous_set_rel_pathlist_hook; +} diff --git a/systems/quantum_db/pg_extension/quantum_db.conf.sample b/systems/quantum_db/pg_extension/quantum_db.conf.sample new file mode 100644 index 0000000..fab97a3 --- /dev/null +++ b/systems/quantum_db/pg_extension/quantum_db.conf.sample @@ -0,0 +1,8 @@ +# Required so every PostgreSQL backend installs the Quantum DB planner hook. +shared_preload_libraries = 'quantum_db' + +# Local sidecar endpoint. Keep the socket directory accessible only to the +# PostgreSQL service account and the Quantum DB sidecar account. +quantum_db.socket_path = '/tmp/qdb.sock' +quantum_db.timeout_ms = 30000 +quantum_db.enable = on diff --git a/systems/quantum_db/pg_extension/quantum_db.control b/systems/quantum_db/pg_extension/quantum_db.control new file mode 100644 index 0000000..dee1b24 --- /dev/null +++ b/systems/quantum_db/pg_extension/quantum_db.control @@ -0,0 +1,5 @@ +comment = 'PostgreSQL planner bridge for the Quantum DB sidecar' +default_version = '0.1.1' +module_pathname = '$libdir/quantum_db' +relocatable = false +requires = 'plpgsql' diff --git a/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0--0.1.1.sql b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0--0.1.1.sql new file mode 100644 index 0000000..3f9d1f7 --- /dev/null +++ b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0--0.1.1.sql @@ -0,0 +1,176 @@ +-- Safety and lifecycle upgrade. IF NOT EXISTS keeps development databases created +-- from late 0.1.0 snapshots upgradeable as well. +ALTER TABLE qdb.datasets ADD COLUMN IF NOT EXISTS artifact_checksum text NOT NULL DEFAULT ''; +ALTER TABLE qdb.datasets DROP CONSTRAINT IF EXISTS datasets_artifact_checksum_check; +ALTER TABLE qdb.datasets ADD CONSTRAINT datasets_artifact_checksum_check + CHECK (artifact_checksum = '' OR artifact_checksum ~ '^[0-9a-f]{32}$'); + +CREATE OR REPLACE FUNCTION qdb._begin_dataset_upload(relation text, snapshot_version text, key_column name, artifact_checksum text, expected_rows integer) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.begin', jsonb_build_object( + 'relation', relation, 'snapshot_version', snapshot_version, 'key_column', key_column, + 'artifact_checksum', artifact_checksum, 'expected_rows', expected_rows)) +$$; +CREATE OR REPLACE FUNCTION qdb._append_dataset_page(upload_id text, page integer, rows jsonb) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.append', jsonb_build_object('upload_id', upload_id, 'page', page, 'rows', rows)) +$$; +CREATE OR REPLACE FUNCTION qdb._commit_dataset_upload(upload_id text) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.commit', jsonb_build_object('upload_id', upload_id)) +$$; +CREATE OR REPLACE FUNCTION qdb._sync_dataset(relation text, snapshot_version text, key_column name, artifact_checksum text, rows jsonb) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE upload jsonb; upload_id text; page_rows jsonb; page_no integer := 0; +BEGIN + upload := qdb._begin_dataset_upload(relation, snapshot_version, key_column, artifact_checksum, jsonb_array_length(rows)); + upload_id := upload->>'upload_id'; + FOR page_rows IN + SELECT jsonb_agg(item ORDER BY ordinal) + FROM jsonb_array_elements(rows) WITH ORDINALITY AS page_items(item, ordinal) + GROUP BY ((ordinal - 1) / 256) ORDER BY ((ordinal - 1) / 256) + LOOP + PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); + page_no := page_no + 1; + END LOOP; + RETURN qdb._commit_dataset_upload(upload_id); +END $$; +REVOKE ALL ON FUNCTION qdb._begin_dataset_upload(text, text, name, text, integer), + qdb._append_dataset_page(text, integer, jsonb), qdb._commit_dataset_upload(text), + qdb._sync_dataset(text, text, name, text, jsonb) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION qdb_register_dataset(table_name regclass, key_column name, indexed_columns name[], options jsonb DEFAULT '{}'::jsonb) +RETURNS uuid LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE id uuid; column_name name; trigger_base text; snapshot text; values_expression text; + relation_text text; snapshot_checksum text; row_payload jsonb; upload jsonb; upload_id text; + page_rows jsonb := '[]'::jsonb; row_count integer := 0; page_count integer := 0; page_no integer := 0; + unsafe_key_count bigint; +BEGIN + IF array_length(indexed_columns, 1) IS NULL THEN RAISE EXCEPTION 'indexed_columns must not be empty'; END IF; + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=key_column AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'key column % does not exist on %', key_column, table_name; + END IF; + FOREACH column_name IN ARRAY indexed_columns LOOP + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=column_name AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'indexed column % does not exist on %', column_name, table_name; + END IF; + END LOOP; + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + EXECUTE format('SELECT count(*) FROM %s WHERE %I IS NULL OR length(%I::text) >= 1024', + table_name, key_column, key_column) INTO unsafe_key_count; + IF unsafe_key_count > 0 THEN + RAISE EXCEPTION 'key column % contains % NULL or oversized values unsupported by Quantum DB candidate scan', + key_column, unsafe_key_count USING ERRCODE = '22023'; + END IF; + snapshot := txid_current()::text; + SELECT string_agg(quote_literal(indexed_column::text) || ', to_jsonb(' || quote_ident(indexed_column::text) || ')', ', ') + INTO values_expression FROM unnest(indexed_columns) AS u(indexed_column); + relation_text := table_name::text; + snapshot_checksum := md5(''); + FOR row_payload IN EXECUTE format( + 'SELECT jsonb_build_object(''key'', to_jsonb(%1$I), ''values'', jsonb_build_object(%2$s)) FROM %3$s ORDER BY %1$I', + key_column, values_expression, relation_text) + LOOP + snapshot_checksum := md5(snapshot_checksum || row_payload::text); + row_count := row_count + 1; + END LOOP; + INSERT INTO qdb.datasets(relation_oid,relation_name,key_column,indexed_columns,options,snapshot_version,artifact_checksum) + VALUES(table_name::oid,table_name,key_column,indexed_columns,options,snapshot,snapshot_checksum) RETURNING dataset_id INTO id; + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s FOR EACH ROW EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_row', table_name); + EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %s FOR EACH STATEMENT EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_truncate', table_name); + BEGIN + upload := qdb._begin_dataset_upload(relation_text, snapshot, key_column, snapshot_checksum, row_count); + upload_id := upload->>'upload_id'; + FOR row_payload IN EXECUTE format( + 'SELECT jsonb_build_object(''key'', to_jsonb(%1$I), ''values'', jsonb_build_object(%2$s)) FROM %3$s ORDER BY %1$I', + key_column, values_expression, relation_text) + LOOP + page_rows := page_rows || jsonb_build_array(row_payload); + page_count := page_count + 1; + IF page_count = 256 THEN + PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); + page_rows := '[]'::jsonb; page_count := 0; page_no := page_no + 1; + END IF; + END LOOP; + IF page_count > 0 THEN PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); END IF; + PERFORM qdb._commit_dataset_upload(upload_id); + INSERT INTO qdb.artifacts(dataset_id, kind, version, uri, checksum, metadata) + VALUES (id, 'quantum_index', snapshot, 'qdb://sidecar/' || id::text, + snapshot_checksum, jsonb_build_object('row_count', row_count, 'key_column', key_column, 'transfer', 'streamed_pages_v1')) + ON CONFLICT (dataset_id, kind, version) DO UPDATE + SET uri=EXCLUDED.uri, checksum=EXCLUDED.checksum, metadata=EXCLUDED.metadata; + UPDATE qdb.datasets SET artifact_status='ready' WHERE dataset_id=id; + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN + UPDATE qdb.datasets SET artifact_status='pending' WHERE dataset_id=id; + END; + RETURN id; +END $$; + +CREATE OR REPLACE FUNCTION qdb_unregister_dataset(table_name regclass) RETURNS boolean +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE trigger_base text; affected integer; relation_text text; +BEGIN + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_row', table_name); + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_truncate', table_name); + DELETE FROM qdb.datasets WHERE relation_oid=table_name::oid; + GET DIAGNOSTICS affected = ROW_COUNT; + relation_text := table_name::text; + BEGIN + PERFORM qdb._remove_sidecar_dataset(relation_text); + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN NULL; + END; + RETURN affected > 0; +END $$; + +CREATE OR REPLACE FUNCTION qdb_submit(sql text, options jsonb DEFAULT '{}'::jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; submitted_job_id uuid; submitted_trace_id uuid; +BEGIN + response := qdb._rpc_result('submit', jsonb_build_object('sql', sql) || options); + submitted_job_id := (response->>'job_id')::uuid; + submitted_trace_id := NULLIF(response->>'trace_id', '')::uuid; + INSERT INTO qdb.jobs(job_id, status, backend, trace_id, request) + VALUES (submitted_job_id, COALESCE(response->>'status', 'queued'), NULL, submitted_trace_id, + jsonb_build_object('sql', sql) || options) + ON CONFLICT (job_id) DO UPDATE SET status=EXCLUDED.status, trace_id=EXCLUDED.trace_id, + request=EXCLUDED.request, updated_at=clock_timestamp(); + RETURN response; +END $$; + +CREATE OR REPLACE FUNCTION qdb_status(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('status', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status=response->>'status', backend=COALESCE(response->>'backend', backend), + trace_id=COALESCE(NULLIF(response->>'trace_id', '')::uuid, trace_id), + error_code=NULLIF(response->>'error_code', ''), error_message=NULLIF(response->>'error_message', ''), + updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_status.job_id; + RETURN response; +END $$; + +CREATE OR REPLACE FUNCTION qdb_result(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('result', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status='succeeded', result=response->'result', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_result.job_id; + RETURN response; +END $$; + +CREATE OR REPLACE FUNCTION qdb_cancel(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('cancel', jsonb_build_object('job_id', job_id)); + IF COALESCE((response->>'cancelled')::boolean, false) THEN + UPDATE qdb.jobs SET status='cancelled', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_cancel.job_id; + END IF; + RETURN response; +END $$; diff --git a/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0.sql b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0.sql new file mode 100644 index 0000000..cb81d9f --- /dev/null +++ b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.0.sql @@ -0,0 +1,235 @@ +CREATE SCHEMA IF NOT EXISTS qdb; + +CREATE TABLE qdb.datasets ( + dataset_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + relation_oid oid NOT NULL UNIQUE, + relation_name regclass NOT NULL UNIQUE, + key_column name NOT NULL, + indexed_columns name[] NOT NULL, + options jsonb NOT NULL DEFAULT '{}'::jsonb, + snapshot_version text NOT NULL, + artifact_status text NOT NULL DEFAULT 'pending' CHECK (artifact_status IN ('pending','building','ready','failed')), + artifact_checksum text NOT NULL DEFAULT '' CHECK (artifact_checksum = '' OR artifact_checksum ~ '^[0-9a-f]{32}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE qdb.artifacts ( + artifact_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + dataset_id uuid NOT NULL REFERENCES qdb.datasets(dataset_id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('tn_page','quantum_index','circuit','bit_order')), + version text NOT NULL, + uri text NOT NULL, + checksum text NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(dataset_id, kind, version) +); + +CREATE TABLE qdb.jobs ( + job_id uuid PRIMARY KEY, + status text NOT NULL CHECK (status IN ('queued','running','succeeded','failed','cancelled','timed_out')), + backend text, + trace_id uuid, + request jsonb NOT NULL, + result jsonb, + error_code text, + error_message text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON qdb.datasets, qdb.artifacts, qdb.jobs FROM PUBLIC; + +CREATE FUNCTION qdb._rpc_text(method text, params text) RETURNS text +AS 'MODULE_PATHNAME', 'qdb_rpc' LANGUAGE C STRICT; + +CREATE FUNCTION qdb._rpc(method text, params jsonb) RETURNS jsonb LANGUAGE sql AS $$ + SELECT qdb._rpc_text(method, params::text)::jsonb +$$; + +CREATE FUNCTION qdb._rpc_result(method text, params jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc(method, params); + IF response ? 'error' THEN + RAISE EXCEPTION 'Quantum DB sidecar error [%]: %', + response #>> '{error,code}', response #>> '{error,message}' + USING ERRCODE = '58000', DETAIL = (response->'error')::text; + END IF; + IF NOT response ? 'result' THEN + RAISE EXCEPTION 'Quantum DB sidecar returned an invalid response' + USING ERRCODE = '58000'; + END IF; + RETURN response->'result'; +END $$; + +REVOKE ALL ON FUNCTION qdb._rpc_text(text, text), qdb._rpc(text, jsonb), qdb._rpc_result(text, jsonb) FROM PUBLIC; + +CREATE FUNCTION qdb._begin_dataset_upload(relation text, snapshot_version text, key_column name, artifact_checksum text, expected_rows integer) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.begin', jsonb_build_object( + 'relation', relation, 'snapshot_version', snapshot_version, 'key_column', key_column, + 'artifact_checksum', artifact_checksum, 'expected_rows', expected_rows)) +$$; + +CREATE FUNCTION qdb._append_dataset_page(upload_id text, page integer, rows jsonb) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.append', jsonb_build_object('upload_id', upload_id, 'page', page, 'rows', rows)) +$$; + +CREATE FUNCTION qdb._commit_dataset_upload(upload_id text) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.commit', jsonb_build_object('upload_id', upload_id)) +$$; + +CREATE FUNCTION qdb._sync_dataset(relation text, snapshot_version text, key_column name, artifact_checksum text, rows jsonb) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE upload jsonb; upload_id text; page_rows jsonb; page_no integer := 0; +BEGIN + upload := qdb._begin_dataset_upload(relation, snapshot_version, key_column, artifact_checksum, jsonb_array_length(rows)); + upload_id := upload->>'upload_id'; + FOR page_rows IN + SELECT jsonb_agg(item ORDER BY ordinal) + FROM jsonb_array_elements(rows) WITH ORDINALITY AS page_items(item, ordinal) + GROUP BY ((ordinal - 1) / 256) ORDER BY ((ordinal - 1) / 256) + LOOP + PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); + page_no := page_no + 1; + END LOOP; + RETURN qdb._commit_dataset_upload(upload_id); +END $$; + +CREATE FUNCTION qdb._remove_sidecar_dataset(relation text) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.unregister', jsonb_build_object('relation', relation)) +$$; + +REVOKE ALL ON FUNCTION qdb._begin_dataset_upload(text, text, name, text, integer), + qdb._append_dataset_page(text, integer, jsonb), qdb._commit_dataset_upload(text), + qdb._sync_dataset(text, text, name, text, jsonb), qdb._remove_sidecar_dataset(text) FROM PUBLIC; + +CREATE FUNCTION qdb._reject_write() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'Quantum DB dataset % is read-only', TG_TABLE_NAME USING ERRCODE = '25006'; +END $$; + +CREATE FUNCTION qdb_register_dataset(table_name regclass, key_column name, indexed_columns name[], options jsonb DEFAULT '{}'::jsonb) +RETURNS uuid LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE id uuid; column_name name; trigger_base text; snapshot text; values_expression text; + snapshot_rows jsonb; relation_text text; snapshot_checksum text; +BEGIN + IF array_length(indexed_columns, 1) IS NULL THEN RAISE EXCEPTION 'indexed_columns must not be empty'; END IF; + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=key_column AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'key column % does not exist on %', key_column, table_name; + END IF; + FOREACH column_name IN ARRAY indexed_columns LOOP + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=column_name AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'indexed column % does not exist on %', column_name, table_name; + END IF; + END LOOP; + -- Registration is an immutable snapshot operation. This excludes concurrent DML and DDL + -- until the read-only triggers and sidecar snapshot have been installed atomically. + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + snapshot := txid_current()::text; + INSERT INTO qdb.datasets(relation_oid,relation_name,key_column,indexed_columns,options,snapshot_version) + VALUES(table_name::oid,table_name,key_column,indexed_columns,options,snapshot) RETURNING dataset_id INTO id; + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s FOR EACH ROW EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_row', table_name); + EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %s FOR EACH STATEMENT EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_truncate', table_name); + SELECT string_agg(quote_literal(indexed_column::text) || ', to_jsonb(' || quote_ident(indexed_column::text) || ')', ', ') + INTO values_expression FROM unnest(indexed_columns) AS u(indexed_column); + relation_text := table_name::text; + EXECUTE format( + 'SELECT coalesce(jsonb_agg(jsonb_build_object(''key'', to_jsonb(%1$I), ''values'', jsonb_build_object(%2$s))), ''[]''::jsonb) FROM %3$s', + key_column, values_expression, relation_text) + INTO snapshot_rows; + snapshot_checksum := md5(snapshot_rows::text); + UPDATE qdb.datasets SET artifact_checksum=snapshot_checksum WHERE dataset_id=id; + BEGIN + PERFORM qdb._sync_dataset(relation_text, snapshot, key_column, snapshot_checksum, snapshot_rows); + INSERT INTO qdb.artifacts(dataset_id, kind, version, uri, checksum, metadata) + VALUES (id, 'quantum_index', snapshot, 'qdb://sidecar/' || id::text, + snapshot_checksum, jsonb_build_object('row_count', jsonb_array_length(snapshot_rows), 'key_column', key_column)) + ON CONFLICT (dataset_id, kind, version) DO UPDATE + SET uri=EXCLUDED.uri, checksum=EXCLUDED.checksum, metadata=EXCLUDED.metadata; + UPDATE qdb.datasets SET artifact_status='ready' WHERE dataset_id=id; + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN + -- The database remains immutable and correct; a later re-registration can build the artifact. + UPDATE qdb.datasets SET artifact_status='pending' WHERE dataset_id=id; + END; + RETURN id; +END $$; + +CREATE FUNCTION qdb_unregister_dataset(table_name regclass) RETURNS boolean +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE trigger_base text; affected integer; relation_text text; +BEGIN + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_row', table_name); + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_truncate', table_name); + DELETE FROM qdb.datasets WHERE relation_oid=table_name::oid; + GET DIAGNOSTICS affected = ROW_COUNT; + relation_text := table_name::text; + BEGIN + PERFORM qdb._remove_sidecar_dataset(relation_text); + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN + NULL; + END; + RETURN affected > 0; +END $$; + +CREATE FUNCTION qdb_explain(sql text) RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('explain', jsonb_build_object('sql', sql)) +$$; + +CREATE FUNCTION qdb_submit(sql text, options jsonb DEFAULT '{}'::jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; submitted_job_id uuid; submitted_trace_id uuid; +BEGIN + response := qdb._rpc_result('submit', jsonb_build_object('sql', sql) || options); + submitted_job_id := (response->>'job_id')::uuid; + submitted_trace_id := NULLIF(response->>'trace_id', '')::uuid; + INSERT INTO qdb.jobs(job_id, status, backend, trace_id, request) + VALUES (submitted_job_id, COALESCE(response->>'status', 'queued'), NULL, submitted_trace_id, + jsonb_build_object('sql', sql) || options) + ON CONFLICT (job_id) DO UPDATE SET status=EXCLUDED.status, trace_id=EXCLUDED.trace_id, + request=EXCLUDED.request, updated_at=clock_timestamp(); + RETURN response; +END $$; + +CREATE FUNCTION qdb_status(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('status', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status=response->>'status', backend=COALESCE(response->>'backend', backend), + trace_id=COALESCE(NULLIF(response->>'trace_id', '')::uuid, trace_id), + error_code=NULLIF(response->>'error_code', ''), error_message=NULLIF(response->>'error_message', ''), + updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_status.job_id; + RETURN response; +END $$; + +CREATE FUNCTION qdb_result(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('result', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status='succeeded', result=response->'result', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_result.job_id; + RETURN response; +END $$; + +CREATE FUNCTION qdb_cancel(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('cancel', jsonb_build_object('job_id', job_id)); + IF COALESCE((response->>'cancelled')::boolean, false) THEN + UPDATE qdb.jobs SET status='cancelled', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_cancel.job_id; + END IF; + RETURN response; +END $$; diff --git a/systems/quantum_db/pg_extension/sql/quantum_db--0.1.1.sql b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.1.sql new file mode 100644 index 0000000..2374cbb --- /dev/null +++ b/systems/quantum_db/pg_extension/sql/quantum_db--0.1.1.sql @@ -0,0 +1,259 @@ +CREATE SCHEMA IF NOT EXISTS qdb; + +CREATE TABLE qdb.datasets ( + dataset_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + relation_oid oid NOT NULL UNIQUE, + relation_name regclass NOT NULL UNIQUE, + key_column name NOT NULL, + indexed_columns name[] NOT NULL, + options jsonb NOT NULL DEFAULT '{}'::jsonb, + snapshot_version text NOT NULL, + artifact_status text NOT NULL DEFAULT 'pending' CHECK (artifact_status IN ('pending','building','ready','failed')), + artifact_checksum text NOT NULL DEFAULT '' CHECK (artifact_checksum = '' OR artifact_checksum ~ '^[0-9a-f]{32}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE qdb.artifacts ( + artifact_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + dataset_id uuid NOT NULL REFERENCES qdb.datasets(dataset_id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('tn_page','quantum_index','circuit','bit_order')), + version text NOT NULL, + uri text NOT NULL, + checksum text NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(dataset_id, kind, version) +); + +CREATE TABLE qdb.jobs ( + job_id uuid PRIMARY KEY, + status text NOT NULL CHECK (status IN ('queued','running','succeeded','failed','cancelled','timed_out')), + backend text, + trace_id uuid, + request jsonb NOT NULL, + result jsonb, + error_code text, + error_message text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +REVOKE INSERT, UPDATE, DELETE, TRUNCATE ON qdb.datasets, qdb.artifacts, qdb.jobs FROM PUBLIC; + +CREATE FUNCTION qdb._rpc_text(method text, params text) RETURNS text +AS 'MODULE_PATHNAME', 'qdb_rpc' LANGUAGE C STRICT; + +CREATE FUNCTION qdb._rpc(method text, params jsonb) RETURNS jsonb LANGUAGE sql AS $$ + SELECT qdb._rpc_text(method, params::text)::jsonb +$$; + +CREATE FUNCTION qdb._rpc_result(method text, params jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc(method, params); + IF response ? 'error' THEN + RAISE EXCEPTION 'Quantum DB sidecar error [%]: %', + response #>> '{error,code}', response #>> '{error,message}' + USING ERRCODE = '58000', DETAIL = (response->'error')::text; + END IF; + IF NOT response ? 'result' THEN + RAISE EXCEPTION 'Quantum DB sidecar returned an invalid response' + USING ERRCODE = '58000'; + END IF; + RETURN response->'result'; +END $$; + +REVOKE ALL ON FUNCTION qdb._rpc_text(text, text), qdb._rpc(text, jsonb), qdb._rpc_result(text, jsonb) FROM PUBLIC; + +CREATE FUNCTION qdb._begin_dataset_upload(relation text, snapshot_version text, key_column name, artifact_checksum text, expected_rows integer) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.begin', jsonb_build_object( + 'relation', relation, 'snapshot_version', snapshot_version, 'key_column', key_column, + 'artifact_checksum', artifact_checksum, 'expected_rows', expected_rows)) +$$; + +CREATE FUNCTION qdb._append_dataset_page(upload_id text, page integer, rows jsonb) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.append', jsonb_build_object('upload_id', upload_id, 'page', page, 'rows', rows)) +$$; + +CREATE FUNCTION qdb._commit_dataset_upload(upload_id text) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.commit', jsonb_build_object('upload_id', upload_id)) +$$; + +CREATE FUNCTION qdb._sync_dataset(relation text, snapshot_version text, key_column name, artifact_checksum text, rows jsonb) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE upload jsonb; upload_id text; page_rows jsonb; page_no integer := 0; +BEGIN + upload := qdb._begin_dataset_upload(relation, snapshot_version, key_column, artifact_checksum, jsonb_array_length(rows)); + upload_id := upload->>'upload_id'; + FOR page_rows IN + SELECT jsonb_agg(item ORDER BY ordinal) + FROM jsonb_array_elements(rows) WITH ORDINALITY AS page_items(item, ordinal) + GROUP BY ((ordinal - 1) / 256) ORDER BY ((ordinal - 1) / 256) + LOOP + PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); + page_no := page_no + 1; + END LOOP; + RETURN qdb._commit_dataset_upload(upload_id); +END $$; + +CREATE FUNCTION qdb._remove_sidecar_dataset(relation text) +RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('dataset.unregister', jsonb_build_object('relation', relation)) +$$; + +REVOKE ALL ON FUNCTION qdb._begin_dataset_upload(text, text, name, text, integer), + qdb._append_dataset_page(text, integer, jsonb), qdb._commit_dataset_upload(text), + qdb._sync_dataset(text, text, name, text, jsonb), qdb._remove_sidecar_dataset(text) FROM PUBLIC; + +CREATE FUNCTION qdb._reject_write() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'Quantum DB dataset % is read-only', TG_TABLE_NAME USING ERRCODE = '25006'; +END $$; + +CREATE FUNCTION qdb_register_dataset(table_name regclass, key_column name, indexed_columns name[], options jsonb DEFAULT '{}'::jsonb) +RETURNS uuid LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE id uuid; column_name name; trigger_base text; snapshot text; values_expression text; + relation_text text; snapshot_checksum text; row_payload jsonb; upload jsonb; upload_id text; + page_rows jsonb := '[]'::jsonb; row_count integer := 0; page_count integer := 0; page_no integer := 0; + unsafe_key_count bigint; +BEGIN + IF array_length(indexed_columns, 1) IS NULL THEN RAISE EXCEPTION 'indexed_columns must not be empty'; END IF; + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=key_column AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'key column % does not exist on %', key_column, table_name; + END IF; + FOREACH column_name IN ARRAY indexed_columns LOOP + IF NOT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=table_name AND attname=column_name AND attnum>0 AND NOT attisdropped) THEN + RAISE EXCEPTION 'indexed column % does not exist on %', column_name, table_name; + END IF; + END LOOP; + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + -- The Custom Scan hash uses textual key output with a fixed 1024-byte entry. + -- Reject values it cannot represent; otherwise a sidecar candidate could omit + -- a real tuple before PostgreSQL has a chance to recheck it. + EXECUTE format('SELECT count(*) FROM %s WHERE %I IS NULL OR length(%I::text) >= 1024', + table_name, key_column, key_column) INTO unsafe_key_count; + IF unsafe_key_count > 0 THEN + RAISE EXCEPTION 'key column % contains % NULL or oversized values unsupported by Quantum DB candidate scan', + key_column, unsafe_key_count USING ERRCODE = '22023'; + END IF; + snapshot := txid_current()::text; + SELECT string_agg(quote_literal(indexed_column::text) || ', to_jsonb(' || quote_ident(indexed_column::text) || ')', ', ') + INTO values_expression FROM unnest(indexed_columns) AS u(indexed_column); + relation_text := table_name::text; + snapshot_checksum := md5(''); + FOR row_payload IN EXECUTE format( + 'SELECT jsonb_build_object(''key'', to_jsonb(%1$I), ''values'', jsonb_build_object(%2$s)) FROM %3$s ORDER BY %1$I', + key_column, values_expression, relation_text) + LOOP + snapshot_checksum := md5(snapshot_checksum || row_payload::text); + row_count := row_count + 1; + END LOOP; + INSERT INTO qdb.datasets(relation_oid,relation_name,key_column,indexed_columns,options,snapshot_version,artifact_checksum) + VALUES(table_name::oid,table_name,key_column,indexed_columns,options,snapshot,snapshot_checksum) RETURNING dataset_id INTO id; + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s FOR EACH ROW EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_row', table_name); + EXECUTE format('CREATE TRIGGER %I BEFORE TRUNCATE ON %s FOR EACH STATEMENT EXECUTE FUNCTION qdb._reject_write()', trigger_base || '_truncate', table_name); + BEGIN + upload := qdb._begin_dataset_upload(relation_text, snapshot, key_column, snapshot_checksum, row_count); + upload_id := upload->>'upload_id'; + FOR row_payload IN EXECUTE format( + 'SELECT jsonb_build_object(''key'', to_jsonb(%1$I), ''values'', jsonb_build_object(%2$s)) FROM %3$s ORDER BY %1$I', + key_column, values_expression, relation_text) + LOOP + page_rows := page_rows || jsonb_build_array(row_payload); + page_count := page_count + 1; + IF page_count = 256 THEN + PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); + page_rows := '[]'::jsonb; page_count := 0; page_no := page_no + 1; + END IF; + END LOOP; + IF page_count > 0 THEN PERFORM qdb._append_dataset_page(upload_id, page_no, page_rows); END IF; + PERFORM qdb._commit_dataset_upload(upload_id); + INSERT INTO qdb.artifacts(dataset_id, kind, version, uri, checksum, metadata) + VALUES (id, 'quantum_index', snapshot, 'qdb://sidecar/' || id::text, + snapshot_checksum, jsonb_build_object('row_count', row_count, 'key_column', key_column, 'transfer', 'streamed_pages_v1')) + ON CONFLICT (dataset_id, kind, version) DO UPDATE + SET uri=EXCLUDED.uri, checksum=EXCLUDED.checksum, metadata=EXCLUDED.metadata; + UPDATE qdb.datasets SET artifact_status='ready' WHERE dataset_id=id; + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN + UPDATE qdb.datasets SET artifact_status='pending' WHERE dataset_id=id; + END; + RETURN id; +END $$; + +CREATE FUNCTION qdb_unregister_dataset(table_name regclass) RETURNS boolean +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, qdb AS $$ +DECLARE trigger_base text; affected integer; relation_text text; +BEGIN + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', table_name); + trigger_base := 'qdb_readonly_' || table_name::oid::text; + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_row', table_name); + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %s', trigger_base || '_truncate', table_name); + DELETE FROM qdb.datasets WHERE relation_oid=table_name::oid; + GET DIAGNOSTICS affected = ROW_COUNT; + relation_text := table_name::text; + BEGIN + PERFORM qdb._remove_sidecar_dataset(relation_text); + EXCEPTION WHEN SQLSTATE '08006' OR SQLSTATE '58000' THEN + NULL; + END; + RETURN affected > 0; +END $$; + +CREATE FUNCTION qdb_explain(sql text) RETURNS jsonb LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ + SELECT qdb._rpc_result('explain', jsonb_build_object('sql', sql)) +$$; + +CREATE FUNCTION qdb_submit(sql text, options jsonb DEFAULT '{}'::jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; submitted_job_id uuid; submitted_trace_id uuid; +BEGIN + response := qdb._rpc_result('submit', jsonb_build_object('sql', sql) || options); + submitted_job_id := (response->>'job_id')::uuid; + submitted_trace_id := NULLIF(response->>'trace_id', '')::uuid; + INSERT INTO qdb.jobs(job_id, status, backend, trace_id, request) + VALUES (submitted_job_id, COALESCE(response->>'status', 'queued'), NULL, submitted_trace_id, + jsonb_build_object('sql', sql) || options) + ON CONFLICT (job_id) DO UPDATE SET status=EXCLUDED.status, trace_id=EXCLUDED.trace_id, + request=EXCLUDED.request, updated_at=clock_timestamp(); + RETURN response; +END $$; + +CREATE FUNCTION qdb_status(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('status', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status=response->>'status', backend=COALESCE(response->>'backend', backend), + trace_id=COALESCE(NULLIF(response->>'trace_id', '')::uuid, trace_id), + error_code=NULLIF(response->>'error_code', ''), error_message=NULLIF(response->>'error_message', ''), + updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_status.job_id; + RETURN response; +END $$; + +CREATE FUNCTION qdb_result(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('result', jsonb_build_object('job_id', job_id)); + UPDATE qdb.jobs SET status='succeeded', result=response->'result', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_result.job_id; + RETURN response; +END $$; + +CREATE FUNCTION qdb_cancel(job_id uuid) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, qdb AS $$ +DECLARE response jsonb; +BEGIN + response := qdb._rpc_result('cancel', jsonb_build_object('job_id', job_id)); + IF COALESCE((response->>'cancelled')::boolean, false) THEN + UPDATE qdb.jobs SET status='cancelled', updated_at=clock_timestamp() + WHERE qdb.jobs.job_id=qdb_cancel.job_id; + END IF; + RETURN response; +END $$; diff --git a/systems/quantum_db/pg_extension/sql/quantum_db.sql b/systems/quantum_db/pg_extension/sql/quantum_db.sql new file mode 100644 index 0000000..230f20d --- /dev/null +++ b/systems/quantum_db/pg_extension/sql/quantum_db.sql @@ -0,0 +1,8 @@ +CREATE EXTENSION quantum_db; +CREATE TABLE qdb_test(id bigint PRIMARY KEY, value integer, category text); +INSERT INTO qdb_test VALUES (1, 10, 'a'), (2, 20, 'b'); +SELECT qdb_register_dataset('qdb_test', 'id', ARRAY['value','category']) IS NOT NULL AS registered; +SELECT count(*) FROM qdb_test WHERE value = 10; +SELECT qdb_unregister_dataset('qdb_test'); +DROP TABLE qdb_test; +DROP EXTENSION quantum_db CASCADE; diff --git a/systems/quantum_db/pg_extension/sql/rpc_integration.sql b/systems/quantum_db/pg_extension/sql/rpc_integration.sql new file mode 100644 index 0000000..ffde5d8 --- /dev/null +++ b/systems/quantum_db/pg_extension/sql/rpc_integration.sql @@ -0,0 +1,9 @@ +\set ON_ERROR_STOP on + +SELECT qdb_explain('/*+ QDB_CLASSICAL */ SELECT count(*) FROM qdb_test WHERE value = 10') AS explanation; + +SELECT (qdb_submit('/*+ QDB_CLASSICAL */ SELECT count(*) FROM qdb_test WHERE value = 10', + '{"wait": true, "timeout_ms": 5000}'::jsonb)->>'job_id')::uuid AS job_id \gset + +SELECT qdb_status(:'job_id') AS status; +SELECT qdb_result(:'job_id') AS result; diff --git a/systems/quantum_db/protocol/commands.py b/systems/quantum_db/protocol/commands.py new file mode 100644 index 0000000..ff7022c --- /dev/null +++ b/systems/quantum_db/protocol/commands.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +ExecutionBackend = Literal["classical", "tn", "quantum", "hybrid"] + + +@dataclass(frozen=True, slots=True) +class ExecuteQueryCommand: + query_id: str + relational_ir: dict[str, Any] + backend: ExecutionBackend + timeout_ms: int | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class CancelTaskCommand: + task_id: str + reason: str = "user_requested" diff --git a/systems/quantum_db/protocol/dto.py b/systems/quantum_db/protocol/dto.py new file mode 100644 index 0000000..7d699a6 --- /dev/null +++ b/systems/quantum_db/protocol/dto.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +Lane = Literal["command", "event", "data", "status"] +MetricCategory = Literal["resource", "performance", "custom"] +HealthLevel = Literal["healthy", "warning", "critical", "unknown"] + + +@dataclass(slots=True, frozen=True) +class MessageDTO: + message_id: str + lane: Lane + name: str + payload: Any = None + timestamp_ms: int = 0 + source: str | None = None + target: str | None = None + trace_id: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True, frozen=True) +class RequestDTO: + request_id: str + user_id: str + request_type: str + raw_text: str + structured_params: dict[str, Any] = field(default_factory=dict) + session_id: str | None = None + deadline_ms: int | None = None + priority_hint: str | None = None + + +@dataclass(slots=True, frozen=True) +class TaskDTO: + task_id: str + priority: int = 0 + dependencies: tuple[str, ...] = () + payload: Any = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True, frozen=True) +class TaskStateDTO: + task_id: str + state: str + updated_at: float + detail: str = "" + error_code: str | None = None + error_message: str | None = None + retryable: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True, frozen=True) +class MetricDTO: + component: str + category: MetricCategory + name: str + value: float + timestamp: float + + +@dataclass(slots=True, frozen=True) +class HealthDTO: + component: str + level: HealthLevel + updated_at: float + detail: str = "" + + +@dataclass(slots=True, frozen=True) +class ErrorDTO: + code: str + message: str + detail: str = "" + retryable: bool = False + cause_id: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True, frozen=True) +class AckDTO: + ok: bool + message_id: str + timestamp_ms: int + error: ErrorDTO | None = None diff --git a/systems/quantum_db/protocol/errors.py b/systems/quantum_db/protocol/errors.py new file mode 100644 index 0000000..f7ca9c0 --- /dev/null +++ b/systems/quantum_db/protocol/errors.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ErrorCode: + code: str + retryable: bool = False + + +INVALID_REQUEST = ErrorCode("INVALID_REQUEST") +SIDECAR_UNAVAILABLE = ErrorCode("SIDECAR_UNAVAILABLE", True) +PREFLIGHT_FAILED = ErrorCode("PREFLIGHT_FAILED") +BACKEND_TIMEOUT = ErrorCode("BACKEND_TIMEOUT", True) +EXECUTION_FAILED = ErrorCode("EXECUTION_FAILED", True) +SNAPSHOT_MISMATCH = ErrorCode("SNAPSHOT_MISMATCH") diff --git a/systems/quantum_db/protocol/events.py b/systems/quantum_db/protocol/events.py new file mode 100644 index 0000000..dadbcd2 --- /dev/null +++ b/systems/quantum_db/protocol/events.py @@ -0,0 +1,41 @@ +"""Immutable events emitted by execution and monitoring components.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +from .status import TaskStatus + +EventKind = Literal["task_state", "task_completed", "alert"] + + +@dataclass(frozen=True, slots=True) +class Event: + event_id: str + kind: EventKind + timestamp_ms: int + source: str + trace_id: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class TaskStateEvent(Event): + task_id: str = "" + status: TaskStatus = TaskStatus.QUEUED + detail: str = "" + + +@dataclass(frozen=True, slots=True) +class TaskCompletedEvent(Event): + task_id: str = "" + status: TaskStatus = TaskStatus.SUCCEEDED + result_ref: str | None = None + error_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class AlertEvent(Event): + component: str = "" + severity: Literal["warning", "critical"] = "warning" + message: str = "" diff --git a/systems/quantum_db/protocol/status.py b/systems/quantum_db/protocol/status.py new file mode 100644 index 0000000..3a0645e --- /dev/null +++ b/systems/quantum_db/protocol/status.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from enum import StrEnum + + +class TaskStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + TIMED_OUT = "timed_out" + + +TERMINAL_TASK_STATUSES = frozenset({TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED, TaskStatus.TIMED_OUT}) diff --git a/systems/quantum_db/pyproject.toml b/systems/quantum_db/pyproject.toml new file mode 100644 index 0000000..bc2c415 --- /dev/null +++ b/systems/quantum_db/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "quantum-db" +version = "0.1.1" +description = "PostgreSQL extension and Python sidecar for fail-safe quantum/TN query acceleration" +readme = "README.md" +requires-python = ">=3.11" +authors = [{name = "Quantum DB contributors"}] +classifiers = ["Development Status :: 3 - Alpha", "Programming Language :: Python :: 3.11", "Topic :: Database"] +dependencies = [] + +[project.optional-dependencies] +dev = [] +wukong = [] + +[project.scripts] +qdb-sidecar = "qdb.server:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["qdb*", "qute*", "data_plane*", "control_plane*", "infra*", "interface*", "protocol*"] +exclude = ["tests*", "storage-main 2*"] +namespaces = true diff --git a/systems/quantum_db/qdb/__init__.py b/systems/quantum_db/qdb/__init__.py new file mode 100644 index 0000000..9ad4142 --- /dev/null +++ b/systems/quantum_db/qdb/__init__.py @@ -0,0 +1,5 @@ +"""Quantum database sidecar runtime.""" + +from .service import QuantumDatabaseService + +__all__ = ["QuantumDatabaseService"] diff --git a/systems/quantum_db/qdb/__main__.py b/systems/quantum_db/qdb/__main__.py new file mode 100644 index 0000000..0a68a64 --- /dev/null +++ b/systems/quantum_db/qdb/__main__.py @@ -0,0 +1,3 @@ +from .server import main + +raise SystemExit(main()) diff --git a/systems/quantum_db/qdb/candidate_executor.py b/systems/quantum_db/qdb/candidate_executor.py new file mode 100644 index 0000000..99d0d3a --- /dev/null +++ b/systems/quantum_db/qdb/candidate_executor.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import math +from typing import Any + +from data_plane.tn.compute import TNCompute +from qute.quantum.backend.ideal_simulator import IdealSimulatorBackend +from qute.quantum.task_model import make_task + + +class CandidateExecutor: + """Run bounded TN/quantum probes without making their samples authoritative. + + The immutable classical snapshot is always used to form the candidate superset. + These probes provide the real TN-page construction and Grover-style amplitude + sampling used for routing/telemetry; they are intentionally unable to remove a + key from that superset. + """ + + def __init__(self, *, max_qubits: int, max_probe_rows: int = 256, shots: int = 256) -> None: + self.max_qubits = max(1, max_qubits) + self.max_probe_rows = max(2, max_probe_rows) + self.shots = max(1, shots) + self._simulator = IdealSimulatorBackend(max_qubits=self.max_qubits) + + def execute(self, *, route: str, rows: list[Any], candidate_keys: list[Any], shots: int | None = None, + hardware_grover_iterations: int | None = None) -> dict[str, Any]: + result: dict[str, Any] = { + "executed": False, + "advisory_only": True, + "candidate_mutation": "forbidden", + "route": route, + } + if route not in {"quantum", "tn", "hybrid"}: + return result + selected_rows = self._sample_rows(rows) + if route in {"tn", "hybrid"}: + result["tn"] = self._tn_probe(selected_rows, candidate_keys) + if route in {"quantum", "hybrid"}: + result["quantum"] = self._grover_probe(selected_rows, candidate_keys, shots=shots or self.shots, + hardware_grover_iterations=hardware_grover_iterations) + result["executed"] = any(isinstance(result.get(name), dict) and result[name].get("executed") for name in ("tn", "quantum")) + return result + + def _sample_rows(self, rows: list[Any]) -> list[dict[str, Any]]: + valid = [row for row in rows if isinstance(row, dict) and "key" in row and isinstance(row.get("values"), dict)] + if len(valid) <= self.max_probe_rows: + return valid + stride = len(valid) / self.max_probe_rows + return [valid[min(len(valid) - 1, int(index * stride))] for index in range(self.max_probe_rows)] + + def _tn_probe(self, rows: list[dict[str, Any]], candidate_keys: list[Any]) -> dict[str, Any]: + if not rows: + return {"executed": False, "reason": "empty_snapshot"} + candidates = {self._key_token(key) for key in candidate_keys} + samples = [[1.0 if self._key_token(row["key"]) in candidates else 0.0] for row in rows] + try: + num_sites = min(8, max(1, int(math.ceil(math.log2(max(2, len(samples))))))) + compute = TNCompute(seed=7) + page = compute.fit_mps_from_samples(samples, page_id="qdb-candidate-tn", num_sites=num_sites, phys_dim=2, chi_max=4) + return { + "executed": True, + "engine": "TNCompute.fit_mps_from_samples", + "estimated_selectivity": compute.estimate_selectivity(samples, lambda item: bool(item[0])), + "page": page.summary(), + } + except Exception as exc: + return {"executed": False, "reason": f"{type(exc).__name__}: {exc}"} + + def _grover_probe(self, rows: list[dict[str, Any]], candidate_keys: list[Any], *, shots: int, + hardware_grover_iterations: int | None = None) -> dict[str, Any]: + if not rows: + return {"executed": False, "reason": "empty_snapshot"} + candidate_set = {self._key_token(key) for key in candidate_keys} + width = max(1, int(math.ceil(math.log2(max(2, len(rows)))))) + if width > self.max_qubits: + return {"executed": False, "reason": "qubit_budget_exceeded", "required_qubits": width} + marked = [index for index, row in enumerate(rows) if self._key_token(row["key"]) in candidate_set] + marked_set = set(marked) + # An ideal Grover oracle gives marked addresses more measurement mass. This + # is a bounded probe rather than a candidate producer, so sampling error can + # never cause a SQL false negative. + amplification = max(1, min(8, int(math.sqrt(max(1, len(rows) / max(1, len(marked))))))) + iterations = hardware_grover_iterations if hardware_grover_iterations is not None else max(1, min(2, amplification)) + if iterations not in {1, 2}: + return {"executed": False, "reason": "hardware_grover_iterations must be 1 or 2"} + if len(rows) == (1 << width) and len(marked) == 1: + theta = math.asin(1 / math.sqrt(len(rows))) + marked_probability = math.sin((2 * iterations + 1) * theta) ** 2 + non_marked_probability = (1.0 - marked_probability) / (len(rows) - 1) + probabilities = {format(index, f"0{width}b"): marked_probability if index in marked_set else non_marked_probability + for index in range(len(rows))} + else: + weights = {index: (amplification if index in marked_set else 1) for index in range(len(rows))} + total = sum(weights.values()) + probabilities = {format(index, f"0{width}b"): weight / total for index, weight in weights.items()} + marked_probability = sum(probabilities[format(index, f"0{width}b")] for index in marked_set) + qpanda3_ir = self._qpanda3_ir(width=width, amplification=amplification, marked_addresses=marked, + iterations=iterations) + task = make_task("QMEASURE", { + "probabilities": probabilities, + "circuit_skeleton": "grover_candidate_probe", + "circuit_depth": max(1, 2 * iterations * width + 2), + "two_qubit_gate_count": max(0, iterations * max(0, width - 1)), + "marked_addresses": len(marked), + }, shots=shots, seed=7, quality_budget={"max_error": 1.0}, + resource_budget={"max_qubits": self.max_qubits, "max_depth": 10_000, "max_shots": shots}) + try: + measured = self._simulator.execute(task) + sampled_keys = [rows[int(state, 2)]["key"] for state, count in measured.histogram.items() if count and int(state, 2) < len(rows)] + return { + "executed": True, + "engine": "IdealSimulatorBackend", + "operator": "QMEASURE", + "marked_addresses": len(marked), + "ideal_marked_probability": marked_probability, + "hardware_grover_iterations": iterations, + "histogram": measured.histogram, + "sampled_keys": sampled_keys, + "resource": {"logical_qubits": measured.resources.logical_qubits, "circuit_depth": measured.resources.circuit_depth, + "two_qubit_gate_count": measured.resources.two_qubit_gate_count}, + "qpanda3_ir": qpanda3_ir, + } + except Exception as exc: + return {"executed": False, "reason": f"{type(exc).__name__}: {exc}"} + + @staticmethod + def _key_token(value: Any) -> str: + return repr(value) + + @staticmethod + def _qpanda3_ir(*, width: int, amplification: int, marked_addresses: list[int], + iterations: int | None = None) -> dict[str, Any] | None: + """Emit an executable three-qubit Grover oracle/diffuser in U3+CZ. + + The previous hardware smoke circuit only applied nearest-neighbour CZ + gates, so an even number of iterations cancelled and no candidate was + amplified. A general multi-address oracle needs a larger compiler; for + the fixed eight-row probe we instead emit a complete, auditable Grover + circuit for exactly one marked three-bit address. Other shapes keep the + ideal probe but intentionally do not submit a misleading hardware circuit. + """ + if width != 3 or len(marked_addresses) != 1 or not 0 <= marked_addresses[0] < 8: + return None + + operations: list[dict[str, Any]] = [] + iterations = max(1, min(2, int(amplification if iterations is None else iterations))) + marked = marked_addresses[0] + for qubit in range(width): + CandidateExecutor._append_h(operations, qubit) + for _ in range(iterations): + CandidateExecutor._append_marked_phase_oracle(operations, marked) + CandidateExecutor._append_diffuser(operations) + operations.extend({"gate": "measure", "qubits": [qubit], "cbit": qubit} for qubit in range(width)) + two_qubit_gate_count = sum(op["gate"] == "cz" for op in operations) + operation_count = len(operations) + ideal_marked_probability = math.sin((2 * iterations + 1) * math.asin(1 / math.sqrt(8))) ** 2 + return { + "ir_version": "qdb-grover-probe-v2", "model_id": "qdb-grover-candidate-probe", + "target_backend": "qpanda3", "logical_to_physical": {str(qubit): qubit for qubit in range(width)}, + "probe_contract": "grover-oracle-diffuser-v1", "marked_addresses": [marked], "iterations": iterations, + "expected_marked_state": f"0x{marked:x}", "ideal_marked_probability": ideal_marked_probability, + "operation_count": operation_count, "two_qubit_gate_count": two_qubit_gate_count, "operations": operations, + } + + @staticmethod + def _append_u3(operations: list[dict[str, Any]], qubit: int, theta: float, phi: float, lam: float) -> None: + operations.append({"gate": "u3", "qubits": [qubit], "params": [theta, phi, lam]}) + + @staticmethod + def _append_h(operations: list[dict[str, Any]], qubit: int) -> None: + CandidateExecutor._append_u3(operations, qubit, math.pi / 2, 0.0, math.pi) + + @staticmethod + def _append_x(operations: list[dict[str, Any]], qubit: int) -> None: + CandidateExecutor._append_u3(operations, qubit, math.pi, 0.0, math.pi) + + @staticmethod + def _append_phase(operations: list[dict[str, Any]], qubit: int, angle: float) -> None: + CandidateExecutor._append_u3(operations, qubit, 0.0, 0.0, angle) + + @staticmethod + def _append_cx(operations: list[dict[str, Any]], control: int, target: int) -> None: + CandidateExecutor._append_h(operations, target) + operations.append({"gate": "cz", "qubits": [control, target]}) + CandidateExecutor._append_h(operations, target) + + @staticmethod + def _append_ccz(operations: list[dict[str, Any]], first: int, second: int, target: int) -> None: + """CCZ decomposition using only U3 and CZ (via H-CZ-H CNOTs).""" + CandidateExecutor._append_cx(operations, second, target) + CandidateExecutor._append_phase(operations, target, -math.pi / 4) + CandidateExecutor._append_cx(operations, first, target) + CandidateExecutor._append_phase(operations, target, math.pi / 4) + CandidateExecutor._append_cx(operations, second, target) + CandidateExecutor._append_phase(operations, target, -math.pi / 4) + CandidateExecutor._append_cx(operations, first, target) + CandidateExecutor._append_phase(operations, second, math.pi / 4) + CandidateExecutor._append_phase(operations, target, math.pi / 4) + CandidateExecutor._append_cx(operations, first, second) + CandidateExecutor._append_phase(operations, first, math.pi / 4) + CandidateExecutor._append_phase(operations, second, -math.pi / 4) + CandidateExecutor._append_cx(operations, first, second) + + @staticmethod + def _append_marked_phase_oracle(operations: list[dict[str, Any]], marked: int) -> None: + bits = format(marked, "03b") + for qubit, bit in enumerate(bits): + if bit == "0": + CandidateExecutor._append_x(operations, qubit) + CandidateExecutor._append_ccz(operations, 0, 1, 2) + for qubit, bit in enumerate(bits): + if bit == "0": + CandidateExecutor._append_x(operations, qubit) + + @staticmethod + def _append_diffuser(operations: list[dict[str, Any]]) -> None: + for qubit in range(3): + CandidateExecutor._append_h(operations, qubit) + CandidateExecutor._append_x(operations, qubit) + CandidateExecutor._append_ccz(operations, 0, 1, 2) + for qubit in range(3): + CandidateExecutor._append_x(operations, qubit) + CandidateExecutor._append_h(operations, qubit) diff --git a/systems/quantum_db/qdb/client.py b/systems/quantum_db/qdb/client.py new file mode 100644 index 0000000..353dbbe --- /dev/null +++ b/systems/quantum_db/qdb/client.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import json +import socket +import uuid +from typing import Any + +from .protocol import MAX_REQUEST_BYTES, PROTOCOL_VERSION, QDBError + + +def call(socket_path: str, method: str, params: dict[str, Any], *, timeout: float = 30.0) -> Any: + request_id = str(uuid.uuid4()) + payload = json.dumps({"version": PROTOCOL_VERSION, "id": request_id, "method": method, "params": params}, separators=(",", ":")).encode() + b"\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as conn: + conn.settimeout(timeout) + conn.connect(socket_path) + conn.sendall(payload) + file = conn.makefile("rb") + raw_response = file.readline(MAX_REQUEST_BYTES + 1) + if not raw_response or len(raw_response) > MAX_REQUEST_BYTES: + raise QDBError("RESPONSE_TOO_LARGE", "sidecar response exceeds size limit", retryable=True) + response = json.loads(raw_response) + if response.get("id") != request_id: + raise QDBError("INVALID_RESPONSE", "response id mismatch") + if "error" in response: + error = response["error"] + raise QDBError(str(error.get("code", "RPC_ERROR")), str(error.get("message", "sidecar error")), retryable=bool(error.get("retryable"))) + return response.get("result") diff --git a/systems/quantum_db/qdb/jobs.py b/systems/quantum_db/qdb/jobs.py new file mode 100644 index 0000000..15d82a7 --- /dev/null +++ b/systems/quantum_db/qdb/jobs.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import threading +import time +import uuid +from pathlib import Path +from typing import Any + +from .protocol import JobStatus, QDBError, TERMINAL_STATUSES + + +class JobStore: + def __init__(self, path: str | Path) -> None: + self.path = str(path) + Path(self.path).parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._init_schema() + os.chmod(self.path, 0o600) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.path, timeout=10) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA busy_timeout=10000") + return conn + + def _init_schema(self) -> None: + with self._connect() as conn: + conn.execute("""CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, status TEXT NOT NULL, request_json TEXT NOT NULL, + result_json TEXT, error_code TEXT, error_message TEXT, backend TEXT, + trace_id TEXT NOT NULL, created_at REAL NOT NULL, updated_at REAL NOT NULL, + cancel_requested INTEGER NOT NULL DEFAULT 0)""") + conn.execute("""CREATE TABLE IF NOT EXISTS datasets ( + relation TEXT PRIMARY KEY, snapshot_version TEXT NOT NULL, + key_column TEXT NOT NULL, artifact_checksum TEXT NOT NULL DEFAULT '', rows_json TEXT NOT NULL, + created_at REAL NOT NULL, updated_at REAL NOT NULL)""") + # Row payloads live separately from metadata. The legacy rows_json column + # remains readable so an in-place upgrade never invalidates a registered + # immutable snapshot. + conn.execute("""CREATE TABLE IF NOT EXISTS dataset_rows ( + relation TEXT NOT NULL, ordinal INTEGER NOT NULL, key_json TEXT NOT NULL, + values_json TEXT NOT NULL, PRIMARY KEY(relation, ordinal), + FOREIGN KEY(relation) REFERENCES datasets(relation) ON DELETE CASCADE)""") + conn.execute("""CREATE TABLE IF NOT EXISTS dataset_uploads ( + upload_id TEXT PRIMARY KEY, relation TEXT NOT NULL, snapshot_version TEXT NOT NULL, + key_column TEXT NOT NULL, artifact_checksum TEXT NOT NULL, expected_rows INTEGER NOT NULL, + next_page INTEGER NOT NULL DEFAULT 0, received_rows INTEGER NOT NULL DEFAULT 0, + created_at REAL NOT NULL)""") + conn.execute("""CREATE TABLE IF NOT EXISTS dataset_upload_rows ( + upload_id TEXT NOT NULL, ordinal INTEGER NOT NULL, key_json TEXT NOT NULL, + values_json TEXT NOT NULL, PRIMARY KEY(upload_id, ordinal), + FOREIGN KEY(upload_id) REFERENCES dataset_uploads(upload_id) ON DELETE CASCADE)""") + columns = {str(row[1]) for row in conn.execute("PRAGMA table_info(datasets)")} + if "artifact_checksum" not in columns: + conn.execute("ALTER TABLE datasets ADD COLUMN artifact_checksum TEXT NOT NULL DEFAULT ''") + conn.execute("UPDATE jobs SET status=?, error_code=?, error_message=?, updated_at=? WHERE status=?", + (JobStatus.FAILED, "SIDECAR_RESTARTED", "sidecar restarted while job was running", time.time(), JobStatus.RUNNING)) + + def create(self, request: dict[str, Any], *, backend: str, trace_id: str) -> str: + job_id, now = str(uuid.uuid4()), time.time() + with self._lock, self._connect() as conn: + conn.execute("INSERT INTO jobs(job_id,status,request_json,backend,trace_id,created_at,updated_at) VALUES(?,?,?,?,?,?,?)", + (job_id, JobStatus.QUEUED, json.dumps(request), backend, trace_id, now, now)) + return job_id + + def get(self, job_id: str) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone() + if row is None: + raise QDBError("JOB_NOT_FOUND", f"unknown job: {job_id}") + out = dict(row) + out["request"] = json.loads(out.pop("request_json")) + out["result"] = json.loads(out.pop("result_json")) if out.get("result_json") else None + return out + + def update(self, job_id: str, status: JobStatus, *, result: Any = None, error_code: str | None = None, + error_message: str | None = None, preserve_cancelled: bool = False) -> bool: + with self._lock, self._connect() as conn: + statement = "UPDATE jobs SET status=?,result_json=?,error_code=?,error_message=?,updated_at=? WHERE job_id=?" + args: tuple[Any, ...] = (status, json.dumps(result) if result is not None else None, error_code, + error_message, time.time(), job_id) + if preserve_cancelled: + statement += " AND status<>?" + args += (JobStatus.CANCELLED,) + cursor = conn.execute(statement, args) + return cursor.rowcount > 0 + + def cancel(self, job_id: str) -> bool: + with self._lock, self._connect() as conn: + row = conn.execute("SELECT status FROM jobs WHERE job_id=?", (job_id,)).fetchone() + if row is None: + raise QDBError("JOB_NOT_FOUND", f"unknown job: {job_id}") + if JobStatus(row["status"]) in TERMINAL_STATUSES: + return False + cursor = conn.execute("""UPDATE jobs SET cancel_requested=1,status=?,updated_at=? + WHERE job_id=? AND status NOT IN (?,?,?,?)""", + (JobStatus.CANCELLED, time.time(), job_id, JobStatus.SUCCEEDED, + JobStatus.FAILED, JobStatus.CANCELLED, JobStatus.TIMED_OUT)) + return cursor.rowcount > 0 + + def register_dataset(self, *, relation: str, snapshot_version: str, key_column: str, + artifact_checksum: str, rows: list[dict[str, Any]]) -> dict[str, Any]: + if not relation or not snapshot_version or not key_column or not artifact_checksum: + raise QDBError("INVALID_DATASET", "relation, snapshot_version, key_column, and artifact_checksum are required") + self._validate_checksum(artifact_checksum) + self._validate_rows(rows) + encoded = json.dumps(rows, separators=(",", ":"), ensure_ascii=False) + now = time.time() + with self._lock, self._connect() as conn: + conn.execute( + """INSERT INTO datasets(relation,snapshot_version,key_column,artifact_checksum,rows_json,created_at,updated_at) + VALUES(?,?,?,?,?,?,?) + ON CONFLICT(relation) DO UPDATE SET snapshot_version=excluded.snapshot_version, + key_column=excluded.key_column, artifact_checksum=excluded.artifact_checksum, + rows_json=excluded.rows_json, updated_at=excluded.updated_at""", + (relation, snapshot_version, key_column, artifact_checksum, encoded, now, now), + ) + self._replace_rows(conn, relation, rows) + return {"relation": relation, "snapshot_version": snapshot_version, + "artifact_checksum": artifact_checksum, "row_count": len(rows)} + + def begin_dataset_upload(self, *, relation: str, snapshot_version: str, key_column: str, + artifact_checksum: str, expected_rows: int) -> dict[str, Any]: + if not relation or not snapshot_version or not key_column or expected_rows < 0: + raise QDBError("INVALID_DATASET", "relation, snapshot_version, key_column, and non-negative expected_rows are required") + self._validate_checksum(artifact_checksum) + upload_id, now = str(uuid.uuid4()), time.time() + with self._lock, self._connect() as conn: + # An incomplete upload is never visible to candidate queries. + conn.execute("DELETE FROM dataset_upload_rows WHERE upload_id IN (SELECT upload_id FROM dataset_uploads WHERE relation=?)", (relation,)) + conn.execute("DELETE FROM dataset_uploads WHERE relation=?", (relation,)) + conn.execute("""INSERT INTO dataset_uploads(upload_id,relation,snapshot_version,key_column,artifact_checksum,expected_rows,created_at) + VALUES(?,?,?,?,?,?,?)""", + (upload_id, relation, snapshot_version, key_column, artifact_checksum, expected_rows, now)) + return {"upload_id": upload_id, "relation": relation, "expected_rows": expected_rows} + + def append_dataset_page(self, *, upload_id: str, page: int, rows: list[dict[str, Any]]) -> dict[str, Any]: + self._validate_rows(rows) + with self._lock, self._connect() as conn: + upload = conn.execute("SELECT * FROM dataset_uploads WHERE upload_id=?", (upload_id,)).fetchone() + if upload is None: + raise QDBError("UPLOAD_NOT_FOUND", f"unknown dataset upload: {upload_id}") + if page != upload["next_page"]: + raise QDBError("UPLOAD_PAGE_ORDER", f"expected page {upload['next_page']}, got {page}", retryable=True) + first_ordinal = int(upload["received_rows"]) + if first_ordinal + len(rows) > int(upload["expected_rows"]): + raise QDBError("UPLOAD_TOO_MANY_ROWS", "page exceeds declared dataset row count") + conn.executemany("""INSERT INTO dataset_upload_rows(upload_id,ordinal,key_json,values_json) VALUES(?,?,?,?)""", + [(upload_id, first_ordinal + index, + json.dumps(item["key"], separators=(",", ":"), ensure_ascii=False), + json.dumps(item["values"], separators=(",", ":"), ensure_ascii=False)) + for index, item in enumerate(rows)]) + conn.execute("UPDATE dataset_uploads SET next_page=next_page+1, received_rows=received_rows+1*? WHERE upload_id=?", + (len(rows), upload_id)) + return {"upload_id": upload_id, "page": page, "received_rows": first_ordinal + len(rows)} + + def commit_dataset_upload(self, *, upload_id: str) -> dict[str, Any]: + now = time.time() + with self._lock, self._connect() as conn: + upload = conn.execute("SELECT * FROM dataset_uploads WHERE upload_id=?", (upload_id,)).fetchone() + if upload is None: + raise QDBError("UPLOAD_NOT_FOUND", f"unknown dataset upload: {upload_id}") + if int(upload["received_rows"]) != int(upload["expected_rows"]): + raise QDBError("UPLOAD_INCOMPLETE", "not all dataset rows have been uploaded", retryable=True) + # Metadata and all rows become visible together. rows_json is kept as a + # compact legacy fallback, while normal reads use dataset_rows. + conn.execute("""INSERT INTO datasets(relation,snapshot_version,key_column,artifact_checksum,rows_json,created_at,updated_at) + VALUES(?,?,?,?,?,?,?) + ON CONFLICT(relation) DO UPDATE SET snapshot_version=excluded.snapshot_version, + key_column=excluded.key_column, artifact_checksum=excluded.artifact_checksum, + rows_json=excluded.rows_json, updated_at=excluded.updated_at""", + (upload["relation"], upload["snapshot_version"], upload["key_column"], + upload["artifact_checksum"], "[]", now, now)) + conn.execute("DELETE FROM dataset_rows WHERE relation=?", (upload["relation"],)) + conn.execute("""INSERT INTO dataset_rows(relation,ordinal,key_json,values_json) + SELECT ?, ordinal, key_json, values_json FROM dataset_upload_rows + WHERE upload_id=? ORDER BY ordinal""", (upload["relation"], upload_id)) + conn.execute("DELETE FROM dataset_uploads WHERE upload_id=?", (upload_id,)) + return {"relation": upload["relation"], "snapshot_version": upload["snapshot_version"], + "artifact_checksum": upload["artifact_checksum"], "row_count": int(upload["expected_rows"])} + + def get_dataset(self, relation: str, *, snapshot_version: str = "") -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute("SELECT * FROM datasets WHERE relation=?", (relation,)).fetchone() + stored_rows = [ + {"key": json.loads(item["key_json"]), "values": json.loads(item["values_json"])} + for item in conn.execute("SELECT key_json,values_json FROM dataset_rows WHERE relation=? ORDER BY ordinal", (relation,)) + ] + if row is None: + return None + if snapshot_version and row["snapshot_version"] != snapshot_version: + raise QDBError("SNAPSHOT_MISMATCH", f"sidecar snapshot for {relation} does not match PostgreSQL registration", retryable=True) + return { + "relation": row["relation"], "snapshot_version": row["snapshot_version"], + "key_column": row["key_column"], "artifact_checksum": row["artifact_checksum"], + "rows": stored_rows if stored_rows else json.loads(row["rows_json"]), + } + + def get_unqualified_dataset(self, relation_name: str, *, snapshot_version: str = "") -> dict[str, Any] | None: + """Resolve an unqualified SQL relation only when it has one safe match. + + PostgreSQL's Custom Scan always supplies ``schema.table`` while ordinary + SQL commonly uses a table visible through ``search_path``. Sidecar jobs + must not guess between identically named tables in multiple schemas, so a + suffix match is accepted only when it is unique. + """ + if not relation_name or "." in relation_name: + return None + escaped_name = relation_name.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + with self._connect() as conn: + matches = conn.execute( + "SELECT relation FROM datasets WHERE relation LIKE ? ESCAPE '\\'", + (f"%.{escaped_name}",), + ).fetchall() + if len(matches) != 1: + return None + return self.get_dataset(str(matches[0]["relation"]), snapshot_version=snapshot_version) + + def unregister_dataset(self, relation: str) -> bool: + with self._lock, self._connect() as conn: + cursor = conn.execute("DELETE FROM datasets WHERE relation=?", (relation,)) + conn.execute("DELETE FROM dataset_upload_rows WHERE upload_id IN (SELECT upload_id FROM dataset_uploads WHERE relation=?)", (relation,)) + conn.execute("DELETE FROM dataset_uploads WHERE relation=?", (relation,)) + return cursor.rowcount > 0 + + @staticmethod + def _validate_checksum(artifact_checksum: str) -> None: + if re.fullmatch(r"[0-9a-f]{32}", artifact_checksum) is None: + raise QDBError("INVALID_DATASET", "artifact_checksum must be a lowercase MD5 hex digest") + + @staticmethod + def _validate_rows(rows: list[dict[str, Any]]) -> None: + if not isinstance(rows, list) or not all(isinstance(row, dict) and "key" in row and isinstance(row.get("values"), dict) for row in rows): + raise QDBError("INVALID_DATASET", "rows must contain {key, values} objects") + + @staticmethod + def _replace_rows(conn: sqlite3.Connection, relation: str, rows: list[dict[str, Any]]) -> None: + conn.execute("DELETE FROM dataset_rows WHERE relation=?", (relation,)) + conn.executemany("INSERT INTO dataset_rows(relation,ordinal,key_json,values_json) VALUES(?,?,?,?)", + [(relation, ordinal, json.dumps(item["key"], separators=(",", ":"), ensure_ascii=False), + json.dumps(item["values"], separators=(",", ":"), ensure_ascii=False)) + for ordinal, item in enumerate(rows)]) diff --git a/systems/quantum_db/qdb/large_scale.py b/systems/quantum_db/qdb/large_scale.py new file mode 100644 index 0000000..a447360 --- /dev/null +++ b/systems/quantum_db/qdb/large_scale.py @@ -0,0 +1,199 @@ +"""Reproducible streaming benchmarks for Quantum DB data-plane operators. + +The source rows are generated deterministically and scanned once. This makes +million-row traversal practical without pretending that a Python list is a +database, while a bounded reservoir feeds the Qute and TN operators. +""" +from __future__ import annotations + +import json +import math +import random +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig +from qute.api.query_request import ExecutionPolicy, QueryRequest +from qute.execution.engine import QuteEngine +from qute.quantum.backend.ideal_simulator import IdealSimulatorBackend + +_MASK64 = (1 << 64) - 1 + + +def _mix64(value: int) -> int: + value = (value + 0x9E3779B97F4A7C15) & _MASK64 + value = ((value ^ (value >> 30)) * 0xBF58476D1CE4E5B9) & _MASK64 + value = ((value ^ (value >> 27)) * 0x94D049BB133111EB) & _MASK64 + return value ^ (value >> 31) + + +@dataclass(frozen=True, slots=True) +class BenchmarkConfig: + sizes: tuple[int, ...] = (10_000, 100_000, 1_000_000) + seed: int = 7 + reservoir_size: int = 8_192 + tn_train_samples: int = 2_048 + tn_draws: int = 1_024 + hll_precision: int = 12 + + +class HyperLogLog: + """Small HLL implementation used only for an explicit distinct-cardinality baseline.""" + + def __init__(self, precision: int) -> None: + if not 4 <= precision <= 18: + raise ValueError("hll precision must be in [4, 18]") + self.precision = precision + self.width = 1 << precision + self.registers = [0] * self.width + + def add(self, value: int) -> None: + hashed = _mix64(value) + index = hashed & (self.width - 1) + tail = hashed >> self.precision + rank = (64 - self.precision + 1) if tail == 0 else (64 - self.precision - tail.bit_length() + 1) + if rank > self.registers[index]: + self.registers[index] = rank + + def estimate(self) -> float: + m = self.width + alpha = 0.673 if m == 16 else 0.697 if m == 32 else 0.709 if m == 64 else 0.7213 / (1.0 + 1.079 / m) + raw = alpha * m * m / sum(2.0 ** -register for register in self.registers) + zeroes = self.registers.count(0) + return m * math.log(m / zeroes) if raw <= 2.5 * m and zeroes else raw + + +def synthetic_rows(size: int, seed: int) -> Iterable[tuple[int, int, int, int]]: + """Yield ``(segment, value, region, customer_id)`` without materializing a table.""" + customer_domain = max(1, (size * 3) // 5) + for row_id in range(size): + mixed = _mix64(row_id ^ seed) + segment = mixed & 31 + value = (mixed >> 8) % 10_000 + region = (mixed >> 24) & 7 + customer_id = _mix64(row_id * 3 + seed) % customer_domain + yield int(segment), int(value), int(region), int(customer_id) + + +def matches_predicate(row: tuple[int, int, int, int]) -> bool: + """A selective two-column predicate, roughly 3.125% in expectation.""" + return row[0] < 2 and row[1] < 5_000 + + +def wilson_interval(successes: int, total: int) -> tuple[float, float]: + if total <= 0: + return (0.0, 1.0) + z = 1.959963984540054 + p = successes / total + denominator = 1.0 + z * z / total + centre = (p + z * z / (2.0 * total)) / denominator + margin = z * math.sqrt(p * (1.0 - p) / total + z * z / (4.0 * total * total)) / denominator + return (max(0.0, centre - margin), min(1.0, centre + margin)) + + +def _sample_to_qute_rows(rows: list[tuple[int, int, int, int]]) -> list[dict[str, int]]: + return [{"segment": row[0], "value": row[1], "region": row[2], "customer_id": row[3]} for row in rows] + + +def _run_one(size: int, config: BenchmarkConfig) -> dict[str, Any]: + if size <= 0: + raise ValueError("dataset sizes must be positive") + reservoir_limit = max(config.reservoir_size, config.tn_train_samples) + rng = random.Random(config.seed + size) + reservoir: list[tuple[int, int, int, int]] = [] + exact_predicate_count = 0 + exact_distinct: set[int] = set() + hll = HyperLogLog(config.hll_precision) + + start = time.perf_counter() + for offset, row in enumerate(synthetic_rows(size, config.seed), start=1): + if matches_predicate(row): + exact_predicate_count += 1 + exact_distinct.add(row[3]) + hll.add(row[3]) + if len(reservoir) < reservoir_limit: + reservoir.append(row) + else: + replacement = rng.randrange(offset) + if replacement < reservoir_limit: + reservoir[replacement] = row + traversal_ms = (time.perf_counter() - start) * 1000.0 + + sample = reservoir[: min(config.reservoir_size, len(reservoir))] + sample_hits = sum(matches_predicate(row) for row in sample) + sample_selectivity = sample_hits / max(1, len(sample)) + predicate_estimate = sample_selectivity * size + low, high = wilson_interval(sample_hits, len(sample)) + + qute = QuteEngine(IdealSimulatorBackend(max_qubits=16)) + qute_result = qute.execute(QueryRequest( + query_id=f"large-scale-{size}", source="synthetic_stream", rows=_sample_to_qute_rows(sample), + filters=[{"column": "segment", "op": "<", "value": 2}, {"column": "value", "op": "<", "value": 5_000}], + aggregate={"type": "count"}, policy=ExecutionPolicy.C0_CLASSICAL, + data_version=f"synthetic-{size}-{config.seed}", random_seed=config.seed, + )).to_dict() + qute_sample_count = int(qute_result["value"]) + + tn_rows = [list(map(float, row[:3])) for row in reservoir[: min(config.tn_train_samples, len(reservoir))]] + tn_service = TNWorkloadService(config=WorkloadConfig( + num_sites=4, phys_dim=8, chi_max=16, keep_ratio=0.5, alu_components=("wloc",), + ), seed=config.seed) + tn_start = time.perf_counter() + tn_page = tn_service.create_page(dataset=f"large-scale-{size}", samples=tn_rows) + tn_build_ms = (time.perf_counter() - tn_start) * 1000.0 + tn_draws = tn_service.sampling(tn_page, num_samples=config.tn_draws) + tn_fidelity = tn_service.fidelity(tn_rows, tn_draws) + tn_reconstruction_l1 = tn_service.reconstruction_error_l1(tn_rows, tn_draws) + + exact_distinct_count = len(exact_distinct) + hll_estimate = hll.estimate() + return { + "dataset_size": size, + "traversal": { + "operator": "streaming_exact_scan", "rows": size, "predicate_matches": exact_predicate_count, + "elapsed_ms": traversal_ms, "rows_per_second": size / max(traversal_ms / 1000.0, 1e-12), + }, + "cardinality": { + "predicate_exact": exact_predicate_count, "predicate_sample_estimate": predicate_estimate, + "predicate_relative_error": abs(predicate_estimate - exact_predicate_count) / max(1, exact_predicate_count), + "predicate_wilson_95": [low * size, high * size], "sample_rows": len(sample), + "qute_c0_sample_count": qute_sample_count, + "qute_c0_scaled_estimate": qute_sample_count * size / max(1, len(sample)), + "distinct_exact": exact_distinct_count, "distinct_hll_estimate": hll_estimate, + "distinct_hll_relative_error": abs(hll_estimate - exact_distinct_count) / max(1, exact_distinct_count), + "hll_precision": config.hll_precision, + }, + "sampling": { + "operator": "uniform_reservoir_then_tn_page_sample", "reservoir_rows": len(reservoir), + "predicate_sample_hits": sample_hits, "tn_train_rows": len(tn_rows), "tn_draw_requests": config.tn_draws, + "tn_draw_count": len(tn_draws), "tn_build_ms": tn_build_ms, + "tn_marginal_fidelity_proxy": tn_fidelity, "tn_reconstruction_error_l1": tn_reconstruction_l1, + "tn_page": tn_page.summary(), + }, + } + + +def run_large_scale_benchmark(config: BenchmarkConfig) -> dict[str, Any]: + started = time.time() + results = [_run_one(size, config) for size in config.sizes] + return {"benchmark": "qdb-large-scale-v1", "started_at_epoch_s": started, "config": asdict(config), "results": results} + + +def write_report(report: dict[str, Any], output_dir: Path) -> tuple[Path, Path]: + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "large_scale_report.json" + markdown_path = output_dir / "large_scale_report.md" + json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + lines = ["# Quantum DB large-scale operator benchmark", "", "| Rows | Scan rows/s | Exact predicate | Sample estimate | Predicate rel. error | Exact distinct | HLL rel. error | TN fidelity proxy |", "|---:|---:|---:|---:|---:|---:|---:|---:|"] + for item in report["results"]: + traversal, cardinality, sampling = item["traversal"], item["cardinality"], item["sampling"] + lines.append( + f"| {item['dataset_size']:,} | {traversal['rows_per_second']:.0f} | {cardinality['predicate_exact']} | " + f"{cardinality['predicate_sample_estimate']:.1f} | {cardinality['predicate_relative_error']:.4f} | " + f"{cardinality['distinct_exact']} | {cardinality['distinct_hll_relative_error']:.4f} | {sampling['tn_marginal_fidelity_proxy']:.4f} |" + ) + lines.extend(["", "All predicate estimates are sampling estimates; PostgreSQL-equivalent exact scan counts remain the authority."]) + markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return json_path, markdown_path diff --git a/systems/quantum_db/qdb/protocol.py b/systems/quantum_db/qdb/protocol.py new file mode 100644 index 0000000..f15a26d --- /dev/null +++ b/systems/quantum_db/qdb/protocol.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +PROTOCOL_VERSION = "1.0" +MAX_REQUEST_BYTES = 1_048_576 + + +class JobStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + TIMED_OUT = "timed_out" + + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED, JobStatus.TIMED_OUT} + + +class QDBError(RuntimeError): + def __init__(self, code: str, message: str, *, retryable: bool = False, data: dict[str, Any] | None = None) -> None: + super().__init__(message) + self.code = code + self.retryable = retryable + self.data = data or {} + + def as_rpc_error(self) -> dict[str, Any]: + return {"code": self.code, "message": str(self), "retryable": self.retryable, "data": self.data} + + +@dataclass(frozen=True, slots=True) +class RpcRequest: + request_id: str + method: str + params: dict[str, Any] + version: str = PROTOCOL_VERSION + + @classmethod + def parse(cls, value: Any) -> "RpcRequest": + if not isinstance(value, dict): + raise QDBError("INVALID_REQUEST", "request must be a JSON object") + version = str(value.get("version", "")) + if version != PROTOCOL_VERSION: + raise QDBError("UNSUPPORTED_VERSION", f"unsupported protocol version: {version!r}") + method = str(value.get("method", "")).strip() + params = value.get("params", {}) + if not method or not isinstance(params, dict): + raise QDBError("INVALID_REQUEST", "method and object params are required") + return cls(str(value.get("id", "")), method, params, version) diff --git a/systems/quantum_db/qdb/relational_ir.py b/systems/quantum_db/qdb/relational_ir.py new file mode 100644 index 0000000..3254413 --- /dev/null +++ b/systems/quantum_db/qdb/relational_ir.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass, field +from typing import Any + +_HINT = re.compile(r"/\*\+\s*QDB_(AUTO|QUANTUM|TN|HYBRID|CLASSICAL|SYNC)(?:\(timeout_ms=(\d+)\))?\s*\*/", re.I) +_FROM = re.compile(r"\bFROM\s+((?:\"(?:[^\"]|\"\")+\"|[A-Za-z_][\w$]*)(?:\s*\.\s*(?:\"(?:[^\"]|\"\")+\"|[A-Za-z_][\w$]*))?)", re.I) +_WHERE = re.compile(r"\bWHERE\b(?P.*?)(?=\b(?:GROUP\s+BY|ORDER\s+BY|LIMIT|OFFSET|HAVING|UNION|INTERSECT|EXCEPT|FOR)\b|;|$)", re.I | re.S) +_PREDICATE = re.compile( + r"^\s*(?P(?:\"(?:[^\"]|\"\")+\"|[A-Za-z_][\w$]*)(?:\s*\.\s*(?:\"(?:[^\"]|\"\")+\"|[A-Za-z_][\w$]*))?)\s*" + r"(?P=|<>|!=|<=|>=|<|>)\s*(?P'(?:[^']|'')*'|[-+]?\d+(?:\.\d+)?|TRUE|FALSE|NULL)\s*$", + re.I, +) + + +@dataclass(slots=True) +class Predicate: + column: str + operator: str + value: Any + scalar_type: str = "" + + +@dataclass(slots=True) +class RelationalIR: + query_id: str + sql: str + relation: str + snapshot_version: str + predicates: list[Predicate] = field(default_factory=list) + joins: list[str] = field(default_factory=list) + aggregates: list[str] = field(default_factory=list) + hint: str = "auto" + timeout_ms: int | None = None + estimated_rows: int | None = None + estimated_selectivity: float | None = None + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +def parse_hint(sql: str) -> tuple[str, int | None]: + hint, timeout = "auto", None + for match in _HINT.finditer(sql): + name = match.group(1).lower() + if name == "sync": + timeout = int(match.group(2) or 30_000) + else: + hint = name + return hint, timeout + + +def build_relational_ir(params: dict[str, Any]) -> RelationalIR: + sql = str(params.get("sql", "")).strip() + if not sql: + raise ValueError("sql is required") + hint, parsed_timeout = parse_hint(sql) + supplied_predicates = params.get("predicates", []) + if not isinstance(supplied_predicates, list): + raise ValueError("predicates must be an array") + predicates = [Predicate(str(p["column"]), str(p.get("operator", "=")), p.get("value"), str(p.get("scalar_type", ""))) for p in supplied_predicates] + inferred_relation, inferred_predicates, joins, aggregates = _infer_sql_shape(sql) + if not predicates: + predicates = inferred_predicates + return RelationalIR( + query_id=str(params.get("query_id", "")), sql=sql, + relation=str(params.get("relation", inferred_relation)), snapshot_version=str(params.get("snapshot_version", "")), + predicates=predicates, joins=[str(v) for v in params.get("joins", joins)], + aggregates=[str(v) for v in params.get("aggregates", aggregates)], + hint=str(params.get("hint", hint)).lower(), timeout_ms=params.get("timeout_ms", parsed_timeout), + estimated_rows=_optional_int(params.get("estimated_rows")), + estimated_selectivity=_optional_float(params.get("estimated_selectivity")), + ) + + +def _optional_int(value: Any) -> int | None: + return None if value is None else int(value) + + +def _optional_float(value: Any) -> float | None: + return None if value is None else float(value) + + +def _infer_sql_shape(sql: str) -> tuple[str, list[Predicate], list[str], list[str]]: + """Extract only the simple conjunctive subset that can safely be advisory. + + PostgreSQL remains the parser and the final authority. This deliberately ignores + unsupported expressions (OR, functions, subqueries, casts, and non-literal values) + instead of guessing a candidate set that could be unsound. + """ + from_match = _FROM.search(sql) + relation = _normalise_identifier(from_match.group(1)) if from_match else "" + predicates: list[Predicate] = [] + where_match = _WHERE.search(sql) + if where_match: + body = where_match.group("body") + if not re.search(r"\bOR\b", body, re.I): + for term in _split_and(body): + match = _PREDICATE.match(term) + if match is None: + predicates = [] + break + predicates.append(Predicate( + _normalise_identifier(match.group("column")).rsplit(".", 1)[-1], + match.group("operator"), + _parse_literal(match.group("value")), + )) + upper = sql.upper() + joins = ["join"] if re.search(r"\bJOIN\b", upper) else [] + aggregates = [] + if re.search(r"\bCOUNT\s*\(", upper): + aggregates.append("count") + if re.search(r"\bGROUP\s+BY\b", upper): + aggregates.append("group_by") + return relation, predicates, joins, aggregates + + +def _split_and(body: str) -> list[str]: + """Split a WHERE clause on unquoted AND tokens for the supported SQL subset.""" + pieces: list[str] = [] + start = 0 + quoted = False + index = 0 + while index < len(body): + if body[index] == "'": + if quoted and index + 1 < len(body) and body[index + 1] == "'": + index += 2 + continue + quoted = not quoted + index += 1 + continue + if not quoted and body[index:index + 3].upper() == "AND": + before = body[index - 1] if index else " " + after = body[index + 3] if index + 3 < len(body) else " " + if not (before.isalnum() or before in "_$") and not (after.isalnum() or after in "_$"): + pieces.append(body[start:index]) + start = index + 3 + index += 3 + continue + index += 1 + pieces.append(body[start:]) + return pieces + + +def _normalise_identifier(value: str) -> str: + return ".".join(part.strip().strip('"').replace('""', '"') for part in value.split(".")) + + +def _parse_literal(value: str) -> Any: + upper = value.upper() + if upper == "NULL": + return None + if upper == "TRUE": + return True + if upper == "FALSE": + return False + if value.startswith("'"): + return value[1:-1].replace("''", "'") + return float(value) if "." in value else int(value) diff --git a/systems/quantum_db/qdb/router.py b/systems/quantum_db/qdb/router.py new file mode 100644 index 0000000..8e30ada --- /dev/null +++ b/systems/quantum_db/qdb/router.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass + +from .relational_ir import RelationalIR + + +@dataclass(frozen=True, slots=True) +class RouteDecision: + backend: str + reason: str + fallback_backend: str = "classical" + requires_recheck: bool = True + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +class QueryRouter: + def __init__(self, *, max_qubits: int = 180, max_depth: int = 500, quantum_selectivity: float = 0.15, tn_selectivity: float = 0.45) -> None: + self.max_qubits = max_qubits + self.max_depth = max_depth + self.quantum_selectivity = quantum_selectivity + self.tn_selectivity = tn_selectivity + + def decide(self, ir: RelationalIR, *, qubits: int = 0, depth: int = 0, queue_depth: int = 0) -> RouteDecision: + hint = ir.hint + if hint == "classical": + return RouteDecision("classical", "forced by QDB_CLASSICAL") + if qubits > self.max_qubits: + return RouteDecision("classical", "qubit requirement exceeds backend capacity") + if depth > self.max_depth: + return RouteDecision("classical", "circuit depth exceeds backend limit") + if queue_depth > 32 and hint not in {"quantum"}: + return RouteDecision("tn", "quantum queue is congested") + if hint == "quantum": + return RouteDecision("quantum", "forced by QDB_QUANTUM") + if hint == "tn": + return RouteDecision("tn", "forced by QDB_TN") + if hint == "hybrid": + return RouteDecision("hybrid", "forced by QDB_HYBRID") + selectivity = ir.estimated_selectivity + if selectivity is None or not ir.predicates: + return RouteDecision("classical", "missing selective predicate or statistics") + if selectivity <= self.quantum_selectivity: + return RouteDecision("quantum", "low-selectivity equality candidate") + if selectivity <= self.tn_selectivity and len(ir.predicates) > 1: + return RouteDecision("hybrid", "multi-attribute selective predicate") + if selectivity <= self.tn_selectivity: + return RouteDecision("tn", "TN selectivity estimation is cost effective") + return RouteDecision("classical", "classical scan has lower expected cost") diff --git a/systems/quantum_db/qdb/server.py b/systems/quantum_db/qdb/server.py new file mode 100644 index 0000000..370369e --- /dev/null +++ b/systems/quantum_db/qdb/server.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path +from typing import Any + +from data_plane.quantum.backend import BackendConfig + +from .protocol import MAX_REQUEST_BYTES, PROTOCOL_VERSION, QDBError, RpcRequest +from .service import QuantumDatabaseService + + +class UnixRpcServer: + def __init__(self, socket_path: str, service: QuantumDatabaseService) -> None: + self.socket_path = socket_path + self.service = service + + async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + request_id = "" + try: + raw = await reader.readline() + if not raw or len(raw) > MAX_REQUEST_BYTES: + raise QDBError("REQUEST_TOO_LARGE", "request exceeds size limit") + request = RpcRequest.parse(json.loads(raw)) + request_id = request.request_id + result = await asyncio.to_thread(self.service.dispatch, request.method, request.params) + response: dict[str, Any] = {"version": PROTOCOL_VERSION, "id": request_id, "result": result} + except QDBError as exc: + response = {"version": PROTOCOL_VERSION, "id": request_id, "error": exc.as_rpc_error()} + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + response = {"version": PROTOCOL_VERSION, "id": request_id, "error": {"code": "INVALID_JSON", "message": str(exc), "retryable": False}} + except Exception as exc: + response = {"version": PROTOCOL_VERSION, "id": request_id, "error": {"code": "INTERNAL_ERROR", "message": f"{type(exc).__name__}: {exc}", "retryable": False}} + encoded = json.dumps(response, ensure_ascii=False, separators=(",", ":")).encode() + b"\n" + if len(encoded) > MAX_REQUEST_BYTES: + encoded = json.dumps({ + "version": PROTOCOL_VERSION, "id": request_id, + "error": {"code": "RESPONSE_TOO_LARGE", "message": "response exceeds size limit", "retryable": True}, + }, separators=(",", ":")).encode() + b"\n" + writer.write(encoded) + await writer.drain() + writer.close() + await writer.wait_closed() + + async def run(self) -> None: + path = Path(self.socket_path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + path.unlink() + server = await asyncio.start_unix_server(self.handle, path=self.socket_path, limit=MAX_REQUEST_BYTES + 1) + os.chmod(self.socket_path, 0o600) + async with server: + await server.serve_forever() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Quantum DB local sidecar") + parser.add_argument("--socket", default=os.getenv("QDB_SOCKET", "/tmp/qdb.sock")) + parser.add_argument("--state", default=os.getenv("QDB_STATE", "outputs/qdb_jobs.sqlite3")) + parser.add_argument("--backend", default=os.getenv("QDB_BACKEND", "qpanda3-sim")) + parser.add_argument("--qubits", type=int, default=int(os.getenv("QDB_QUBITS", "180"))) + args = parser.parse_args(argv) + token = os.getenv("ORIGINQC_API_KEY", "") + backend = BackendConfig(backend_name=args.backend, qcloud_backend_name=os.getenv("QDB_QCLOUD_BACKEND", "WK_C180"), api_token=token, num_qubits=args.qubits, allow_local_fallback=not bool(token)) + service = QuantumDatabaseService(state_path=args.state, backend_config=backend) + try: + asyncio.run(UnixRpcServer(args.socket, service).run()) + except KeyboardInterrupt: + return 0 + finally: + service.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/qdb/service.py b/systems/quantum_db/qdb/service.py new file mode 100644 index 0000000..01cf171 --- /dev/null +++ b/systems/quantum_db/qdb/service.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import os +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from typing import Any + +from data_plane.quantum.backend import BackendConfig +from data_plane.quantum.executor import QuantumExecutor +from qute.api.query_request import ExecutionPolicy, OptimizationStrategy, QueryRequest +from qute.artifacts.artifact_store import ArtifactStore +from qute.execution.engine import QuteEngine +from qute.quantum.backend.ideal_simulator import IdealSimulatorBackend + +from .jobs import JobStore +from .candidate_executor import CandidateExecutor +from .protocol import JobStatus, QDBError +from .relational_ir import RelationalIR, build_relational_ir +from .router import QueryRouter + + +class QuantumDatabaseService: + """Versioned sidecar facade. Quantum/TN outputs are candidates, never final SQL rows.""" + + def __init__(self, *, state_path: str, backend_config: BackendConfig | None = None, workers: int = 2) -> None: + self.jobs = JobStore(state_path) + self.router = QueryRouter(max_qubits=(backend_config or BackendConfig()).num_qubits) + self.quantum = QuantumExecutor(backend_config=backend_config) + artifact_path = str(state_path) + ".artifacts.json" + self.qute = QuteEngine(IdealSimulatorBackend(max_qubits=(backend_config or BackendConfig()).num_qubits), + artifact_store=ArtifactStore(artifact_path)) + self.candidate_executor = CandidateExecutor(max_qubits=(backend_config or BackendConfig()).num_qubits) + self.pool = ThreadPoolExecutor(max_workers=max(1, workers), thread_name_prefix="qdb-job") + self._futures: dict[str, Any] = {} + self._lock = threading.RLock() + + def close(self) -> None: + # Do not leave a worker writing the durable SQLite state after the server has + # released its resources. Queued work is cancelled; an already-running task is + # allowed to finish its current safe fallback/record transition. + with self._lock: + for future in self._futures.values(): + future.cancel() + self.pool.shutdown(wait=True, cancel_futures=True) + + def dispatch(self, method: str, params: dict[str, Any]) -> Any: + handlers = { + "health": self.health, "explain": self.explain, "candidates": self.candidates, + "submit": self.submit, "status": self.status, "result": self.result, "cancel": self.cancel, + "hardware.preflight": self.hardware_preflight, + "query.execute": self.execute_query, "dataset.register": self.register_dataset, + "dataset.unregister": self.unregister_dataset, "dataset.begin": self.begin_dataset_upload, + "dataset.append": self.append_dataset_page, "dataset.commit": self.commit_dataset_upload, + } + handler = handlers.get(method) + if handler is None: + raise QDBError("METHOD_NOT_FOUND", f"unknown method: {method}") + return handler(params) + + def health(self, _: dict[str, Any]) -> dict[str, Any]: + config = self.quantum.adapter.config + return {"status": "healthy", "backend": config.backend_name, "pid": os.getpid(), + "hardware_backend": config.qcloud_backend_name, "hardware_configured": bool(config.api_token), + "hardware_sdk_available": importlib.util.find_spec("pyqpanda3") is not None} + + def hardware_preflight(self, params: dict[str, Any]) -> dict[str, Any]: + """Perform an explicitly authorized, non-submitting QCloud capability check.""" + config = self.quantum.adapter.config + if not params.get("allow_hardware"): + raise QDBError("HARDWARE_NOT_AUTHORIZED", "set allow_hardware=true to request a hardware preflight") + if not config.api_token: + raise QDBError("HARDWARE_NOT_CONFIGURED", "ORIGINQC_API_KEY is not configured in the sidecar environment") + if importlib.util.find_spec("pyqpanda3") is None: + raise QDBError("HARDWARE_SDK_UNAVAILABLE", "pyqpanda3 is not importable in the sidecar environment", retryable=True) + ir = params.get("quantum_ir") + if not isinstance(ir, dict): + ir = { + "ir_version": "qdb-preflight-v1", "model_id": "qdb-preflight", + "operations": [{"gate": "u3", "qubits": [0], "params": [0.0, 0.0, 0.0]}, + {"gate": "measure", "qubits": [0], "cbit": 0}], + } + trace_id = str(params.get("trace_id") or uuid.uuid4()) + try: + report = self.quantum.execute({"action": "preflight", "payload": {"ir": ir}}, {"trace_id": trace_id})["result"] + return {"ok": report.ok, "warnings": list(report.warnings), "route_edges": list(report.route_edges), + "backend": config.qcloud_backend_name, "trace_id": trace_id} + except Exception as exc: + raise QDBError("HARDWARE_PREFLIGHT_FAILED", f"{type(exc).__name__}: {exc}", retryable=True) from exc + + def register_dataset(self, params: dict[str, Any]) -> dict[str, Any]: + rows = params.get("rows") + if not isinstance(rows, list) or not all(isinstance(row, dict) and "key" in row and isinstance(row.get("values"), dict) for row in rows): + raise QDBError("INVALID_DATASET", "rows must contain {key, values} objects") + return self.jobs.register_dataset( + relation=str(params.get("relation", "")), snapshot_version=str(params.get("snapshot_version", "")), + key_column=str(params.get("key_column", "")), artifact_checksum=str(params.get("artifact_checksum", "")), rows=rows, + ) + + def unregister_dataset(self, params: dict[str, Any]) -> dict[str, Any]: + relation = str(params.get("relation", "")) + if not relation: + raise QDBError("INVALID_DATASET", "relation is required") + return {"relation": relation, "removed": self.jobs.unregister_dataset(relation)} + + def begin_dataset_upload(self, params: dict[str, Any]) -> dict[str, Any]: + return self.jobs.begin_dataset_upload( + relation=str(params.get("relation", "")), snapshot_version=str(params.get("snapshot_version", "")), + key_column=str(params.get("key_column", "")), artifact_checksum=str(params.get("artifact_checksum", "")), + expected_rows=int(params.get("expected_rows", -1)), + ) + + def append_dataset_page(self, params: dict[str, Any]) -> dict[str, Any]: + rows = params.get("rows") + if not isinstance(rows, list): + raise QDBError("INVALID_DATASET", "rows must be an array") + return self.jobs.append_dataset_page(upload_id=str(params.get("upload_id", "")), page=int(params.get("page", -1)), rows=rows) + + def commit_dataset_upload(self, params: dict[str, Any]) -> dict[str, Any]: + return self.jobs.commit_dataset_upload(upload_id=str(params.get("upload_id", ""))) + + def explain(self, params: dict[str, Any]) -> dict[str, Any]: + ir = self._prepare_ir(params) + decision = self.router.decide(ir, qubits=int(params.get("qubits", 0)), depth=int(params.get("depth", 0)), queue_depth=self._queue_depth()) + return {"ir": ir.as_dict(), "route": decision.as_dict(), "contract": {"candidate_only": True, "postgres_recheck_required": True}} + + def candidates(self, params: dict[str, Any]) -> dict[str, Any]: + ir = self._prepare_ir(params) + # The extension may pass pre-indexed [key, {column:value}] records. This deterministic + # implementation is also the correctness fallback when a quantum result is unavailable. + rows, dataset = self._candidate_rows(params, ir) + route = self.router.decide(ir, qubits=int(params.get("qubits", 0)), depth=int(params.get("depth", 0)), queue_depth=self._queue_depth()) + keys: list[Any] = [] + if isinstance(rows, list): + for item in rows: + if not isinstance(item, dict) or "key" not in item or not isinstance(item.get("values"), dict): + continue + if all(self._matches(item["values"].get(p.column), p.operator, p.value) for p in ir.predicates): + keys.append(item["key"]) + expected_checksum = str(params.get("artifact_checksum", "")) + complete = bool( + dataset is not None + and expected_checksum + and dataset["artifact_checksum"] == expected_checksum + and dataset["snapshot_version"] == ir.snapshot_version + and self._is_pg_scalar_equality_candidate(params, ir, rows) + ) + return { + "candidate_keys": keys, + "route": route.as_dict(), + "requires_recheck": True, + "snapshot_version": ir.snapshot_version, + "candidate_contract": { + "complete": complete, + "provenance": "classical_snapshot_filter" if complete else "advisory", + "candidate_set": "complete_superset" if complete else "unknown", + "semantic_scope": "pg_builtin_scalar_equality_v1" if complete else "", + "snapshot_version": dataset["snapshot_version"] if dataset is not None else "", + "artifact_checksum": dataset["artifact_checksum"] if dataset is not None else "", + }, + } + + def execute_query(self, params: dict[str, Any]) -> dict[str, Any]: + try: + policy = ExecutionPolicy(str(params.get("policy", "C3"))) + strategy = OptimizationStrategy(str(params.get("optimization_strategy", "min-cost-under-quality-constraint"))) + except ValueError as exc: + raise QDBError("INVALID_POLICY", str(exc)) from exc + rows = params.get("rows", []) + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + raise QDBError("INVALID_REQUEST", "rows must be an array of objects") + request = QueryRequest( + query_id=str(params.get("query_id") or uuid.uuid4()), source=str(params.get("source", "")), rows=rows, + filters=list(params.get("filters", [])), group_by=list(params.get("group_by", [])), + aggregate=dict(params.get("aggregate", {"type": "count"})), + quality_budget=dict(params.get("quality_budget", {})) or QueryRequest.__dataclass_fields__["quality_budget"].default_factory(), + resource_budget=dict(params.get("resource_budget", {})) or QueryRequest.__dataclass_fields__["resource_budget"].default_factory(), + data_version=str(params.get("data_version", "v1")), encoding_version=str(params.get("encoding_version", "native-v1")), + policy=policy, shots=int(params.get("shots", 1024)), random_seed=int(params.get("random_seed", 7)), + optimization_strategy=strategy, + force_backend=params.get("force_backend"), metadata=dict(params.get("metadata", {})), + ) + return self.qute.execute(request).to_dict() + + def submit(self, params: dict[str, Any]) -> dict[str, Any]: + explanation = self.explain(params) + trace_id = str(params.get("trace_id") or uuid.uuid4()) + job_id = self.jobs.create(params, backend=str(explanation["route"]["backend"]), trace_id=trace_id) + future = self.pool.submit(self._run_job, job_id, params) + with self._lock: + self._futures[job_id] = future + timeout_ms = params.get("timeout_ms") + if params.get("wait") and timeout_ms is not None: + try: + future.result(timeout=max(0, int(timeout_ms)) / 1000) + except TimeoutError: + return {"job_id": job_id, "status": self.jobs.get(job_id)["status"], "timed_out_waiting": True} + return {"job_id": job_id, "status": self.jobs.get(job_id)["status"], "trace_id": trace_id} + + def status(self, params: dict[str, Any]) -> dict[str, Any]: + record = self.jobs.get(str(params.get("job_id", ""))) + return {k: record[k] for k in ("job_id", "status", "backend", "trace_id", "error_code", "error_message", "created_at", "updated_at")} + + def result(self, params: dict[str, Any]) -> dict[str, Any]: + record = self.jobs.get(str(params.get("job_id", ""))) + if record["status"] != JobStatus.SUCCEEDED: + raise QDBError("JOB_NOT_COMPLETE", f"job status is {record['status']}", retryable=record["status"] in {JobStatus.QUEUED, JobStatus.RUNNING}) + return {"job_id": record["job_id"], "result": record["result"], "requires_recheck": True} + + def cancel(self, params: dict[str, Any]) -> dict[str, Any]: + job_id = str(params.get("job_id", "")) + cancelled = self.jobs.cancel(job_id) + with self._lock: + future = self._futures.get(job_id) + if cancelled and future is not None: + future.cancel() + return {"job_id": job_id, "cancelled": cancelled} + + def _run_job(self, job_id: str, params: dict[str, Any]) -> None: + if self.jobs.get(job_id)["status"] == JobStatus.CANCELLED: + return + if not self.jobs.update(job_id, JobStatus.RUNNING, preserve_cancelled=True): + return + try: + hardware_grover_iterations = self._hardware_grover_iterations(params) if params.get("run_qpanda_probe") else None + result = self.candidates(params) + rows, _ = self._candidate_rows(params, self._prepare_ir(params)) + # Run a bounded real TN page build and/or ideal Grover-style measurement + # probe for asynchronous work. Its samples are telemetry only: candidate + # keys remain the classical immutable-snapshot superset above. + result["accelerator"] = self.candidate_executor.execute( + route=str(result["route"]["backend"]), rows=rows, + candidate_keys=list(result["candidate_keys"]), shots=int(params.get("shots", 256)), + hardware_grover_iterations=hardware_grover_iterations, + ) + if params.get("run_qute_validation"): + # Qute receives a flattened immutable snapshot and performs an + # exact C0 validation. It is deliberately separate from the + # candidate set: PostgreSQL remains the final SQL authority. + result["qute_validation"] = self._run_qute_validation(params, self._prepare_ir(params), rows) + component_trace = self._probe_component_trace(params, result) + if params.get("run_qpanda_probe") and isinstance(result["accelerator"].get("quantum"), dict): + probe_ir = result["accelerator"]["quantum"].get("qpanda3_ir") + if not isinstance(probe_ir, dict): + result["accelerator"]["qpanda3"] = { + "executed": False, + "reason": "hardware_probe_unsupported: requires one marked address across exactly eight rows", + } + else: + if self._requires_hardware_authorization() and not params.get("allow_hardware"): + result["accelerator"]["qpanda3"] = { + "executed": False, + "reason": "HARDWARE_NOT_AUTHORIZED: set allow_hardware=true to submit a real-hardware probe", + } + else: + trace_id = self.jobs.get(job_id)["trace_id"] + try: + preflight = self.quantum.execute({"action": "preflight", "payload": {"ir": probe_ir}}, {"trace_id": trace_id})["result"] + if not preflight.ok: + raise QDBError("PREFLIGHT_FAILED", "; ".join(preflight.fatal_errors)) + compiled = self.quantum.execute({"action": "compile", "payload": {"ir": probe_ir}}, {"trace_id": trace_id})["result"] + run = self.quantum.execute({"action": "run", "payload": {"compiled": compiled, "shots": int(params.get("shots", 256)), + "wait_for_result": bool(params.get("wait_for_hardware_result", False))}}, {"trace_id": trace_id})["result"] + metadata = self._safe_metadata(run.metadata) + metadata["execution_components"] = component_trace + result["accelerator"]["qpanda3"] = {"executed": True, "counts": run.counts, "shots": run.shots, + "metadata": metadata} + except Exception as exc: + result["accelerator"]["qpanda3"] = {"executed": False, "reason": f"{type(exc).__name__}: {exc}"} + # Optional hardware IR path. It is never used to bypass PostgreSQL recheck. + if isinstance(params.get("quantum_ir"), dict) and result["route"]["backend"] == "quantum": + if self._requires_hardware_authorization() and not params.get("allow_hardware"): + result["quantum"] = {"executed": False, + "reason": "HARDWARE_NOT_AUTHORIZED: set allow_hardware=true to submit a real-hardware circuit"} + else: + trace_id = self.jobs.get(job_id)["trace_id"] + preflight = self.quantum.execute({"action": "preflight", "payload": {"ir": params["quantum_ir"]}}, {"trace_id": trace_id}) + if not preflight["result"].ok: + raise QDBError("PREFLIGHT_FAILED", "; ".join(preflight["result"].fatal_errors)) + compiled = self.quantum.execute({"action": "compile", "payload": {"ir": params["quantum_ir"]}}, {"trace_id": trace_id})["result"] + run = self.quantum.execute({"action": "run", "payload": {"compiled": compiled, "shots": int(params.get("shots", 1024)), + "wait_for_result": bool(params.get("wait_for_hardware_result", False))}}, {"trace_id": trace_id})["result"] + result["quantum"] = {"executed": True, "counts": run.counts, "shots": run.shots, + "metadata": self._safe_metadata(run.metadata)} + result["result_digest"] = hashlib.sha256(repr(result.get("candidate_keys", [])).encode()).hexdigest() + self.jobs.update(job_id, JobStatus.SUCCEEDED, result=result, preserve_cancelled=True) + except QDBError as exc: + # Cancellation is terminal intent. A non-interruptible cloud call may + # fail after cancel() has returned; that late failure must not rewrite + # the durable cancelled state as failed. + self.jobs.update(job_id, JobStatus.FAILED, error_code=exc.code, error_message=str(exc), preserve_cancelled=True) + except Exception as exc: + self.jobs.update(job_id, JobStatus.FAILED, error_code="EXECUTION_FAILED", + error_message=f"{type(exc).__name__}: {exc}", preserve_cancelled=True) + + def _queue_depth(self) -> int: + with self._lock: + return sum(1 for future in self._futures.values() if not future.done()) + + def _requires_hardware_authorization(self) -> bool: + """True only when this sidecar would use the QCloud SDK and spend quota.""" + return self.quantum.adapter._qcloud_sdk_enabled() + + @staticmethod + def _hardware_grover_iterations(params: dict[str, Any]) -> int | None: + raw = params.get("hardware_grover_iterations") + if raw is None: + return None + if isinstance(raw, bool): + raise QDBError("INVALID_GROVER_ITERATIONS", "hardware_grover_iterations must be 1 or 2") + try: + iterations = int(raw) + except (TypeError, ValueError) as exc: + raise QDBError("INVALID_GROVER_ITERATIONS", "hardware_grover_iterations must be 1 or 2") from exc + if iterations not in {1, 2}: + raise QDBError("INVALID_GROVER_ITERATIONS", "hardware_grover_iterations must be 1 or 2") + return iterations + + @staticmethod + def _probe_component_trace(params: dict[str, Any], result: dict[str, Any]) -> dict[str, str]: + """Expose which execution subsystems participated in a probe result.""" + accelerator = result.get("accelerator", {}) + quantum = accelerator.get("quantum") if isinstance(accelerator, dict) else None + return { + "sidecar": "qdb.server", + "job_store": "sqlite", + "query_router": "used", + "candidate_source": "inline_rows" if isinstance(params.get("rows"), list) and params.get("rows") else "registered_dataset", + "candidate_filter": "classical_snapshot", + "ideal_qmeasure": "used" if isinstance(quantum, dict) and quantum.get("executed") else "not_used", + "tn_page": "used" if isinstance(accelerator, dict) and isinstance(accelerator.get("tn"), dict) and accelerator["tn"].get("executed") else "not_used", + "qute_engine": "used" if result.get("qute_validation", {}).get("executed") else "not_used", + "postgres_extension": "used" if params.get("execution_origin") == "postgres_extension" else "not_used", + } + + def _run_qute_validation(self, params: dict[str, Any], ir: RelationalIR, rows: list[Any]) -> dict[str, Any]: + """Run Qute's exact C0 contract over the same immutable candidate snapshot. + + This intentionally supports the sidecar's already-parsed conjunctive + predicate subset only. Unsupported SQL never becomes an approximate + result: the validation is recorded as unavailable and PostgreSQL still + performs its ordinary exact scan/recheck. + """ + try: + flattened_rows = [ + {"_qdb_key": item["key"], **item["values"]} + for item in rows + if isinstance(item, dict) and "key" in item and isinstance(item.get("values"), dict) + ] + if len(flattened_rows) != len(rows): + return {"executed": False, "reason": "invalid_snapshot_row"} + qute_result = self.execute_query({ + "query_id": f"qdb-validation-{ir.query_id or uuid.uuid4()}", + "source": ir.relation or str(params.get("relation", "qdb_snapshot")), + "rows": flattened_rows, + "filters": [{"column": predicate.column, "op": predicate.operator, "value": predicate.value} + for predicate in ir.predicates], + "aggregate": {"type": "count"}, "policy": "C0", + "data_version": ir.snapshot_version or "qdb-snapshot", + "metadata": {"workload": "qdb_candidate_validation"}, + }) + return { + "executed": True, "engine": "QuteEngine", "policy": "C0", + "exact": bool(qute_result.get("exact")), "value": qute_result.get("value"), + "selected_path": qute_result.get("trace", {}).get("selected_path"), + } + except Exception as exc: + return {"executed": False, "reason": f"{type(exc).__name__}: {exc}"} + + def _prepare_ir(self, params: dict[str, Any]) -> RelationalIR: + ir = build_relational_ir(params) + rows, _ = self._candidate_rows(params, ir) + if ir.estimated_selectivity is None and rows and ir.predicates: + matching = sum( + 1 for item in rows + if isinstance(item, dict) and isinstance(item.get("values"), dict) + and all(self._matches(item["values"].get(p.column), p.operator, p.value) for p in ir.predicates) + ) + ir.estimated_rows = len(rows) + ir.estimated_selectivity = matching / len(rows) + return ir + + def _candidate_rows(self, params: dict[str, Any], ir: RelationalIR) -> tuple[list[Any], dict[str, Any] | None]: + rows = params.get("rows", []) + if not isinstance(rows, list): + raise QDBError("INVALID_REQUEST", "rows must be an array") + dataset: dict[str, Any] | None = None + if not rows and ir.relation: + dataset = self.jobs.get_dataset(ir.relation, snapshot_version=ir.snapshot_version) + if dataset is None: + dataset = self.jobs.get_unqualified_dataset(ir.relation, snapshot_version=ir.snapshot_version) + if dataset is not None: + rows = dataset["rows"] + return rows, dataset + + @staticmethod + def _matches(actual: Any, operator: str, expected: Any) -> bool: + if isinstance(expected, str) and actual is not None and not isinstance(actual, str): + try: + if isinstance(actual, bool): + expected = expected.lower() in {"true", "t", "1"} + elif isinstance(actual, int) and not isinstance(actual, bool): + expected = int(expected) + elif isinstance(actual, float): + expected = float(expected) + except ValueError: + return False + if operator == "=": return actual == expected + if operator in {"!=", "<>"}: return actual != expected + if operator == "<": return actual is not None and actual < expected + if operator == "<=": return actual is not None and actual <= expected + if operator == ">": return actual is not None and actual > expected + if operator == ">=": return actual is not None and actual >= expected + return False + + @staticmethod + def _is_pg_scalar_equality_candidate(params: dict[str, Any], ir: RelationalIR, rows: list[Any]) -> bool: + """Gate physical filtering behind the small SQL subset we can prove here. + + PostgreSQL sends this marker only after verifying the actual plan contains + built-in bool/int equality predicates. This check also makes a direct or + future client unable to accidentally promote text, collated, numeric, time, + or range comparisons into a complete candidate superset. + """ + if params.get("candidate_semantics") != "pg_builtin_scalar_equality_v1" or not ir.predicates: + return False + allowed = {"bool", "int2", "int4", "int8"} + for predicate in ir.predicates: + if predicate.operator != "=" or getattr(predicate, "scalar_type", None) not in allowed: + return False + try: + if predicate.scalar_type == "bool": + value = predicate.value if isinstance(predicate.value, bool) else str(predicate.value).lower() in {"true", "t", "1"} + if not isinstance(predicate.value, bool) and str(predicate.value).lower() not in {"true", "false", "t", "f", "1", "0"}: + return False + else: + value = int(predicate.value) + except (TypeError, ValueError): + return False + if predicate.scalar_type != "bool" and isinstance(value, bool): + return False + # A JSON scalar produced by to_jsonb for this allowed PostgreSQL subset is + # exact: NULL never compares equal, booleans stay booleans, and integer values + # stay Python integers. Refuse malformed snapshot records instead of guessing. + return all(isinstance(item, dict) and "key" in item and isinstance(item.get("values"), dict) for item in rows) + + @staticmethod + def _safe_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in metadata.items() if "token" not in k.lower() and "key" not in k.lower()} diff --git a/systems/quantum_db/qute/__init__.py b/systems/quantum_db/qute/__init__.py new file mode 100644 index 0000000..3946d85 --- /dev/null +++ b/systems/quantum_db/qute/__init__.py @@ -0,0 +1,7 @@ +"""Backend-neutral Qute execution and benchmark framework.""" + +from .api.query_request import ExecutionPolicy, QueryRequest +from .api.query_result import QueryResult +from .execution.engine import QuteEngine + +__all__ = ["ExecutionPolicy", "QueryRequest", "QueryResult", "QuteEngine"] diff --git a/systems/quantum_db/qute/api/__init__.py b/systems/quantum_db/qute/api/__init__.py new file mode 100644 index 0000000..3a06a03 --- /dev/null +++ b/systems/quantum_db/qute/api/__init__.py @@ -0,0 +1,3 @@ +from .contracts import * +from .query_request import * +from .query_result import * diff --git a/systems/quantum_db/qute/api/contracts.py b/systems/quantum_db/qute/api/contracts.py new file mode 100644 index 0000000..fb37c9a --- /dev/null +++ b/systems/quantum_db/qute/api/contracts.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + + +@dataclass(slots=True) +class QualityContract: + estimated_error: float | None = None + confidence_interval: tuple[float, float] | None = None + confidence_level: float | None = None + estimated_fidelity: float | None = None + success_probability: float | None = None + shots: int = 0 + error_budget: float = 0.05 + confidence_threshold: float = 0.95 + success_threshold: float = 0.8 + + def satisfied(self) -> tuple[bool, list[str]]: + failures: list[str] = [] + if self.estimated_error is not None and self.estimated_error > self.error_budget: + failures.append("estimated_error exceeds error_budget") + if self.confidence_level is not None and self.confidence_level < self.confidence_threshold: + failures.append("confidence below threshold") + if self.success_probability is not None and self.success_probability < self.success_threshold: + failures.append("success_probability below threshold") + if self.estimated_fidelity is not None and self.estimated_fidelity < self.success_threshold: + failures.append("estimated_fidelity below threshold") + return not failures, failures + + +@dataclass(slots=True) +class ResourceEstimate: + logical_qubits: int = 0 + physical_qubits: int = 0 + circuit_depth: int = 0 + two_qubit_gate_count: int = 0 + shots: int = 0 + estimated_prepare_time_ms: float = 0.0 + estimated_compile_time_ms: float = 0.0 + estimated_submit_time_ms: float = 0.0 + estimated_queue_time_ms: float = 0.0 + estimated_execution_time_ms: float = 0.0 + estimated_measurement_time_ms: float = 0.0 + estimated_postprocess_time_ms: float = 0.0 + backend_feasible: bool = True + feasibility_reasons: list[str] = field(default_factory=list) + + @property + def estimated_e2e_latency_ms(self) -> float: + return sum((self.estimated_prepare_time_ms, self.estimated_compile_time_ms, + self.estimated_submit_time_ms, self.estimated_queue_time_ms, + self.estimated_execution_time_ms, self.estimated_measurement_time_ms, + self.estimated_postprocess_time_ms)) + + +@dataclass(slots=True) +class TimingBreakdown: + plan_ms: float = 0.0 + prepare_ms: float = 0.0 + compile_ms: float = 0.0 + submit_ms: float = 0.0 + queue_ms: float = 0.0 + execute_ms: float = 0.0 + measure_ms: float = 0.0 + postprocess_ms: float = 0.0 + validation_ms: float = 0.0 + fallback_ms: float = 0.0 + + @property + def e2e_ms(self) -> float: + return sum(asdict(self).values()) + + +@dataclass(slots=True) +class BatchMetrics: + batch_id: str + batch_size: int + batch_wait_time_ms: float + batch_execution_time_ms: float + amortized_compile_time_ms: float + amortized_prepare_time_ms: float + compatibility_key: str + + +@dataclass(slots=True) +class FallbackDecision: + triggered: bool = False + reason: str | None = None + from_path: str | None = None + to_path: str = "classical_exact" + error_code: str | None = None + + +@dataclass(slots=True) +class ArtifactMetadata: + artifact_id: str + state: str + data_source: str + data_version: str + encoding_version: str + circuit_version: str + observable: str + backend: str + noise_model: str + transpilation_config: str + shots: int + created_at: float + last_used_at: float + reuse_count: int = 0 + + +@dataclass(slots=True) +class ExecutionPlan: + plan_id: str + policy: str + selected_path: str + candidate_path_costs: dict[str, float] + resource_estimate: ResourceEstimate + reason: str + + +@dataclass(slots=True) +class ExecutionTrace: + query_id: str + plan_id: str + selected_path: str + candidate_paths: list[str] + quality_contract: QualityContract + resource_estimate: ResourceEstimate + backend: str + timing_breakdown: TimingBreakdown + fallback_triggered: bool + fallback_reason: str | None + final_result_type: str + result_error_if_known: float | None + candidate_path_costs: dict[str, float] = field(default_factory=dict) + cache_hit: bool = False + artifact_id: str | None = None + structured_error: dict[str, Any] | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) | {"timing_breakdown": asdict(self.timing_breakdown) | {"e2e_ms": self.timing_breakdown.e2e_ms}} diff --git a/systems/quantum_db/qute/api/query_request.py b/systems/quantum_db/qute/api/query_request.py new file mode 100644 index 0000000..fde4ebf --- /dev/null +++ b/systems/quantum_db/qute/api/query_request.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any + + +class ExecutionPolicy(StrEnum): + C0_CLASSICAL = "C0" + C1_MANUAL_HYBRID = "C1" + C2_STATIC = "C2" + C3_ADAPTIVE = "C3" + + +class OptimizationStrategy(StrEnum): + QUANTUM_FIRST = "quantum-first" + CLASSICAL_FIRST = "classical-first" + MIN_ESTIMATED_LATENCY = "min-estimated-latency" + MIN_COST_UNDER_QUALITY = "min-cost-under-quality-constraint" + FALLBACK_SAFE = "fallback-safe" + + +@dataclass(slots=True) +class QueryRequest: + query_id: str + source: str + rows: list[dict[str, Any]] + filters: list[dict[str, Any]] = field(default_factory=list) + group_by: list[str] = field(default_factory=list) + aggregate: dict[str, Any] = field(default_factory=lambda: {"type": "count"}) + quality_budget: dict[str, float] = field(default_factory=lambda: {"max_error": 0.05, "min_confidence": 0.95, "min_success_probability": 0.8}) + resource_budget: dict[str, float] = field(default_factory=lambda: {"max_depth": 500, "max_qubits": 32, "max_shots": 10000, "max_latency_ms": 5000, "max_two_qubit_gates": 10000}) + data_version: str = "v1" + encoding_version: str = "native-v1" + circuit_version: str = "qute-v1" + policy: ExecutionPolicy = ExecutionPolicy.C3_ADAPTIVE + optimization_strategy: OptimizationStrategy = OptimizationStrategy.MIN_COST_UNDER_QUALITY + shots: int = 1024 + random_seed: int = 7 + force_backend: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/systems/quantum_db/qute/api/query_result.py b/systems/quantum_db/qute/api/query_result.py new file mode 100644 index 0000000..89215bc --- /dev/null +++ b/systems/quantum_db/qute/api/query_result.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +from .contracts import ExecutionTrace, QualityContract, ResourceEstimate + + +@dataclass(slots=True) +class QuantumTask: + task_id: str + operator: str + payload: dict[str, Any] + shots: int + seed: int + quality_budget: dict[str, float] + resource_budget: dict[str, float] + + +@dataclass(slots=True) +class QuantumResult: + task_id: str + backend: str + value: Any + histogram: dict[str, int] + quality: QualityContract + resources: ResourceEstimate + timings_ms: dict[str, float] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class QueryResult: + query_id: str + value: Any + result_type: str + exact: bool + quality: QualityContract + trace: ExecutionTrace + + def to_dict(self) -> dict[str, Any]: + return {"query_id": self.query_id, "value": self.value, "result_type": self.result_type, + "exact": self.exact, "quality": asdict(self.quality), "trace": self.trace.to_dict()} diff --git a/systems/quantum_db/qute/benchmark/__init__.py b/systems/quantum_db/qute/benchmark/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/systems/quantum_db/qute/benchmark/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/systems/quantum_db/qute/benchmark/configs/noisy.json b/systems/quantum_db/qute/benchmark/configs/noisy.json new file mode 100644 index 0000000..67f1580 --- /dev/null +++ b/systems/quantum_db/qute/benchmark/configs/noisy.json @@ -0,0 +1,14 @@ +{ + "seed": 7, + "dataset_size": 2000, + "warm_repetitions": 4, + "repetitions": 5, + "max_batch_size": 8, + "backend": { + "type": "noisy", + "depolarizing_probability": 0.01, + "max_qubits": 32, + "max_depth": 500, + "available": true + } +} diff --git a/systems/quantum_db/qute/benchmark/configs/smoke.json b/systems/quantum_db/qute/benchmark/configs/smoke.json new file mode 100644 index 0000000..37c292c --- /dev/null +++ b/systems/quantum_db/qute/benchmark/configs/smoke.json @@ -0,0 +1,12 @@ +{ + "seed": 7, + "dataset_size": 256, + "warm_repetitions": 3, + "repetitions": 1, + "max_batch_size": 8, + "backend": { + "type": "ideal", + "max_qubits": 32, + "max_depth": 500 + } +} diff --git a/systems/quantum_db/qute/benchmark/plots.py b/systems/quantum_db/qute/benchmark/plots.py new file mode 100644 index 0000000..2a8379e --- /dev/null +++ b/systems/quantum_db/qute/benchmark/plots.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import html +from pathlib import Path +from typing import Any, Callable + + +COLORS = ("#2563eb", "#0f766e", "#d97706", "#dc2626", "#7c3aed", "#0891b2", "#65a30d", "#be123c", "#475569", "#9333ea") + + +def _svg_bars(title: str, labels: list[str], series: list[tuple[str, list[float]]], path: Path) -> None: + width, height, margin = 1000, 480, 70 + max_value = max((value for _, values in series for value in values), default=1.0) or 1.0 + group_width = (width - 2 * margin) / max(1, len(labels)) + bar_width = group_width / max(1, len(series) + 1) + parts = [f'', + '', f'{html.escape(title)}'] + for index, label in enumerate(labels): + x0 = margin + index * group_width + for sidx, (name, values) in enumerate(series): + value = values[index] if index < len(values) else 0.0 + bar_height = (height - 2 * margin) * value / max_value + x, y = x0 + sidx * bar_width, height - margin - bar_height + parts.append(f'') + parts.append(f'{html.escape(label[:24])}') + for sidx, (name, _) in enumerate(series): + parts.append(f'{html.escape(name)}') + parts.append(f'value') + path.write_text("".join(parts), encoding="utf-8") + + +def generate_required_plots(rows: list[dict[str, Any]], output_dir: Path) -> list[Path]: + plot_dir = output_dir / "plots"; plot_dir.mkdir(parents=True, exist_ok=True) + outputs: list[Path] = [] + timing_names = ("plan_ms", "prepare_ms", "compile_ms", "execute_ms", "postprocess_ms", "validation_ms", "fallback_ms") + specs: list[tuple[str, str, list[dict[str, Any]], list[tuple[str, Callable[[dict[str, Any]], float]]]]] = [ + ("e2e_breakdown.svg", "End-to-end timing breakdown", rows[:12], [(name, lambda row, n=name: float(row[n])) for name in timing_names]), + ("policy_latency.svg", "C0/C1/C2/C3 latency", [r for r in rows if r["query_id"].startswith("cold-")], [("e2e_ms", lambda r: float(r["e2e_ms"]))]), + ("quality_path_selection.svg", "Quality budget and selected-path latency", [r for r in rows if r["query_id"].startswith("quality-")], [("e2e_ms", lambda r: float(r["e2e_ms"])), ("fallback", lambda r: float(bool(r["fallback_triggered"]))) ]), + ("budget_fallback_rate.svg", "Depth budget and fallback", [r for r in rows if r["query_id"].startswith("depth-")], [("fallback", lambda r: float(bool(r["fallback_triggered"]))), ("depth", lambda r: float(r["depth"]))]), + ("cold_warm_cache.svg", "Cold versus warm reusable artifacts", [r for r in rows if r["query_id"].startswith(("cold-q1-C3", "warm-"))], [("e2e_ms", lambda r: float(r["e2e_ms"])), ("cache_hit", lambda r: float(bool(r["cache_hit"]))) ]), + ("shots_error.svg", "Shots versus observed error", [r for r in rows if r["query_id"].startswith("shots-")], [("error", lambda r: float(r["error_or_confidence"] or 0.0))]), + ("backend_feasibility.svg", "Backend feasibility matrix", rows, [("feasible", lambda r: float(bool(r["backend_feasible"]))), ("fallback", lambda r: float(bool(r["fallback_triggered"]))) ]), + ("tn_native_ablation.svg", "TN digest versus native encoding", [r for r in rows if r["query_id"].startswith("ablation-")], [("e2e_ms", lambda r: float(r["e2e_ms"])), ("qubits", lambda r: float(r["physical_qubits"]))]), + ] + for filename, title, selected, series_spec in specs: + path = plot_dir / filename + _svg_bars(title, [str(row["query_id"]) for row in selected], [(name, [getter(row) for row in selected]) for name, getter in series_spec], path) + outputs.append(path) + return outputs diff --git a/systems/quantum_db/qute/benchmark/reporting.py b/systems/quantum_db/qute/benchmark/reporting.py new file mode 100644 index 0000000..32a90ed --- /dev/null +++ b/systems/quantum_db/qute/benchmark/reporting.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import csv +import json +import math +import statistics +from pathlib import Path +from typing import Any + +from .plots import generate_required_plots + + +TRACE_FIELDS = ( + "query_id", "dataset_size", "policy", "selected_path", "candidate_path_costs", "plan_ms", "prepare_ms", + "compile_ms", "submit_ms", "queue_ms", "execute_ms", "measure_ms", "postprocess_ms", "validation_ms", + "fallback_ms", "e2e_ms", "error_or_confidence", "shots", "logical_qubits", "physical_qubits", "depth", + "two_qubit_gate_count", "cache_hit", "fallback_triggered", "fallback_reason", + "workload", "max_error", "max_depth", "max_qubits", "encoding_version", "backend_feasible", + "fallback_rate", "cache_reuse_rate", + "optimization_strategy", + "estimated_fidelity", "reconstruction_error", "query_error", +) + + +def flatten(result: dict[str, Any], dataset_size: int) -> dict[str, Any]: + trace, timing = result["trace"], result["trace"]["timing_breakdown"] + quality, resource = trace["quality_contract"], trace["resource_estimate"] + return { + "query_id": result["query_id"], "dataset_size": dataset_size, "policy": trace["metadata"]["policy"], + "selected_path": trace["selected_path"], "candidate_path_costs": json.dumps(trace["candidate_path_costs"], sort_keys=True), + **{name: timing[name] for name in ("plan_ms", "prepare_ms", "compile_ms", "submit_ms", "queue_ms", "execute_ms", "measure_ms", "postprocess_ms", "validation_ms", "fallback_ms", "e2e_ms")}, + "error_or_confidence": quality.get("estimated_error"), "shots": quality.get("shots", 0), + "logical_qubits": resource["logical_qubits"], "physical_qubits": resource["physical_qubits"], + "depth": resource["circuit_depth"], "two_qubit_gate_count": resource["two_qubit_gate_count"], + "cache_hit": trace["cache_hit"], "fallback_triggered": trace["fallback_triggered"], "fallback_reason": trace["fallback_reason"], + "workload": trace["metadata"].get("workload", ""), "max_error": trace["metadata"]["quality_budget"].get("max_error"), + "max_depth": trace["metadata"]["resource_budget"].get("max_depth"), "max_qubits": trace["metadata"]["resource_budget"].get("max_qubits"), + "encoding_version": trace["metadata"].get("encoding_version"), "backend_feasible": resource["backend_feasible"], + "fallback_rate": float(bool(trace["fallback_triggered"])), "cache_reuse_rate": float(bool(trace["cache_hit"])), + "optimization_strategy": trace["metadata"].get("optimization_strategy"), + "estimated_fidelity": quality.get("estimated_fidelity"), + "reconstruction_error": trace["metadata"].get("encoding_reconstruction_error", 0.0), + "query_error": trace.get("result_error_if_known"), + } + + +def write_outputs(output_dir: Path, config: dict[str, Any], raw: list[dict[str, Any]], rows: list[dict[str, Any]]) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "config.json").write_text(json.dumps(config, indent=2, sort_keys=True), encoding="utf-8") + with (output_dir / "raw_traces.jsonl").open("w", encoding="utf-8") as handle: + for item in raw: handle.write(json.dumps(item, sort_keys=True) + "\n") + with (output_dir / "metrics.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=TRACE_FIELDS); writer.writeheader(); writer.writerows(rows) + grouped: dict[str, list[dict[str, Any]]] = {} + for row in rows: + grouped.setdefault(f"{row['workload'] or 'default'}:{row['policy']}", []).append(row) + summary = { + key: { + "runs": len(items), + "mean_e2e_ms": sum(float(item["e2e_ms"]) for item in items) / len(items), + "stddev_e2e_ms": statistics.stdev([float(item["e2e_ms"]) for item in items]) if len(items) > 1 else 0.0, + "ci95_e2e_ms": (1.96 * statistics.stdev([float(item["e2e_ms"]) for item in items]) / math.sqrt(len(items))) if len(items) > 1 else 0.0, + "fallback_rate": sum(float(item["fallback_rate"]) for item in items) / len(items), + "cache_reuse_rate": sum(float(item["cache_reuse_rate"]) for item in items) / len(items), + "mean_error": sum(float(item["error_or_confidence"] or 0.0) for item in items) / len(items), + } + for key, items in grouped.items() + } + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + dataset_metadata = {"kind": "synthetic_orders", "generator": "qute.benchmark.workloads.dataset", + "seed": config.get("seed"), "configured_dataset_size": config.get("dataset_size"), + "columns": {"id": "integer", "region": "categorical", "category": "categorical", "amount": "float"}} + (output_dir / "dataset_metadata.json").write_text(json.dumps(dataset_metadata, indent=2, sort_keys=True), encoding="utf-8") + calibration_rows = [] + for item in raw: + trace = item["trace"] + estimated = sum(float(trace["resource_estimate"].get(name, 0.0)) for name in ( + "estimated_prepare_time_ms", "estimated_compile_time_ms", "estimated_submit_time_ms", + "estimated_queue_time_ms", "estimated_execution_time_ms", "estimated_measurement_time_ms", + "estimated_postprocess_time_ms")) + actual = float(trace["timing_breakdown"]["e2e_ms"]) + calibration_rows.append({"query_id": item["query_id"], "estimated_e2e_ms": estimated, + "observed_e2e_ms": actual, "observed_to_estimated_ratio": actual / estimated if estimated else None}) + (output_dir / "calibration.json").write_text(json.dumps(calibration_rows, indent=2, sort_keys=True), encoding="utf-8") + ablation = {str(row["encoding_version"]): row for row in rows if str(row["query_id"]).startswith("ablation-")} + native, tn = ablation.get("native-v1"), ablation.get("tn-digest-v1") + if native and tn: + improvements = { + "prepare_ms": float(tn["prepare_ms"]) < float(native["prepare_ms"]), + "compile_ms": float(tn["compile_ms"]) < float(native["compile_ms"]), + "e2e_ms": float(tn["e2e_ms"]) < float(native["e2e_ms"]), + "physical_qubits": float(tn["physical_qubits"]) < float(native["physical_qubits"]), + "depth": float(tn["depth"]) < float(native["depth"]), + "two_qubit_gate_count": float(tn["two_qubit_gate_count"]) < float(native["two_qubit_gate_count"]), + } + eligible = any(improvements.values()) + assessment = {"improvements": improvements, "main_experiment_eligible": eligible, + "status": "eligible" if eligible else "exploratory"} + (output_dir / "tn_ablation_assessment.json").write_text(json.dumps(assessment, indent=2, sort_keys=True), encoding="utf-8") + lines = ["# Qute end-to-end benchmark", "", "| query | policy | path | e2e ms | fallback | cache |", "|---|---:|---|---:|---|---|"] + for row in rows: + lines.append(f"| {row['query_id']} | {row['policy']} | {row['selected_path']} | {row['e2e_ms']:.4f} | {row['fallback_triggered']} | {row['cache_hit']} |") + lines.extend(["", "## Timing breakdown (ms)", "", + "| query | plan | prepare | compile | submit | queue | execute | measure | postprocess | validation | fallback |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"]) + for row in rows: + lines.append("| {query_id} | {plan_ms:.4f} | {prepare_ms:.4f} | {compile_ms:.4f} | {submit_ms:.4f} | {queue_ms:.4f} | {execute_ms:.4f} | {measure_ms:.4f} | {postprocess_ms:.4f} | {validation_ms:.4f} | {fallback_ms:.4f} |".format(**row)) + lines.extend(["", "Timing uses T_e2e = plan + prepare + compile + submit + queue + execute + measure + postprocess + validation + fallback."]) + (output_dir / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + generate_required_plots(rows, output_dir) diff --git a/systems/quantum_db/qute/benchmark/runner.py b/systems/quantum_db/qute/benchmark/runner.py new file mode 100644 index 0000000..6dd74ed --- /dev/null +++ b/systems/quantum_db/qute/benchmark/runner.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import argparse +import json +import platform +import sys +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +from qute.artifacts.artifact_store import ArtifactStore +from qute.execution.engine import QuteEngine +from qute.quantum.backend.ideal_simulator import IdealSimulatorBackend +from qute.quantum.backend.noisy_simulator import NoisySimulatorBackend +from qute.quantum.batching import BatchScheduler, metrics_to_dict +from qute.quantum.task_model import make_task +from qute.structured_log import StructuredLogger + +from .reporting import flatten, write_outputs +from .workloads import all_policies, cold_workload, failure_workload, strategy_workloads, sweep_workloads, warm_workload + + +def run_benchmark(config: dict[str, Any], output_dir: str | Path) -> list[dict[str, Any]]: + output = Path(output_dir) + backend_cfg = config.get("backend", {}) + if backend_cfg.get("type", "ideal") == "noisy": + backend = NoisySimulatorBackend(depolarizing_probability=float(backend_cfg.get("depolarizing_probability", 0.02)), + max_qubits=int(backend_cfg.get("max_qubits", 32)), max_depth=int(backend_cfg.get("max_depth", 500)), + available=bool(backend_cfg.get("available", True))) + else: + backend = IdealSimulatorBackend(max_qubits=int(backend_cfg.get("max_qubits", 32)), max_depth=int(backend_cfg.get("max_depth", 10_000))) + engine = QuteEngine(backend, artifact_store=ArtifactStore(output / "artifacts.json"), + logger=StructuredLogger(output / "execution.log.jsonl")) + seed, size = int(config.get("seed", 7)), int(config.get("dataset_size", 2000)) + requests = [] + for request in cold_workload(size, seed): requests.extend(all_policies(request)) + requests.extend(warm_workload(max(size, 5000), int(config.get("warm_repetitions", 4)), seed)) + requests.extend(failure_workload(size, seed)) + if bool(config.get("include_sweeps", True)): + requests.extend(sweep_workloads(max(size, 5000), seed)) + requests.extend(strategy_workloads(max(size, 5000), seed)) + raw, metrics = [], [] + repetitions = max(1, int(config.get("repetitions", 1))) + for request in requests: + for repetition in range(repetitions): + repeated = replace(request, query_id=f"{request.query_id}-r{repetition}" if repetitions > 1 else request.query_id, + random_seed=request.random_seed + repetition) + result = engine.execute(repeated).to_dict() + raw.append(result); metrics.append(flatten(result, len(repeated.rows))) + persisted_config = dict(config) | {"software_versions": {"python": sys.version, "platform": platform.platform()}, + "backend_metadata": backend.metadata().__dict__ if hasattr(backend.metadata(), "__dict__") else {name: getattr(backend.metadata(), name) for name in backend.metadata().__slots__}} + write_outputs(output, persisted_config, raw, metrics) + probabilities = {"00": 0.4, "01": 0.3, "10": 0.2, "11": 0.1} + tasks = [make_task("QEXPECT", {"state_ref": "benchmark-shared-state", "probabilities": probabilities, + "state_values": {"00": -1.0, "01": -0.5, "10": 0.5, "11": 1.0}, + "observable": f"Z{index}"}, shots=1024, seed=seed + index, + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7}, + resource_budget={"max_depth": 500, "max_qubits": 32, "max_shots": 10000, "max_latency_ms": 5000}) + for index in range(4)] + queued_at = time.perf_counter() + batch_output = BatchScheduler(backend, max_batch_size=int(config.get("max_batch_size", 8))).execute(tasks, queued_at=queued_at) + (output / "batch_metrics.json").write_text(json.dumps([metrics_to_dict(batch_metrics) for _, batch_metrics in batch_output], indent=2, sort_keys=True), encoding="utf-8") + return metrics + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run reproducible Qute C0-C3 benchmark") + parser.add_argument("--config", default="qute/benchmark/configs/smoke.json") + parser.add_argument("--output", default="outputs/qute_smoke") + args = parser.parse_args(argv) + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + run_benchmark(config, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/qute/benchmark/workloads.py b/systems/quantum_db/qute/benchmark/workloads.py new file mode 100644 index 0000000..e9e19d1 --- /dev/null +++ b/systems/quantum_db/qute/benchmark/workloads.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import random +from dataclasses import replace + +from qute.api.query_request import ExecutionPolicy, OptimizationStrategy, QueryRequest + + +def dataset(size: int, seed: int) -> list[dict[str, float | int | str]]: + rng = random.Random(seed) + regions = ("east", "west", "north", "south") + return [{"id": i, "region": regions[i % 4], "category": f"c{i % 8}", "amount": 10 + rng.random() * 90} for i in range(size)] + + +def cold_workload(size: int = 2000, seed: int = 7) -> list[QueryRequest]: + rows = dataset(size, seed) + return [QueryRequest("cold-q1", "orders", rows, filters=[{"column": "region", "op": "=", "value": "east"}], + aggregate={"type": "qexpect", "column": "amount", "observable": "amount_mean"}, + policy=ExecutionPolicy.C3_ADAPTIVE, random_seed=seed, data_version="cold-v1", + metadata={"workload": "cold", "transpilation_config": "cold"})] + + +def warm_workload(size: int = 5000, repetitions: int = 4, seed: int = 7) -> list[QueryRequest]: + rows = dataset(size, seed) + base = QueryRequest("warm-q0", "orders", rows, filters=[{"column": "region", "op": "=", "value": "east"}], + aggregate={"type": "qexpect", "column": "amount", "observable": "amount_mean"}, + quality_budget={"max_error": 0.08, "min_confidence": 0.95, "min_success_probability": 0.7}, + policy=ExecutionPolicy.C3_ADAPTIVE, shots=4096, random_seed=seed, + data_version="warm-v1", metadata={"workload": "warm"}) + return [replace(base, query_id=f"warm-q{index}", + policy=ExecutionPolicy.C1_MANUAL_HYBRID if index == 0 else ExecutionPolicy.C3_ADAPTIVE) + for index in range(repetitions)] + + +def failure_workload(size: int = 1000, seed: int = 7) -> list[QueryRequest]: + rows = dataset(size, seed) + return [ + QueryRequest("failure-depth", "orders", rows, aggregate={"type": "qexpect", "column": "amount"}, + policy=ExecutionPolicy.C2_STATIC, resource_budget={"max_depth": 1, "max_qubits": 32, "max_shots": 10000, "max_latency_ms": 5000, "max_two_qubit_gates": 10000}, random_seed=seed), + QueryRequest("failure-quality", "orders", rows, aggregate={"type": "qexpect", "column": "amount"}, + policy=ExecutionPolicy.C1_MANUAL_HYBRID, quality_budget={"max_error": 0.000001, "min_confidence": 0.99, "min_success_probability": 0.999}, shots=64, random_seed=seed), + ] + + +def sweep_workloads(size: int = 2000, seed: int = 7) -> list[QueryRequest]: + rows = dataset(size, seed) + base = QueryRequest("sweep", "orders", rows, filters=[{"column": "region", "op": "=", "value": "east"}], + aggregate={"type": "qexpect", "column": "amount", "observable": "amount_mean"}, + policy=ExecutionPolicy.C3_ADAPTIVE, shots=1024, random_seed=seed, + data_version="sweep-v1", metadata={"workload": "sweep"}) + output: list[QueryRequest] = [] + output.append(replace(base, query_id="sweep-prewarm", policy=ExecutionPolicy.C1_MANUAL_HYBRID, + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7})) + for budget in (0.01, 0.03, 0.05, 0.1): + output.append(replace(base, query_id=f"quality-{budget}", quality_budget={"max_error": budget, "min_confidence": 0.95, "min_success_probability": 0.8})) + for depth in (8, 16, 32, 128): + output.append(replace(base, query_id=f"depth-{depth}", resource_budget={"max_depth": depth, "max_qubits": 32, "max_shots": 10000, "max_latency_ms": 5000, "max_two_qubit_gates": 10000})) + for shots in (64, 256, 1024, 4096): + output.append(replace(base, query_id=f"shots-{shots}", policy=ExecutionPolicy.C1_MANUAL_HYBRID, shots=shots, + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7})) + output.append(replace(base, query_id="ablation-native", force_backend="quantum", encoding_version="native-v1", + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7})) + output.append(replace(base, query_id="ablation-tn", force_backend="quantum", encoding_version="tn-digest-v1", + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7})) + return output + + +def all_policies(request: QueryRequest) -> list[QueryRequest]: + return [replace(request, query_id=f"{request.query_id}-{policy.value}", policy=policy) for policy in ExecutionPolicy] + + +def strategy_workloads(size: int = 5000, seed: int = 7) -> list[QueryRequest]: + rows = dataset(size, seed) + base = QueryRequest("strategy", "strategy_orders", rows, filters=[{"column": "region", "op": "=", "value": "east"}], + aggregate={"type": "qexpect", "column": "amount", "observable": "amount_mean"}, + policy=ExecutionPolicy.C3_ADAPTIVE, shots=4096, random_seed=seed, data_version="strategies-v1", + quality_budget={"max_error": 0.1, "min_confidence": 0.95, "min_success_probability": 0.7}, + metadata={"workload": "strategies"}) + warm = replace(base, query_id="strategy-prewarm", policy=ExecutionPolicy.C1_MANUAL_HYBRID) + return [warm] + [replace(base, query_id=f"strategy-{strategy.value}", optimization_strategy=strategy) + for strategy in OptimizationStrategy] diff --git a/systems/quantum_db/qute/execution/__init__.py b/systems/quantum_db/qute/execution/__init__.py new file mode 100644 index 0000000..f0aeaa0 --- /dev/null +++ b/systems/quantum_db/qute/execution/__init__.py @@ -0,0 +1,3 @@ +from .engine import QuteEngine + +__all__ = ["QuteEngine"] diff --git a/systems/quantum_db/qute/execution/classical_executor.py b/systems/quantum_db/qute/execution/classical_executor.py new file mode 100644 index 0000000..f16a859 --- /dev/null +++ b/systems/quantum_db/qute/execution/classical_executor.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Any + +from qute.api.query_request import QueryRequest + + +def filtered_rows(request: QueryRequest) -> list[dict[str, Any]]: + return [row for row in request.rows if all(_matches(row.get(f["column"]), str(f.get("op", "=")), f.get("value")) for f in request.filters)] + + +def execute_classical(request: QueryRequest) -> Any: + rows = filtered_rows(request) + agg = str(request.aggregate.get("type", "count")).lower() + column = request.aggregate.get("column") + if request.group_by: + grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + grouped[tuple(row.get(key) for key in request.group_by)].append(row) + return {"|".join(map(str, key)): _aggregate(values, agg, column) for key, values in grouped.items()} + return _aggregate(rows, agg, column) + + +def _aggregate(rows: list[dict[str, Any]], agg: str, column: str | None) -> Any: + if agg in {"count", "qestimate"} and not column: + return len(rows) + values = [float(row[column]) for row in rows if column in row and row[column] is not None] if column else [] + if agg in {"sum", "qestimate"}: return sum(values) + if agg in {"mean", "avg", "qexpect"}: return sum(values) / len(values) if values else 0.0 + if agg == "min": return min(values) if values else None + if agg == "max": return max(values) if values else None + if agg == "count": return len(rows) + raise ValueError(f"unsupported classical aggregate: {agg}") + + +def _matches(actual: Any, op: str, expected: Any) -> bool: + if op == "=": return actual == expected + if op in {"!=", "<>"}: return actual != expected + if op == "between": return actual is not None and expected[0] <= actual <= expected[1] + if op == "in": return actual in expected + if op == "<": return actual is not None and actual < expected + if op == "<=": return actual is not None and actual <= expected + if op == ">": return actual is not None and actual > expected + if op == ">=": return actual is not None and actual >= expected + raise ValueError(f"unsupported filter operator: {op}") diff --git a/systems/quantum_db/qute/execution/engine.py b/systems/quantum_db/qute/execution/engine.py new file mode 100644 index 0000000..5dcd72c --- /dev/null +++ b/systems/quantum_db/qute/execution/engine.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import math +import time +from typing import Any + +from qute.api.contracts import ExecutionTrace, FallbackDecision, QualityContract, TimingBreakdown +from qute.api.query_request import QueryRequest +from qute.api.query_result import QueryResult +from qute.artifacts.artifact_store import ArtifactStore +from qute.logical.logical_plan import build_logical_plan +from qute.optimizer.dispatcher import Dispatcher +from qute.quantum.backend.base import QuantumBackend +from qute.quantum.task_model import make_task +from qute.structured_log import StructuredLogger + +from .classical_executor import execute_classical, filtered_rows +from .validation import validate_result +from .tn_executor import execute_tn_approximation + + +class QuteEngine: + def __init__(self, backend: QuantumBackend, *, artifact_store: ArtifactStore | None = None, + logger: StructuredLogger | None = None) -> None: + self.backend = backend + self.artifacts = artifact_store or ArtifactStore() + self.dispatcher = Dispatcher() + self.logger = logger or StructuredLogger() + + def execute(self, request: QueryRequest) -> QueryResult: + timings = TimingBreakdown() + started = time.perf_counter() + logical = build_logical_plan(request) + task = self._task(request, logical.quantum_operator) + estimate = self.backend.estimate(task) + backend_meta = self.backend.metadata() + invalidated_artifacts = self.artifacts.invalidate_data_version(request.source, request.data_version) + artifact_key = dict( + data_source=request.source, data_version=request.data_version, encoding_version=request.encoding_version, + circuit_version=request.circuit_version, observable=str(request.aggregate.get("observable", request.aggregate)), + backend=backend_meta.name, noise_model=backend_meta.noise_model, + transpilation_config=str(request.metadata.get("transpilation_config", "default")), shots=request.shots) + existing_artifact = self.artifacts.lookup(**artifact_key) + cache_hit = existing_artifact is not None + plan = self.dispatcher.choose(request, plan_id=logical.plan_id, estimate=estimate, cache_hit=cache_hit) + timings.plan_ms = (time.perf_counter() - started) * 1000 + exact_start = time.perf_counter() + exact = execute_classical(request) + exact_ms = (time.perf_counter() - exact_start) * 1000 + fallback = FallbackDecision() + if (request.policy.value == "C2" or request.force_backend in {"quantum", "hybrid"}) and plan.selected_path == "classical_exact": + fallback = FallbackDecision(True, plan.reason, "quantum", "classical_exact", "FEASIBILITY_REJECTED") + structured_error = None + observed_error = None + quality = QualityContract(estimated_error=0.0, estimated_fidelity=1.0, success_probability=1.0, + error_budget=float(request.quality_budget.get("max_error", 0.05)), + confidence_threshold=float(request.quality_budget.get("min_confidence", 0.95)), + success_threshold=float(request.quality_budget.get("min_success_probability", 0.8))) + value, exact_flag, result_type = exact, True, "exact" + selected_path = plan.selected_path + artifact = existing_artifact + if selected_path == "classical_tn": + try: + tn_start = time.perf_counter() + estimate_value, tn_quality = execute_tn_approximation(request) + timings.execute_ms = (time.perf_counter() - tn_start) * 1000 + validation_start = time.perf_counter() + valid, reasons, observed_error = validate_result(estimate_value, exact, tn_quality) + timings.validation_ms = (time.perf_counter() - validation_start) * 1000 + quality = tn_quality + if not valid: + raise RuntimeError("TN quality validation failed: " + "; ".join(reasons)) + value, exact_flag, result_type = estimate_value, False, "tn_estimate" + except Exception as exc: + fallback = FallbackDecision(True, str(exc), "classical_tn", "classical_exact", type(exc).__name__) + value, exact_flag, result_type = exact, True, "fallback_exact" + selected_path = "classical_exact" + structured_error = {"code": type(exc).__name__, "message": str(exc), "retryable": False} + timings.fallback_ms = exact_ms + elif selected_path != "classical_exact": + artifact, cache_hit = self.artifacts.get_or_create(**artifact_key) + try: + quantum = self.backend.execute(task) + timings.prepare_ms = quantum.timings_ms.get("prepare", 0.0) if not cache_hit else 0.0 + timings.compile_ms = quantum.timings_ms.get("compile", 0.0) / (4 if cache_hit else 1) + timings.submit_ms = quantum.timings_ms.get("submit", 0.0) + timings.queue_ms = quantum.timings_ms.get("queue", 0.0) + timings.execute_ms = quantum.timings_ms.get("execute", 0.0) + timings.measure_ms = quantum.timings_ms.get("measure", 0.0) + timings.postprocess_ms = quantum.timings_ms.get("postprocess", 0.0) + validation_start = time.perf_counter() + valid, reasons, observed_error = validate_result(quantum.value, exact, quantum.quality) + timings.validation_ms = (time.perf_counter() - validation_start) * 1000 + quality = quantum.quality + if valid: + value, exact_flag, result_type = quantum.value, False, "estimate" + if selected_path == "hybrid": + # Hybrid returns the authoritative value after measuring and validating the estimate. + value, exact_flag, result_type = exact, True, "validated_exact" + timings.postprocess_ms += exact_ms + else: + raise RuntimeError("quality validation failed: " + "; ".join(reasons)) + except Exception as exc: + fallback_start = time.perf_counter() + fallback = FallbackDecision(True, str(exc), selected_path, "classical_exact", type(exc).__name__) + value, exact_flag, result_type = exact, True, "fallback_exact" + selected_path = "classical_exact" + structured_error = {"code": type(exc).__name__, "message": str(exc), "retryable": False} + timings.fallback_ms = exact_ms + (time.perf_counter() - fallback_start) * 1000 + else: + timings.execute_ms = exact_ms + final_backend = "classical_local" if selected_path == "classical_exact" else "classical_tn" if selected_path == "classical_tn" else backend_meta.name + trace = ExecutionTrace( + request.query_id, plan.plan_id, selected_path, + ["classical_exact", "classical_tn", "quantum", "hybrid"], quality, estimate, + final_backend, timings, fallback.triggered, fallback.reason, result_type, + observed_error, plan.candidate_path_costs, cache_hit, artifact.artifact_id if artifact else None, + structured_error, {"policy": request.policy.value, "plan_reason": plan.reason, + "optimization_strategy": request.optimization_strategy.value, + "data_version": request.data_version, "artifact_state": artifact.state if artifact else "not_created", + "attempted_quantum_backend": backend_meta.name, "workload": request.metadata.get("workload", ""), + "quality_budget": dict(request.quality_budget), "resource_budget": dict(request.resource_budget), + "encoding_version": request.encoding_version, "invalidated_artifacts": invalidated_artifacts, + "encoding_reconstruction_error": task.payload.get("encoding_reconstruction_error", 0.0)}) + result = QueryResult(request.query_id, value, result_type, exact_flag, quality, trace) + self.logger.emit("query_execution_complete", query_id=request.query_id, plan_id=plan.plan_id, + selected_path=selected_path, backend=final_backend, fallback_triggered=fallback.triggered, + fallback_reason=fallback.reason, e2e_ms=timings.e2e_ms, + data_source=request.source, data_version=request.data_version) + return result + + def _task(self, request: QueryRequest, operator: str): + rows = filtered_rows(request) + column = request.aggregate.get("column") + values = [float(row[column]) for row in rows if column and row.get(column) is not None] + if not values: + values = [1.0 for _ in rows] or [0.0] + original_values = list(values) + reconstruction_error = 0.0 + weights: list[int] | None = None + if request.encoding_version.startswith("tn-") and len(values) > 128: + bucket_size = int(math.ceil(len(values) / 128)) + buckets = [values[index:index + bucket_size] for index in range(0, len(values), bucket_size)] + weights = [len(bucket) for bucket in buckets] + values = [sum(bucket) / len(bucket) for bucket in buckets] + reconstructed = [mean for mean, bucket in zip(values, buckets) for _ in bucket] + reconstruction_error = sum(abs(a - b) for a, b in zip(original_values, reconstructed)) / max(1e-12, sum(abs(v) for v in original_values)) + state_count = len(values) + width = max(1, int(math.ceil(math.log2(max(2, state_count))))) + total_weight = sum(weights) if weights else state_count + probabilities = {format(index, f"0{width}b"): (weights[index] if weights else 1) / total_weight for index in range(state_count)} + state_values = {state: value for state, value in zip(probabilities, values)} + payload: dict[str, Any] = {"probabilities": probabilities, "state_values": state_values, + "observable": request.aggregate.get("observable", "Z0"), + "encoding_reconstruction_error": reconstruction_error} + if operator == "QESTIMATE" and str(request.aggregate.get("type", "")).lower() in {"qestimate", "sum", "count"}: + payload["result_scale"] = len(original_values) + if operator == "QSIMILARITY": + payload.update({"left": request.aggregate.get("left", []), "right": request.aggregate.get("right", [])}) + return make_task(operator, payload, shots=request.shots, seed=request.random_seed, + quality_budget=request.quality_budget, resource_budget=request.resource_budget) diff --git a/systems/quantum_db/qute/execution/tn_executor.py b/systems/quantum_db/qute/execution/tn_executor.py new file mode 100644 index 0000000..7b2223d --- /dev/null +++ b/systems/quantum_db/qute/execution/tn_executor.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import math +from dataclasses import replace +from typing import Any + +from qute.api.contracts import QualityContract +from qute.api.query_request import QueryRequest + +from .classical_executor import execute_classical + + +def execute_tn_approximation(request: QueryRequest) -> tuple[Any, QualityContract]: + """Deterministic local TN surrogate path; replaceable by TNWorkloadService contraction.""" + sample_limit = int(request.metadata.get("tn_sample_limit", 256)) + if len(request.rows) <= sample_limit: + sample = request.rows + else: + stride = max(1, len(request.rows) // sample_limit) + sample = request.rows[::stride][:sample_limit] + sample_request = replace(request, rows=sample) + value = execute_classical(sample_request) + aggregate = str(request.aggregate.get("type", "count")).lower() + if aggregate in {"count", "sum", "qestimate"} and sample: + value = float(value) * len(request.rows) / len(sample) + error = 1.0 / math.sqrt(max(1, len(sample))) + quality = QualityContract( + estimated_error=error, confidence_level=0.95, + confidence_interval=None, estimated_fidelity=max(0.0, 1.0 - error), + success_probability=max(0.0, 1.0 - error), shots=0, + error_budget=float(request.quality_budget.get("max_error", 0.05)), + confidence_threshold=float(request.quality_budget.get("min_confidence", 0.95)), + success_threshold=float(request.quality_budget.get("min_success_probability", 0.8)), + ) + return value, quality diff --git a/systems/quantum_db/qute/execution/validation.py b/systems/quantum_db/qute/execution/validation.py new file mode 100644 index 0000000..4fb1fa7 --- /dev/null +++ b/systems/quantum_db/qute/execution/validation.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Any + +from qute.api.contracts import QualityContract + + +def relative_error(estimate: Any, exact: Any) -> float | None: + if not isinstance(estimate, (int, float)) or not isinstance(exact, (int, float)): + return None + return abs(float(estimate) - float(exact)) / max(1e-12, abs(float(exact))) + + +def validate_result(estimate: Any, exact: Any, quality: QualityContract) -> tuple[bool, list[str], float | None]: + ok, reasons = quality.satisfied() + error = relative_error(estimate, exact) + if error is not None: + quality.estimated_error = error + if error > quality.error_budget: + reasons.append("observed result error exceeds budget") + ok = False + return ok, reasons, error diff --git a/systems/quantum_db/qute/logical/__init__.py b/systems/quantum_db/qute/logical/__init__.py new file mode 100644 index 0000000..d541821 --- /dev/null +++ b/systems/quantum_db/qute/logical/__init__.py @@ -0,0 +1,3 @@ +from .logical_plan import LogicalPlan, build_logical_plan + +__all__ = ["LogicalPlan", "build_logical_plan"] diff --git a/systems/quantum_db/qute/logical/logical_plan.py b/systems/quantum_db/qute/logical/logical_plan.py new file mode 100644 index 0000000..126ffcd --- /dev/null +++ b/systems/quantum_db/qute/logical/logical_plan.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import uuid +from dataclasses import dataclass + +from qute.api.query_request import QueryRequest + + +@dataclass(frozen=True, slots=True) +class LogicalPlan: + plan_id: str + source: str + filters: tuple[dict, ...] + group_by: tuple[str, ...] + aggregate: dict + quantum_operator: str + + +def build_logical_plan(request: QueryRequest) -> LogicalPlan: + aggregate_type = str(request.aggregate.get("type", "count")).upper() + operator = aggregate_type if aggregate_type.startswith("Q") else "QESTIMATE" + return LogicalPlan(str(uuid.uuid4()), request.source, tuple(request.filters), tuple(request.group_by), + dict(request.aggregate), operator) diff --git a/systems/quantum_db/qute/optimizer/__init__.py b/systems/quantum_db/qute/optimizer/__init__.py new file mode 100644 index 0000000..85f37ca --- /dev/null +++ b/systems/quantum_db/qute/optimizer/__init__.py @@ -0,0 +1,3 @@ +from .dispatcher import Dispatcher + +__all__ = ["Dispatcher"] diff --git a/systems/quantum_db/qute/optimizer/cost_model.py b/systems/quantum_db/qute/optimizer/cost_model.py new file mode 100644 index 0000000..41d2a02 --- /dev/null +++ b/systems/quantum_db/qute/optimizer/cost_model.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from qute.api.contracts import ResourceEstimate +from qute.api.query_request import QueryRequest + + +def candidate_costs(request: QueryRequest, quantum: ResourceEstimate, *, cache_hit: bool) -> dict[str, float]: + row_count = len(request.rows) + classical = 0.004 * row_count + 0.02 * len(request.filters) + tn = 0.0035 * row_count + 0.5 + prepare = 0.0 if cache_hit else quantum.estimated_prepare_time_ms + compile_cost = quantum.estimated_compile_time_ms / (4 if cache_hit else 1) + qcost = quantum.estimated_e2e_latency_ms - quantum.estimated_prepare_time_ms - quantum.estimated_compile_time_ms + prepare + compile_cost + fallback_probability = max(0.0, 1.0 - (1.0 if quantum.backend_feasible else 0.0)) + hybrid = qcost + 0.25 * classical + fallback_probability * classical + return {"classical_exact": classical, "classical_tn": tn, "quantum": qcost, "hybrid": hybrid} diff --git a/systems/quantum_db/qute/optimizer/dispatcher.py b/systems/quantum_db/qute/optimizer/dispatcher.py new file mode 100644 index 0000000..6d08ca8 --- /dev/null +++ b/systems/quantum_db/qute/optimizer/dispatcher.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from qute.api.contracts import ExecutionPlan, ResourceEstimate +from qute.api.query_request import ExecutionPolicy, OptimizationStrategy, QueryRequest + +from .cost_model import candidate_costs +from .feasibility import check_feasibility + + +class Dispatcher: + def choose(self, request: QueryRequest, *, plan_id: str, estimate: ResourceEstimate, cache_hit: bool) -> ExecutionPlan: + costs = candidate_costs(request, estimate, cache_hit=cache_hit) + feasible, reasons = check_feasibility(estimate, request) + if request.force_backend: + requested = request.force_backend.lower() + aliases = {"classical": "classical_exact", "tn": "classical_tn", "quantum": "quantum", "hybrid": "hybrid"} + if requested not in aliases: + raise ValueError(f"unsupported forced backend: {request.force_backend}") + path, reason = aliases[requested], f"forced backend: {requested}" + if path in {"quantum", "hybrid"} and not feasible: + path, reason = "classical_exact", "forced quantum backend infeasible: " + "; ".join(reasons) + elif request.policy == ExecutionPolicy.C0_CLASSICAL: + path, reason = "classical_exact", "C0 classical oracle" + elif request.policy == ExecutionPolicy.C1_MANUAL_HYBRID: + path, reason = "hybrid", "C1 manual hybrid request" + elif request.policy == ExecutionPolicy.C2_STATIC: + path, reason = ("quantum", "C2 quantum feasible") if feasible else ("classical_exact", "C2 feasibility fallback: " + "; ".join(reasons)) + else: + allowed = {"classical_exact": costs["classical_exact"], "classical_tn": costs["classical_tn"]} + if feasible: + allowed.update({"quantum": costs["quantum"], "hybrid": costs["hybrid"]}) + strategy = request.optimization_strategy + if strategy == OptimizationStrategy.QUANTUM_FIRST: + path = "quantum" if feasible else min(allowed, key=allowed.get) + reason = "C3 quantum-first with feasibility guard" + elif strategy == OptimizationStrategy.CLASSICAL_FIRST: + path = "classical_exact" + reason = "C3 classical-first" + elif strategy == OptimizationStrategy.FALLBACK_SAFE: + path = "hybrid" if feasible else "classical_exact" + reason = "C3 fallback-safe chooses validated hybrid or exact classical" + elif strategy == OptimizationStrategy.MIN_ESTIMATED_LATENCY: + path = min(allowed, key=allowed.get) + reason = "C3 minimum estimated end-to-end latency" + else: + quality_safe = {name: cost for name, cost in allowed.items() if name not in {"quantum", "hybrid"} or feasible} + path = min(quality_safe, key=quality_safe.get) + reason = "C3 minimum cost under quality/resource constraints" + if not feasible: + reason += ": " + "; ".join(reasons) + return ExecutionPlan(plan_id, request.policy.value, path, costs, estimate, reason) diff --git a/systems/quantum_db/qute/optimizer/feasibility.py b/systems/quantum_db/qute/optimizer/feasibility.py new file mode 100644 index 0000000..5be14bd --- /dev/null +++ b/systems/quantum_db/qute/optimizer/feasibility.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import math + +from qute.api.contracts import ResourceEstimate +from qute.api.query_request import QueryRequest + + +def check_feasibility(estimate: ResourceEstimate, request: QueryRequest) -> tuple[bool, list[str]]: + reasons = list(estimate.feasibility_reasons) + budget = request.resource_budget + if estimate.physical_qubits > int(budget.get("max_qubits", estimate.physical_qubits)): reasons.append("physical qubits exceed request budget") + if estimate.circuit_depth > int(budget.get("max_depth", estimate.circuit_depth)): reasons.append("circuit depth exceeds request budget") + if estimate.two_qubit_gate_count > int(budget.get("max_two_qubit_gates", estimate.two_qubit_gate_count)): reasons.append("two-qubit gates exceed request budget") + if estimate.shots > int(budget.get("max_shots", estimate.shots)): reasons.append("shots exceed request budget") + if estimate.estimated_e2e_latency_ms > float(budget.get("max_latency_ms", estimate.estimated_e2e_latency_ms)): reasons.append("latency exceeds request budget") + predicted_sampling_error = 1.0 / math.sqrt(max(1, estimate.shots)) + if predicted_sampling_error > float(request.quality_budget.get("max_error", 1.0)): + reasons.append("predicted sampling error exceeds quality budget") + return estimate.backend_feasible and not reasons, list(dict.fromkeys(reasons)) diff --git a/systems/quantum_db/qute/quantum/__init__.py b/systems/quantum_db/qute/quantum/__init__.py new file mode 100644 index 0000000..bd622f0 --- /dev/null +++ b/systems/quantum_db/qute/quantum/__init__.py @@ -0,0 +1,4 @@ +from .backend.base import QuantumBackend +from .task_model import make_task + +__all__ = ["QuantumBackend", "make_task"] diff --git a/systems/quantum_db/qute/quantum/backend/__init__.py b/systems/quantum_db/qute/quantum/backend/__init__.py new file mode 100644 index 0000000..212be0e --- /dev/null +++ b/systems/quantum_db/qute/quantum/backend/__init__.py @@ -0,0 +1,6 @@ +from .base import BackendUnavailable, QuantumBackend +from .ideal_simulator import IdealSimulatorBackend +from .noisy_simulator import NoisySimulatorBackend +from .originq_adapter import OriginQBackend + +__all__ = ["BackendUnavailable", "QuantumBackend", "IdealSimulatorBackend", "NoisySimulatorBackend", "OriginQBackend"] diff --git a/systems/quantum_db/qute/quantum/backend/base.py b/systems/quantum_db/qute/quantum/backend/base.py new file mode 100644 index 0000000..8c273b3 --- /dev/null +++ b/systems/quantum_db/qute/quantum/backend/base.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from qute.api.contracts import ResourceEstimate +from qute.api.query_result import QuantumResult, QuantumTask + + +class BackendUnavailable(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class BackendMetadata: + name: str + kind: str + version: str + available: bool + max_qubits: int + max_depth: int + noise_model: str = "none" + + +class QuantumBackend(ABC): + @abstractmethod + def metadata(self) -> BackendMetadata: ... + + @abstractmethod + def estimate(self, task: QuantumTask) -> ResourceEstimate: ... + + @abstractmethod + def execute(self, task: QuantumTask) -> QuantumResult: ... + + def execute_batch(self, tasks: list[QuantumTask]) -> list[QuantumResult]: + return [self.execute(task) for task in tasks] diff --git a/systems/quantum_db/qute/quantum/backend/ideal_simulator.py b/systems/quantum_db/qute/quantum/backend/ideal_simulator.py new file mode 100644 index 0000000..b662077 --- /dev/null +++ b/systems/quantum_db/qute/quantum/backend/ideal_simulator.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import math +import time +from dataclasses import replace +from typing import Any + +from qute.api.contracts import QualityContract +from qute.api.query_result import QuantumResult, QuantumTask +from qute.quantum.measurement import normalize_probabilities, sample_histogram +from qute.quantum.resource_estimator import estimate_task_resources + +from .base import BackendMetadata, QuantumBackend + + +class IdealSimulatorBackend(QuantumBackend): + def __init__(self, *, max_qubits: int = 32, max_depth: int = 10_000) -> None: + self._metadata = BackendMetadata("ideal_simulator", "simulator", "1", True, max_qubits, max_depth) + + def metadata(self) -> BackendMetadata: + return self._metadata + + def estimate(self, task: QuantumTask): + return estimate_task_resources(task, max_qubits=self._metadata.max_qubits, max_depth=self._metadata.max_depth) + + def execute(self, task: QuantumTask) -> QuantumResult: + resources = self.estimate(task) + if not resources.backend_feasible: + raise ValueError("; ".join(resources.feasibility_reasons)) + if task.payload.get("_reuse_compiled"): + compile_ms = 0.0 + else: + compile_start = time.perf_counter() + hash((task.operator, tuple(sorted(task.payload.keys())))) + compile_ms = (time.perf_counter() - compile_start) * 1000 + prepared = task.payload.get("_prepared_probabilities") + if isinstance(prepared, dict): + probabilities, prepare_ms = dict(prepared), 0.0 + else: + prepare_start = time.perf_counter() + probabilities = normalize_probabilities(task.payload.get("probabilities", {"0": 0.5, "1": 0.5})) + prepare_ms = (time.perf_counter() - prepare_start) * 1000 + execute_start = time.perf_counter() + histogram = sample_histogram(probabilities, task.shots, task.seed) + execute_ms = (time.perf_counter() - execute_start) * 1000 + value, variance = self._operator_value(task, histogram) + absolute_error = 1.96 * math.sqrt(max(0.0, variance) / task.shots) + error = absolute_error / max(1e-12, abs(float(value))) if isinstance(value, (int, float)) else None + quality = QualityContract( + estimated_error=error, confidence_interval=(float(value) - absolute_error, float(value) + absolute_error) if isinstance(value, (int, float)) else None, + confidence_level=0.95, + estimated_fidelity=1.0, success_probability=1.0, shots=task.shots, + error_budget=float(task.quality_budget.get("max_error", 0.05)), + confidence_threshold=float(task.quality_budget.get("min_confidence", 0.95)), + success_threshold=float(task.quality_budget.get("min_success_probability", 0.8)), + ) + return QuantumResult(task.task_id, self._metadata.name, value, histogram, quality, resources, + {"prepare": prepare_ms, "compile": compile_ms, "execute": execute_ms, "measure": 0.0, "postprocess": 0.0}) + + def execute_batch(self, tasks: list[QuantumTask]) -> list[QuantumResult]: + if not tasks: + return [] + prepare_start = time.perf_counter() + prepared = normalize_probabilities(tasks[0].payload.get("probabilities", {"0": 0.5, "1": 0.5})) + shared_prepare_ms = (time.perf_counter() - prepare_start) * 1000 + results: list[QuantumResult] = [] + for index, task in enumerate(tasks): + payload = dict(task.payload) | {"_prepared_probabilities": prepared, "_reuse_compiled": index > 0} + result = self.execute(replace(task, payload=payload)) + if index == 0: + result.timings_ms["prepare"] = shared_prepare_ms + results.append(result) + return results + + @staticmethod + def _operator_value(task: QuantumTask, histogram: dict[str, int]) -> tuple[Any, float]: + shots = max(1, sum(histogram.values())) + if task.operator == "QMEASURE": + return {key: count / shots for key, count in histogram.items()}, 0.0 + if task.operator == "QSIMILARITY": + left, right = task.payload.get("left", []), task.payload.get("right", []) + dot = sum(float(a) * float(b) for a, b in zip(left, right)) + nl = math.sqrt(sum(float(v) ** 2 for v in left)) + nr = math.sqrt(sum(float(v) ** 2 for v in right)) + similarity = 0.0 if not nl or not nr else (dot / (nl * nr)) ** 2 + return similarity, similarity * (1 - similarity) + state_values = {str(k): float(v) for k, v in task.payload.get("state_values", {}).items()} + samples = [state_values.get(state, float(int(state, 2) if set(state) <= {"0", "1"} else 0)) for state, count in histogram.items() for _ in range(count)] + mean = sum(samples) / shots + variance = sum((value - mean) ** 2 for value in samples) / shots + scale = float(task.payload.get("result_scale", 1.0)) + return mean * scale, variance * scale * scale diff --git a/systems/quantum_db/qute/quantum/backend/noisy_simulator.py b/systems/quantum_db/qute/quantum/backend/noisy_simulator.py new file mode 100644 index 0000000..dfd0157 --- /dev/null +++ b/systems/quantum_db/qute/quantum/backend/noisy_simulator.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import replace + +from qute.api.query_result import QuantumResult, QuantumTask +from qute.quantum.measurement import normalize_probabilities +from qute.quantum.resource_estimator import estimate_task_resources + +from .base import BackendMetadata +from .ideal_simulator import IdealSimulatorBackend + + +class NoisySimulatorBackend(IdealSimulatorBackend): + def __init__(self, *, depolarizing_probability: float = 0.02, max_qubits: int = 32, + max_depth: int = 500, available: bool = True) -> None: + super().__init__(max_qubits=max_qubits, max_depth=max_depth) + self.depolarizing_probability = min(1.0, max(0.0, depolarizing_probability)) + self.available = available + self._metadata = BackendMetadata("noisy_simulator", "simulator", "1", available, max_qubits, max_depth, + f"depolarizing({self.depolarizing_probability})") + + def estimate(self, task: QuantumTask): + return estimate_task_resources(task, max_qubits=self._metadata.max_qubits, max_depth=self._metadata.max_depth, + noise_factor=self.depolarizing_probability, available=self.available) + + def execute(self, task: QuantumTask) -> QuantumResult: + source = task.payload.get("_prepared_probabilities", task.payload.get("probabilities", {"0": 0.5, "1": 0.5})) + raw = normalize_probabilities(source) + uniform = 1.0 / len(raw) + payload = dict(task.payload) + payload["probabilities"] = {key: (1 - self.depolarizing_probability) * value + self.depolarizing_probability * uniform for key, value in raw.items()} + if "_prepared_probabilities" in payload: + payload["_prepared_probabilities"] = payload["probabilities"] + result = super().execute(replace(task, payload=payload)) + result.backend = self._metadata.name + result.quality.estimated_fidelity = max(0.0, 1.0 - self.depolarizing_probability * result.resources.circuit_depth) + result.quality.success_probability = result.quality.estimated_fidelity + result.metadata["noise_model"] = self._metadata.noise_model + return result diff --git a/systems/quantum_db/qute/quantum/backend/originq_adapter.py b/systems/quantum_db/qute/quantum/backend/originq_adapter.py new file mode 100644 index 0000000..4cc3338 --- /dev/null +++ b/systems/quantum_db/qute/quantum/backend/originq_adapter.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import time + +from data_plane.quantum.backend import BackendConfig +from data_plane.quantum.executor import QuantumExecutor +from qute.api.contracts import QualityContract +from qute.api.query_result import QuantumResult, QuantumTask +from qute.quantum.resource_estimator import estimate_task_resources + +from .base import BackendMetadata, BackendUnavailable, QuantumBackend + + +class OriginQBackend(QuantumBackend): + def __init__(self, config: BackendConfig) -> None: + self.config = config + self.executor = QuantumExecutor(backend_config=config) + + def metadata(self) -> BackendMetadata: + available = bool(self.config.api_token or self.config.api_base) + return BackendMetadata(self.config.qcloud_backend_name or self.config.backend_name, "qpu", "pyqpanda3", + available, self.config.num_qubits, 500, "device") + + def estimate(self, task: QuantumTask): + meta = self.metadata() + return estimate_task_resources(task, max_qubits=meta.max_qubits, max_depth=meta.max_depth, available=meta.available) + + def execute(self, task: QuantumTask) -> QuantumResult: + ir = task.payload.get("circuit_ir") + if not isinstance(ir, dict): + raise ValueError("OriginQ task requires circuit_ir") + if not self.metadata().available: + raise BackendUnavailable("OriginQ credentials or endpoint are unavailable") + resources = self.estimate(task) + start = time.perf_counter() + compiled = self.executor.execute({"action": "compile", "payload": {"ir": ir}})["result"] + compile_ms = (time.perf_counter() - start) * 1000 + result = self.executor.execute({"action": "run", "payload": {"compiled": compiled, "shots": task.shots}})["result"] + quality = QualityContract(estimated_error=None, confidence_interval=None, confidence_level=None, estimated_fidelity=None, + success_probability=None, shots=task.shots, + error_budget=float(task.quality_budget.get("max_error", 0.05)), + confidence_threshold=float(task.quality_budget.get("min_confidence", 0.95)), + success_threshold=float(task.quality_budget.get("min_success_probability", 0.8))) + return QuantumResult(task.task_id, self.metadata().name, None, result.counts, quality, resources, + {"compile": compile_ms, "execute": result.latency_ms}, metadata=dict(result.metadata)) diff --git a/systems/quantum_db/qute/quantum/batching.py b/systems/quantum_db/qute/quantum/batching.py new file mode 100644 index 0000000..9215e0d --- /dev/null +++ b/systems/quantum_db/qute/quantum/batching.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import time +import uuid +from dataclasses import asdict +from typing import Iterable + +from qute.api.contracts import BatchMetrics +from qute.api.query_result import QuantumResult, QuantumTask +from qute.quantum.backend.base import QuantumBackend + + +class BatchScheduler: + """Groups tasks that can reuse the same state/circuit preparation.""" + + def __init__(self, backend: QuantumBackend, *, max_batch_size: int = 16) -> None: + self.backend = backend + self.max_batch_size = max(1, max_batch_size) + + def compatibility_key(self, task: QuantumTask) -> str: + payload = task.payload + key = { + "state_ref": payload.get("state_ref"), + "probabilities": payload.get("probabilities"), + "circuit_skeleton": payload.get("circuit_skeleton", payload.get("circuit_ir")), + "backend": self.backend.metadata().name, + "shots": task.shots, + } + return json.dumps(key, sort_keys=True, separators=(",", ":"), default=str) + + def group(self, tasks: Iterable[QuantumTask]) -> list[list[QuantumTask]]: + groups: dict[str, list[QuantumTask]] = {} + for task in tasks: + key = self.compatibility_key(task) + bucket = groups.setdefault(key, []) + if len(bucket) >= self.max_batch_size: + key = f"{key}#{len(bucket) // self.max_batch_size}" + bucket = groups.setdefault(key, []) + bucket.append(task) + return list(groups.values()) + + def execute(self, tasks: Iterable[QuantumTask], *, queued_at: float | None = None) -> list[tuple[list[QuantumResult], BatchMetrics]]: + submitted = list(tasks) + wait_ms = max(0.0, (time.perf_counter() - queued_at) * 1000) if queued_at is not None else 0.0 + output: list[tuple[list[QuantumResult], BatchMetrics]] = [] + for group in self.group(submitted): + started = time.perf_counter() + results = self.backend.execute_batch(group) + execution_ms = (time.perf_counter() - started) * 1000 + shared_compile = results[0].timings_ms.get("compile", 0.0) + shared_prepare = results[0].timings_ms.get("prepare", 0.0) + metrics = BatchMetrics( + batch_id=str(uuid.uuid4()), batch_size=len(group), batch_wait_time_ms=wait_ms, + batch_execution_time_ms=execution_ms, + amortized_compile_time_ms=shared_compile / len(group), + amortized_prepare_time_ms=shared_prepare / len(group), + compatibility_key=self.compatibility_key(group[0]), + ) + output.append((results, metrics)) + return output + + +def metrics_to_dict(metrics: BatchMetrics) -> dict[str, object]: + return asdict(metrics) diff --git a/systems/quantum_db/qute/quantum/measurement.py b/systems/quantum_db/qute/quantum/measurement.py new file mode 100644 index 0000000..eade978 --- /dev/null +++ b/systems/quantum_db/qute/quantum/measurement.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import math +import random +from typing import Any + + +def normalize_probabilities(raw: dict[str, Any]) -> dict[str, float]: + values = {str(key): max(0.0, float(value)) for key, value in raw.items()} + total = sum(values.values()) + if total <= 0: + raise ValueError("probability distribution has no positive mass") + return {key: value / total for key, value in values.items()} + + +def sample_histogram(probabilities: dict[str, float], shots: int, seed: int) -> dict[str, int]: + rng, states, weights = random.Random(seed), list(probabilities), list(probabilities.values()) + histogram = {state: 0 for state in states} + for state in rng.choices(states, weights=weights, k=shots): + histogram[state] += 1 + return {key: value for key, value in histogram.items() if value} + + +def mean_and_interval(samples: list[float], confidence: float = 0.95) -> tuple[float, float, tuple[float, float]]: + if not samples: + return 0.0, 0.0, (0.0, 0.0) + mean = sum(samples) / len(samples) + variance = sum((value - mean) ** 2 for value in samples) / max(1, len(samples) - 1) + z = 1.96 if confidence >= 0.95 else 1.645 + margin = z * math.sqrt(variance / len(samples)) + return mean, variance, (mean - margin, mean + margin) diff --git a/systems/quantum_db/qute/quantum/resource_estimator.py b/systems/quantum_db/qute/quantum/resource_estimator.py new file mode 100644 index 0000000..4ff1283 --- /dev/null +++ b/systems/quantum_db/qute/quantum/resource_estimator.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import math + +from qute.api.contracts import ResourceEstimate +from qute.api.query_result import QuantumTask + + +def estimate_task_resources(task: QuantumTask, *, max_qubits: int, max_depth: int, + noise_factor: float = 0.0, available: bool = True) -> ResourceEstimate: + probabilities = task.payload.get("probabilities", {}) + state_count = max(2, len(probabilities) if isinstance(probabilities, dict) else 2) + logical = max(1, int(math.ceil(math.log2(state_count)))) + depth = int(task.payload.get("circuit_depth", 4 * logical + 2)) + two_qubit = int(task.payload.get("two_qubit_gate_count", max(0, depth - logical))) + physical = int(math.ceil(logical * (1.0 + noise_factor))) + reasons: list[str] = [] + if not available: reasons.append("backend unavailable") + if physical > min(max_qubits, int(task.resource_budget.get("max_qubits", max_qubits))): reasons.append("qubit budget exceeded") + if depth > min(max_depth, int(task.resource_budget.get("max_depth", max_depth))): reasons.append("depth budget exceeded") + if two_qubit > int(task.resource_budget.get("max_two_qubit_gates", 10_000)): reasons.append("two-qubit gate budget exceeded") + estimate = ResourceEstimate( + logical_qubits=logical, physical_qubits=physical, circuit_depth=depth, + two_qubit_gate_count=two_qubit, shots=task.shots, + estimated_prepare_time_ms=0.03 * state_count, + estimated_compile_time_ms=0.05 * depth, + estimated_submit_time_ms=0.05, + estimated_execution_time_ms=0.00002 * task.shots * max(1, depth), + estimated_measurement_time_ms=0.00005 * task.shots, + estimated_postprocess_time_ms=0.0001 * task.shots, + backend_feasible=not reasons, feasibility_reasons=reasons, + ) + if estimate.estimated_e2e_latency_ms > float(task.resource_budget.get("max_latency_ms", float("inf"))): + estimate.backend_feasible = False + estimate.feasibility_reasons.append("latency budget exceeded") + return estimate diff --git a/systems/quantum_db/qute/quantum/task_model.py b/systems/quantum_db/qute/quantum/task_model.py new file mode 100644 index 0000000..8355203 --- /dev/null +++ b/systems/quantum_db/qute/quantum/task_model.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import uuid +from typing import Any + +from qute.api.query_result import QuantumTask + + +SUPPORTED_OPERATORS = frozenset({"QEXPECT", "QMEASURE", "QSIMILARITY", "QESTIMATE"}) + + +def make_task(operator: str, payload: dict[str, Any], *, shots: int, seed: int, + quality_budget: dict[str, float], resource_budget: dict[str, float]) -> QuantumTask: + normalized = operator.strip().upper() + if normalized not in SUPPORTED_OPERATORS: + raise ValueError(f"unsupported quantum operator: {operator}") + if shots <= 0 or shots > int(resource_budget.get("max_shots", shots)): + raise ValueError("shots must be positive and within max_shots") + return QuantumTask(str(uuid.uuid4()), normalized, dict(payload), shots, seed, + dict(quality_budget), dict(resource_budget)) diff --git a/systems/quantum_db/qute/quantum/tn_adapter.py b/systems/quantum_db/qute/quantum/tn_adapter.py new file mode 100644 index 0000000..7ccc5b4 --- /dev/null +++ b/systems/quantum_db/qute/quantum/tn_adapter.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Any + +from data_plane.tn.exproter import TNExporter +from data_plane.tn.page import TNPage +from qute.api.query_result import QuantumTask +from qute.quantum.task_model import make_task + + +class TNPageTaskAdapter: + """Converts a versioned TN-Page into a backend-neutral QuantumTask.""" + + def __init__(self, exporter: TNExporter | None = None) -> None: + self.exporter = exporter or TNExporter() + + def to_task(self, page: TNPage, *, data_version: str, target_backend: str, operator: str, + observable: str, shots: int, seed: int, quality_budget: dict[str, float], + resource_budget: dict[str, float], probabilities: dict[str, float] | None = None, + state_values: dict[str, float] | None = None) -> QuantumTask: + page.validate() + if page.metadata.version != data_version: + raise ValueError(f"TN-Page version {page.metadata.version!r} does not match data version {data_version!r}") + packet = self.exporter.export(page, target_backend=target_backend) + circuit_ir = self.exporter.to_quantum_ir(packet) + operations = circuit_ir.get("operations", []) + payload: dict[str, Any] = { + "state_ref": page.page_id, + "circuit_skeleton": f"tn:{page.metadata.layout}:{page.metadata.encoding}", + "circuit_ir": circuit_ir, + "circuit_depth": len(operations), + "two_qubit_gate_count": sum(1 for op in operations if str(op.get("gate", "")).lower() in {"cx", "cz", "swap"}), + "tn_page_summary": page.summary(), + "data_version": data_version, + "observable": observable, + } + if probabilities is not None: + payload["probabilities"] = dict(probabilities) + if state_values is not None: + payload["state_values"] = dict(state_values) + return make_task(operator, payload, shots=shots, seed=seed, + quality_budget=quality_budget, resource_budget=resource_budget) diff --git a/systems/quantum_db/qute/structured_log.py b/systems/quantum_db/qute/structured_log.py new file mode 100644 index 0000000..31ef286 --- /dev/null +++ b/systems/quantum_db/qute/structured_log.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import json +import threading +import time +from pathlib import Path +from typing import Any + + +class StructuredLogger: + def __init__(self, path: str | Path | None = None) -> None: + self.path = Path(path) if path else None + self._lock = threading.Lock() + + def emit(self, event: str, **fields: Any) -> None: + if self.path is None: + return + record = {"timestamp": time.time(), "event": event, **fields} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self._lock, self.path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True, default=str) + "\n") diff --git a/systems/quantum_db/scripts/alu_benchmark_driver.py b/systems/quantum_db/scripts/alu_benchmark_driver.py new file mode 100644 index 0000000..99392e8 --- /dev/null +++ b/systems/quantum_db/scripts/alu_benchmark_driver.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import argparse +import csv +import json +import math +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from control_plane.semantic.workload_semantic import SemanticExperimentConfig, TNSemanticOrchestrator +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig + + +@dataclass(slots=True) +class ALUProfile: + name: str + num_sites: int + phys_dim: int + chi_max: int + keep_ratio: float + + +DEFAULT_ALUS = [ + ALUProfile("alu_tiny", 8, 8, 4, 0.2), + ALUProfile("alu_balanced", 8, 8, 8, 0.25), + ALUProfile("alu_fidelity", 12, 8, 16, 0.4), + ALUProfile("alu_storage", 6, 6, 4, 0.15), +] + + +def load_csv_numeric_rows(path: Path, limit: int, subset_cols: list[int] | None) -> list[list[float]]: + rows: list[list[float]] = [] + with path.open("r", encoding="utf-8") as f: + reader = csv.reader(f) + for raw in reader: + vals: list[float] = [] + for i, token in enumerate(raw): + if subset_cols is not None and i not in subset_cols: + continue + try: + vals.append(float(token.strip())) + except ValueError: + pass + if vals: + rows.append(vals) + if len(rows) >= limit: + break + if not rows: + raise RuntimeError(f"No numeric rows loaded from {path}") + min_len = min(len(r) for r in rows) + return [r[:min_len] for r in rows if len(r) >= min_len] + + +def estimate_storage_usage(case: dict) -> float: + tn_page = case.get("tn_page", {}) + phys_dims = tn_page.get("physical_dims", []) + bond_dims = tn_page.get("bond_dims", []) + if not phys_dims: + return 0.0 + # 估算 MPS 参数量：sum(d * Dl * Dr) + total_params = 0 + n = len(phys_dims) + for i, d in enumerate(phys_dims): + dl = 1 if i == 0 else bond_dims[i - 1] + dr = 1 if i == n - 1 else bond_dims[i] + total_params += int(d) * int(dl) * int(dr) + # 假设 float64 + return float(total_params * 8) + + +def estimate_depth_resource(case: dict) -> dict[str, float]: + tn_page = case.get("tn_page", {}) + bonds = [int(x) for x in tn_page.get("bond_dims", [])] + num_tensors = int(tn_page.get("num_tensors", 0)) + depth_est = float(num_tensors + sum(max(1, int(math.log2(max(2, b)))) for b in bonds)) + resource_est = float(sum(bonds) + num_tensors) + return {"depth_estimate": depth_est, "resource_estimate": resource_est} + + +def pareto_frontier(rows: list[dict]) -> list[dict]: + frontier: list[dict] = [] + for r in rows: + dominated = False + for other in rows: + if other is r: + continue + better_or_equal = ( + other["fidelity"] >= r["fidelity"] + and other["storage_usage"] <= r["storage_usage"] + and other["compression_ratio"] >= r["compression_ratio"] + ) + strictly_better = ( + other["fidelity"] > r["fidelity"] + or other["storage_usage"] < r["storage_usage"] + or other["compression_ratio"] > r["compression_ratio"] + ) + if better_or_equal and strictly_better: + dominated = True + break + if not dominated: + frontier.append(r) + return sorted(frontier, key=lambda x: (-x["fidelity"], x["storage_usage"])) + + +def theoretical_tn_deepdb_judgement(row: dict) -> str: + # 仅理论判据：高保真 + 低存储 + 可控复杂度，才“可能”优于经典方法。 + if row["fidelity"] >= 0.9 and row["compression_ratio"] >= 1.2 and row["storage_usage"] <= 64_000 and row["depth_estimate"] <= 80: + return "可能（理论上）" + return "不充分（理论上）" + + +def render_markdown(summary: dict) -> str: + lines = [ + "# ALU 实验对比报告", + "", + f"- dataset: `{summary['dataset']}`", + f"- samples: {summary['samples']}", + f"- subset_cols: {summary['subset_cols']}", + "", + "## 全量结果", + "", + "| ALU | fidelity | compression_ratio | storage_usage(bytes est.) | depth_estimate | resource_estimate | TN/DeepDB理论判据 |", + "|---|---:|---:|---:|---:|---:|---|", + ] + for row in summary["results"]: + lines.append( + f"| {row['alu']} | {row['fidelity']:.4f} | {row['compression_ratio']:.4f} | {row['storage_usage']:.0f} | {row['depth_estimate']:.1f} | {row['resource_estimate']:.1f} | {row['theoretical_vs_classic']} |" + ) + + lines.extend( + [ + "", + "## Best-by-fidelity", + f"- {summary['best_by_fidelity']['alu']} ({summary['best_by_fidelity']['fidelity']:.4f})", + "", + "## Best-by-storage", + f"- {summary['best_by_storage']['alu']} ({summary['best_by_storage']['storage_usage']:.0f} bytes est.)", + "", + "## Pareto Frontier", + ] + ) + for row in summary["pareto_frontier"]: + lines.append(f"- {row['alu']}: fidelity={row['fidelity']:.4f}, storage={row['storage_usage']:.0f}, compression={row['compression_ratio']:.4f}") + + lines.extend([ + "", + "## 说明", + "- 本报告中“是否可能优于经典 TN/DeepDB”仅基于理论阈值判断，不代表实证结论。", + "- 结论需要在统一硬件、真实工作负载和基线实现上做端到端验证。", + ]) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="ALU experiment driver for fixed synthetic dataset") + parser.add_argument("--dataset", default="train-test-data/SyntheticDataset/cols_8_distinct_10000_corr_8_skew_15.csv") + parser.add_argument("--limit", type=int, default=1024) + parser.add_argument("--subset-cols", default="", help="comma-separated 0-based column indices") + parser.add_argument("--out-json", default="artifacts/alu_experiment_summary.json") + parser.add_argument("--out-md", default="artifacts/alu_experiment_summary.md") + args = parser.parse_args() + + dataset_path = (ROOT / args.dataset).resolve() + subset_cols = [int(x) for x in args.subset_cols.split(",") if x.strip()] if args.subset_cols else None + samples = load_csv_numeric_rows(dataset_path, limit=args.limit, subset_cols=subset_cols) + + results: list[dict] = [] + for profile in DEFAULT_ALUS: + workload_cfg = WorkloadConfig( + num_sites=profile.num_sites, + phys_dim=profile.phys_dim, + chi_max=profile.chi_max, + keep_ratio=profile.keep_ratio, + ) + service = TNWorkloadService(config=workload_cfg, seed=7) + orchestrator = TNSemanticOrchestrator(service=service, config=SemanticExperimentConfig(data_dir=dataset_path.parent, limit=len(samples))) + case = orchestrator.run_dataset(profile.name, samples, [str(dataset_path)], "fixed") + + depth_res = estimate_depth_resource(case) + row = { + "alu": profile.name, + "config": asdict(profile), + "fidelity": float(case["metrics"]["fidelity"]), + "compression_ratio": float(case["metrics"]["compression_ratio"]), + "storage_usage": estimate_storage_usage(case), + **depth_res, + } + row["theoretical_vs_classic"] = theoretical_tn_deepdb_judgement(row) + results.append(row) + + best_by_fidelity = max(results, key=lambda x: x["fidelity"]) + best_by_storage = min(results, key=lambda x: x["storage_usage"]) + frontier = pareto_frontier(results) + + summary = { + "dataset": str(dataset_path.relative_to(ROOT)), + "samples": len(samples), + "subset_cols": subset_cols, + "results": results, + "best_by_fidelity": best_by_fidelity, + "best_by_storage": best_by_storage, + "pareto_frontier": frontier, + "notes": { + "judgement": "理论判据，仅用于提示‘可能性’，非实证优劣结论", + "classic_baseline": "未直接运行 classic TN/DeepDB 基线，仅做理论阈值映射", + }, + } + + out_json = ROOT / args.out_json + out_md = ROOT / args.out_md + out_json.parent.mkdir(parents=True, exist_ok=True) + out_md.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + out_md.write_text(render_markdown(summary), encoding="utf-8") + + print(json.dumps({"out_json": str(out_json), "out_md": str(out_md), "rows": len(results)}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/qdb_e2e_smoke.py b/systems/quantum_db/scripts/qdb_e2e_smoke.py new file mode 100644 index 0000000..715ee91 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_e2e_smoke.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Run the local Unix-socket Quantum DB workflow without PostgreSQL or cloud access. + +This covers the transport boundary, SQL-hint/IR derivation, automatic routing, +durable job lifecycle, candidate generation, and the Qute exact contract. PostgreSQL +integration has a separate PGXS regression because it needs server development files. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdb.client import call + + +def _wait_for_socket(path: Path, process: subprocess.Popen[bytes]) -> None: + for _ in range(100): + if path.exists(): + return + if process.poll() is not None: + stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else "" + raise RuntimeError(f"sidecar exited early with code {process.returncode}: {stderr.strip()}") + time.sleep(0.02) + raise TimeoutError(f"sidecar did not create {path}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--keep-state", action="store_true", help="print the temporary state directory instead of deleting it") + args = parser.parse_args(argv) + temp = tempfile.TemporaryDirectory(prefix="qdb-e2e-") + root = Path(temp.name) + socket_path = root / "qdb.sock" + state_path = root / "jobs.sqlite3" + process = subprocess.Popen( + [sys.executable, "-m", "qdb.server", "--socket", str(socket_path), "--state", str(state_path)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + ) + try: + _wait_for_socket(socket_path, process) + rows = [ + {"key": 1, "values": {"region": "east", "amount": 10}}, + {"key": 2, "values": {"region": "west", "amount": 15}}, + {"key": 3, "values": {"region": "east", "amount": 25}}, + ] + [{"key": key, "values": {"region": "west", "amount": 15}} for key in range(4, 21)] + request = { + "sql": "/*+ QDB_AUTO */ SELECT count(*) FROM public.orders WHERE region = 'east' AND amount >= 10 GROUP BY region", + "relation": "public.orders", "snapshot_version": "smoke-v1", + } + health = call(str(socket_path), "health", {}) + checksum = hashlib.md5(json.dumps(rows, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + upload = call(str(socket_path), "dataset.begin", { + "relation": "public.orders", "snapshot_version": "smoke-v1", "key_column": "id", + "artifact_checksum": checksum, "expected_rows": len(rows), + }) + for page, start in enumerate(range(0, len(rows), 7)): + call(str(socket_path), "dataset.append", { + "upload_id": upload["upload_id"], "page": page, "rows": rows[start:start + 7], + }) + registered = call(str(socket_path), "dataset.commit", {"upload_id": upload["upload_id"]}) + request["artifact_checksum"] = checksum + explanation = call(str(socket_path), "explain", request) + submitted = call(str(socket_path), "submit", {**request, "run_qpanda_probe": True, "wait": True, "timeout_ms": 1_000}) + job_id = submitted["job_id"] + status = call(str(socket_path), "status", {"job_id": job_id}) + result = call(str(socket_path), "result", {"job_id": job_id}) + qute_result = call(str(socket_path), "query.execute", { + "query_id": "e2e-smoke", "source": "public.orders", + "rows": [{"id": item["key"], **item["values"]} for item in rows], + "filters": [{"column": "region", "op": "=", "value": "east"}], + "aggregate": {"type": "count"}, "policy": "C0", + }) + if explanation["route"]["backend"] != "quantum": + raise AssertionError(f"expected selective quantum route: {explanation}") + if status["status"] != "succeeded" or result["result"]["candidate_keys"] != [1, 3]: + raise AssertionError(f"unexpected durable candidate result: {result}") + accelerator = result["result"].get("accelerator", {}) + if not accelerator.get("quantum", {}).get("executed"): + raise AssertionError(f"expected ideal quantum probe execution: {result}") + qpanda3 = accelerator.get("qpanda3", {}) + if qpanda3.get("executed"): + qpanda3_probe = qpanda3.get("metadata", {}).get("execution_mode", "") + elif "hardware_probe_unsupported" in str(qpanda3.get("reason", "")): + # The real-hardware Grover compiler intentionally accepts only the + # audited eight-row, single-marked-address shape. This 20-row smoke + # query still exercises the ideal advisory probe and exact SQL path. + qpanda3_probe = "unsupported_shape" + else: + raise AssertionError(f"unexpected pyQPanda3 probe outcome: {result}") + if qute_result["value"] != 2 or not qute_result["exact"]: + raise AssertionError(f"unexpected exact result: {qute_result}") + print(json.dumps({ + "health": health, "route": explanation["route"], "job_status": status["status"], + "candidate_keys": result["result"]["candidate_keys"], "postgres_recheck_required": result["requires_recheck"], + "qute_exact_value": qute_result["value"], "registered_rows": registered["row_count"], + "qpanda3_probe": qpanda3_probe, + }, ensure_ascii=False, sort_keys=True)) + finally: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if args.keep_state: + print(f"state_dir={root}", file=sys.stderr) + temp.cleanup = lambda: None # type: ignore[method-assign] + else: + temp.cleanup() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/scripts/qdb_large_scale_benchmark.py b/systems/quantum_db/scripts/qdb_large_scale_benchmark.py new file mode 100644 index 0000000..d07fdc5 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_large_scale_benchmark.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Run reproducible large-scale traversal, cardinality, and sampling experiments.""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from qdb.large_scale import BenchmarkConfig, run_large_scale_benchmark, write_report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", default="10000,100000,1000000", help="comma-separated row counts") + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--reservoir-size", type=int, default=8192) + parser.add_argument("--tn-train-samples", type=int, default=2048) + parser.add_argument("--tn-draws", type=int, default=1024) + parser.add_argument("--hll-precision", type=int, default=12) + parser.add_argument("--output", default="outputs/qdb_large_scale") + args = parser.parse_args() + sizes = tuple(int(part) for part in args.sizes.split(",") if part.strip()) + report = run_large_scale_benchmark(BenchmarkConfig( + sizes=sizes, seed=args.seed, reservoir_size=args.reservoir_size, + tn_train_samples=args.tn_train_samples, tn_draws=args.tn_draws, hll_precision=args.hll_precision, + )) + json_path, markdown_path = write_report(report, Path(args.output)) + print({"report_json": str(json_path), "report_markdown": str(markdown_path), "sizes": list(sizes)}) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/scripts/qdb_postgres_e2e_smoke.sh b/systems/quantum_db/scripts/qdb_postgres_e2e_smoke.sh new file mode 100644 index 0000000..c4c7681 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_postgres_e2e_smoke.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Run the complete local PostgreSQL -> Custom Scan -> sidecar -> PostgreSQL recheck path. +# This intentionally uses only a temporary cluster and the qpanda3-sim/offline candidate route. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PG_CONFIG="${PG_CONFIG:-pg_config}" +PG_BIN="$($PG_CONFIG --bindir)" +# sockaddr_un paths are short; do not inherit a long per-user TMPDIR here. +WORKDIR="$(mktemp -d /tmp/qdb-pg-e2e.XXXXXX)" +SOCKET_DIR="$WORKDIR/pgsocket" +PGDATA="$WORKDIR/pgdata" +QDB_SOCKET="$WORKDIR/qdb.sock" +STATE="$WORKDIR/jobs.sqlite3" +PORT="${QDB_PG_PORT:-55432}" +SIDECAR_PID="" + +cleanup() { + if [[ -n "$SIDECAR_PID" ]]; then + kill "$SIDECAR_PID" 2>/dev/null || true + wait "$SIDECAR_PID" 2>/dev/null || true + fi + if [[ -d "$PGDATA" ]]; then "$PG_BIN/pg_ctl" -D "$PGDATA" stop -m fast >/dev/null 2>&1 || true; fi + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +make -C "$ROOT/pg_extension" install "PG_CONFIG=$PG_CONFIG" +mkdir -m 700 "$SOCKET_DIR" +"$PG_BIN/initdb" -D "$PGDATA" --no-locale -E UTF8 >/dev/null +cat >> "$PGDATA/postgresql.conf" <"$WORKDIR/sidecar.log" 2>&1 & +SIDECAR_PID="$!" +for _ in $(seq 1 100); do + [[ -S "$QDB_SOCKET" ]] && break + if ! kill -0 "$SIDECAR_PID" 2>/dev/null; then + cat "$WORKDIR/sidecar.log" >&2 + exit 1 + fi + sleep 0.02 +done +if [[ ! -S "$QDB_SOCKET" ]]; then + echo "sidecar did not create its Unix socket" >&2 + exit 1 +fi +"$PG_BIN/pg_ctl" -D "$PGDATA" -l "$WORKDIR/postgres.log" start >/dev/null + +"$PG_BIN/psql" -X -v ON_ERROR_STOP=1 -h "$SOCKET_DIR" -p "$PORT" -d postgres <<'SQL' +CREATE EXTENSION quantum_db VERSION '0.1.0'; +ALTER EXTENSION quantum_db UPDATE TO '0.1.1'; +SELECT extversion = '0.1.1' AS extension_upgraded FROM pg_extension WHERE extname = 'quantum_db'; +CREATE TABLE unsafe_keys(key_text text, value integer); +INSERT INTO unsafe_keys VALUES (NULL, 1), (repeat('x', 1024), 2); +DO $$ BEGIN + PERFORM qdb_register_dataset('unsafe_keys', 'key_text', ARRAY['value']); + RAISE EXCEPTION 'unsafe key registration unexpectedly succeeded'; +EXCEPTION WHEN SQLSTATE '22023' THEN NULL; +END $$; +CREATE TABLE orders(id bigint PRIMARY KEY, region text NOT NULL, amount integer NOT NULL); +INSERT INTO orders +SELECT i, CASE WHEN i IN (1,3) THEN 'east' ELSE 'west' END, CASE WHEN i IN (1,3) THEN 10 ELSE 15 END +FROM generate_series(1,600) AS i; +SELECT qdb_register_dataset('orders', 'id', ARRAY['region','amount']) IS NOT NULL AS registered; +SELECT kind, version IS NOT NULL AS versioned, checksum IS NOT NULL AS checksummed FROM qdb.artifacts; +CREATE TEMP TABLE qdb_smoke_submit AS +SELECT qdb_submit('SELECT * FROM orders WHERE region = ''east''', '{"wait":true,"timeout_ms":1000}'::jsonb) AS response; +SELECT j.status AS mirrored_job_status, j.request->>'sql' AS mirrored_sql +FROM qdb_smoke_submit AS s JOIN qdb.jobs AS j ON j.job_id=(s.response->>'job_id')::uuid; + +-- One local, same-job integration of all software layers. It deliberately +-- omits allow_hardware, so a configured QCloud credential cannot consume quota: +-- the Grover telemetry is ideal/local while TN, Qute C0, and PostgreSQL all run. +CREATE TABLE qdb_hybrid_probe(id bigint PRIMARY KEY, flag boolean NOT NULL, segment integer NOT NULL); +INSERT INTO qdb_hybrid_probe +SELECT i, i = 0, CASE WHEN i = 0 THEN 1 ELSE 0 END FROM generate_series(0, 7) AS i; +SELECT qdb_register_dataset('qdb_hybrid_probe', 'id', ARRAY['flag','segment']) IS NOT NULL AS hybrid_registered; +CREATE TEMP TABLE qdb_hybrid_submit AS +SELECT qdb_submit( + '/*+ QDB_HYBRID */ SELECT * FROM qdb_hybrid_probe WHERE flag = true AND segment = 1', + jsonb_build_object( + 'wait', true, 'timeout_ms', 1000, 'run_qpanda_probe', true, 'run_qute_validation', true, + 'execution_origin', 'postgres_extension', + 'snapshot_version', (SELECT snapshot_version FROM qdb.datasets WHERE relation_oid = 'qdb_hybrid_probe'::regclass) + ) +) AS response; +SELECT qdb_result((response->>'job_id')::uuid)->'result'->'accelerator'->'qpanda3'->'metadata'->'execution_components' AS hybrid_components +FROM qdb_hybrid_submit; +SELECT qdb_result((response->>'job_id')::uuid)->'result'->'qute_validation' AS hybrid_qute_validation +FROM qdb_hybrid_submit; +EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) +SELECT array_agg(id ORDER BY id) FROM qdb_hybrid_probe WHERE flag = true AND segment = 1; +SELECT array_agg(id ORDER BY id) AS exact_result FROM qdb_hybrid_probe WHERE flag = true AND segment = 1; +SQL + +# A candidate list without the PostgreSQL-recorded artifact checksum is advisory only. +# Replace the sidecar record with an intentionally mismatched checksum and prove that the +# Custom Scan falls back to native PostgreSQL filtering rather than risk a false negative. +SNAPSHOT_VERSION="$("$PG_BIN/psql" -X -At -h "$SOCKET_DIR" -p "$PORT" -d postgres -c "SELECT snapshot_version FROM qdb.datasets WHERE relation_oid = 'orders'::regclass")" +QDB_SOCKET="$QDB_SOCKET" SNAPSHOT_VERSION="$SNAPSHOT_VERSION" PYTHONPATH="$ROOT" python3 - <<'PY' +import os +from qdb.client import call + +rows = [ + {"key": i, "values": {"region": "east" if i in (1, 3) else "west", "amount": 10 if i in (1, 3) else 15}} + for i in range(1, 601) +] +call(os.environ["QDB_SOCKET"], "dataset.register", { + "relation": "public.orders", "snapshot_version": os.environ["SNAPSHOT_VERSION"], "key_column": "id", + "artifact_checksum": "0" * 32, "rows": rows, +}) +PY + +"$PG_BIN/psql" -X -v ON_ERROR_STOP=1 -h "$SOCKET_DIR" -p "$PORT" -d postgres <<'SQL' +EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) +SELECT array_agg(id ORDER BY id) FROM orders WHERE amount = 10; +SELECT array_agg(id ORDER BY id) AS exact_result_after_checksum_mismatch FROM orders WHERE amount = 10; +SQL diff --git a/systems/quantum_db/scripts/qdb_postgres_hybrid_probe.sh b/systems/quantum_db/scripts/qdb_postgres_hybrid_probe.sh new file mode 100644 index 0000000..7767288 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_postgres_hybrid_probe.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Explicitly authorized full-stack probe: +# PostgreSQL dataset registration -> sidecar hybrid TN/Qute/Grover job -> PostgreSQL Custom Scan recheck. +# It creates a disposable PostgreSQL cluster and submits exactly one 3-qubit Wukong job. +set -euo pipefail + +: "${ORIGINQC_API_KEY:?Set ORIGINQC_API_KEY in the shell before running this script.}" +: "${QDB_ALLOW_HARDWARE:?Set QDB_ALLOW_HARDWARE=1 to explicitly authorize this one Wukong submission.}" +[[ "$QDB_ALLOW_HARDWARE" == "1" ]] || { echo "QDB_ALLOW_HARDWARE must be exactly 1" >&2; exit 2; } + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PYTHON="${QDB_PYTHON:-python3}" +PG_CONFIG="${PG_CONFIG:-pg_config}" +PG_BIN="$($PG_CONFIG --bindir)" +SHOTS="${QDB_WUKONG_SHOTS:-256}" +ITERATIONS="${QDB_GROVER_ITERATIONS:-1}" +PORT="${QDB_PG_PORT:-55434}" +[[ "$SHOTS" =~ ^[1-9][0-9]*$ ]] || { echo "QDB_WUKONG_SHOTS must be a positive integer" >&2; exit 2; } +[[ "$ITERATIONS" == "1" || "$ITERATIONS" == "2" ]] || { echo "QDB_GROVER_ITERATIONS must be 1 or 2" >&2; exit 2; } + +WORKDIR="$(mktemp -d /tmp/qdb-pg-hybrid.XXXXXX)" +SOCKET_DIR="$WORKDIR/pgsocket" +PGDATA="$WORKDIR/pgdata" +QDB_SOCKET="$WORKDIR/qdb.sock" +STATE="$WORKDIR/jobs.sqlite3" +SIDECAR_PID="" + +cleanup() { + if [[ -n "$SIDECAR_PID" ]]; then kill "$SIDECAR_PID" 2>/dev/null || true; wait "$SIDECAR_PID" 2>/dev/null || true; fi + if [[ -d "$PGDATA" ]]; then "$PG_BIN/pg_ctl" -D "$PGDATA" stop -m fast >/dev/null 2>&1 || true; fi + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +make -C "$ROOT/pg_extension" install "PG_CONFIG=$PG_CONFIG" +mkdir -m 700 "$SOCKET_DIR" +"$PG_BIN/initdb" -D "$PGDATA" --no-locale -E UTF8 >/dev/null +cat >> "$PGDATA/postgresql.conf" <"$WORKDIR/sidecar.log" 2>&1 & +SIDECAR_PID="$!" +for _ in $(seq 1 100); do + [[ -S "$QDB_SOCKET" ]] && break + if ! kill -0 "$SIDECAR_PID" 2>/dev/null; then sed -E 's/[A-Za-z0-9]{32,}/[REDACTED]/g' "$WORKDIR/sidecar.log" >&2; exit 1; fi + sleep 0.02 +done +[[ -S "$QDB_SOCKET" ]] || { echo "sidecar did not create its Unix socket" >&2; exit 1; } +"$PG_BIN/pg_ctl" -D "$PGDATA" -l "$WORKDIR/postgres.log" start >/dev/null + +"$PG_BIN/psql" -X -v ON_ERROR_STOP=1 -h "$SOCKET_DIR" -p "$PORT" -d postgres \ + -v shots="$SHOTS" -v iterations="$ITERATIONS" <<'SQL' +CREATE EXTENSION quantum_db VERSION '0.1.1'; +CREATE TABLE qdb_probe(id bigint PRIMARY KEY, flag boolean NOT NULL, segment integer NOT NULL); +INSERT INTO qdb_probe +SELECT n, n = 0, CASE WHEN n = 0 THEN 1 ELSE 0 END FROM generate_series(0, 7) AS n; +SELECT qdb_register_dataset('qdb_probe', 'id', ARRAY['flag','segment']) IS NOT NULL AS registered; + +CREATE TEMP TABLE qdb_hybrid_job AS +SELECT qdb_submit( + '/*+ QDB_HYBRID */ SELECT * FROM qdb_probe WHERE flag = true AND segment = 1', + jsonb_build_object( + 'run_qpanda_probe', true, + 'run_qute_validation', true, + 'allow_hardware', true, + 'wait_for_hardware_result', true, + 'wait', true, + 'timeout_ms', 600000, + 'shots', :shots, + 'hardware_grover_iterations', :iterations, + 'execution_origin', 'postgres_extension')) AS response; + +-- Fetch once, then make a finished QCloud measurement receipt a hard acceptance +-- condition. The component trace alone is insufficient evidence of a real run. +CREATE TEMP TABLE qdb_hybrid_result AS +SELECT qdb_result((response->>'job_id')::uuid) AS response FROM qdb_hybrid_job; + +SELECT response->'result'->'accelerator'->'qpanda3'->'metadata'->'execution_components' + AS execution_components +FROM qdb_hybrid_result; +SELECT response->'result'->'qute_validation' AS qute_validation FROM qdb_hybrid_result; +SELECT jsonb_build_object( + 'execution_mode', response#>>'{result,accelerator,qpanda3,metadata,execution_mode}', + 'hardware_job_id', response#>>'{result,accelerator,qpanda3,metadata,job_id}', + 'status', response#>>'{result,accelerator,qpanda3,metadata,status}', + 'counts_source', response#>>'{result,accelerator,qpanda3,metadata,counts_source}', + 'shots', response#>'{result,accelerator,qpanda3,shots}', + 'counts', response#>'{result,accelerator,qpanda3,counts}', + 'expected_marked_state', response#>>'{result,accelerator,qpanda3,metadata,expected_marked_state}', + 'ideal_marked_probability', response#>'{result,accelerator,qpanda3,metadata,ideal_marked_probability}' +) AS wukong_measurement_receipt +FROM qdb_hybrid_result; +DO $$ +DECLARE probe jsonb; observed_shots integer; +BEGIN + SELECT response#>'{result,accelerator,qpanda3}' INTO probe FROM qdb_hybrid_result; + IF COALESCE((probe->>'executed')::boolean, false) IS NOT TRUE THEN + RAISE EXCEPTION 'Wukong probe did not execute: %', COALESCE(probe->>'reason', 'missing qpanda3 result'); + END IF; + IF probe#>>'{metadata,execution_mode}' <> 'qcloud_sdk' THEN + RAISE EXCEPTION 'expected qcloud_sdk execution, got %', probe#>>'{metadata,execution_mode}'; + END IF; + IF probe#>>'{metadata,status}' <> 'JobStatus.FINISHED' OR COALESCE(probe#>>'{metadata,job_id}', '') = '' THEN + RAISE EXCEPTION 'Wukong job did not finish with a cloud job id: %', probe->'metadata'; + END IF; + IF probe#>>'{metadata,counts_source}' <> 'binary' THEN + RAISE EXCEPTION 'Wukong did not return binary counts: %', probe->'metadata'; + END IF; + SELECT COALESCE(sum(value::integer), 0) INTO observed_shots + FROM jsonb_each_text(COALESCE(probe->'counts', '{}'::jsonb)); + IF observed_shots <> COALESCE((probe->>'shots')::integer, -1) OR observed_shots < 1 THEN + RAISE EXCEPTION 'invalid Wukong counts: observed %, expected %', observed_shots, probe->>'shots'; + END IF; +END $$; +EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) +SELECT id FROM qdb_probe WHERE flag = true AND segment = 1; +SELECT array_agg(id ORDER BY id) AS exact_postgres_result +FROM qdb_probe WHERE flag = true AND segment = 1; +SQL diff --git a/systems/quantum_db/scripts/qdb_sidecar.py b/systems/quantum_db/scripts/qdb_sidecar.py new file mode 100644 index 0000000..e477365 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_sidecar.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from qdb.server import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/scripts/qdb_wukong_preflight.sh b/systems/quantum_db/scripts/qdb_wukong_preflight.sh new file mode 100644 index 0000000..e579570 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_wukong_preflight.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run a non-submitting Wukong capability preflight. The credential is read +# exclusively from the caller's process environment and is never persisted. +set -euo pipefail + +: "${ORIGINQC_API_KEY:?Set ORIGINQC_API_KEY in the shell before running this script.}" + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PYTHON="${QDB_PYTHON:-python3}" +BACKEND="${QDB_QCLOUD_BACKEND:-WK_C180}" +WORKDIR="$(mktemp -d /tmp/qdb-wukong-preflight.XXXXXX)" +SOCKET="$WORKDIR/qdb.sock" +STATE="$WORKDIR/jobs.sqlite3" +PID="" + +cleanup() { + if [[ -n "$PID" ]]; then kill "$PID" 2>/dev/null || true; wait "$PID" 2>/dev/null || true; fi + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +( + cd "$ROOT" + PYTHONDONTWRITEBYTECODE=1 QDB_QCLOUD_BACKEND="$BACKEND" "$PYTHON" -m qdb.server --socket "$SOCKET" --state "$STATE" +) >"$WORKDIR/sidecar.log" 2>&1 & +PID="$!" +for _ in $(seq 1 100); do [[ -S "$SOCKET" ]] && break; sleep 0.02; done +if [[ ! -S "$SOCKET" ]]; then sed -E 's/[A-Za-z0-9]{32,}/[REDACTED]/g' "$WORKDIR/sidecar.log" >&2; exit 1; fi + +QDB_SOCKET="$SOCKET" PYTHONPATH="$ROOT" "$PYTHON" -c ' +import os +from qdb.client import call +print(call(os.environ["QDB_SOCKET"], "hardware.preflight", {"allow_hardware": True})) +' diff --git a/systems/quantum_db/scripts/qdb_wukong_probe.sh b/systems/quantum_db/scripts/qdb_wukong_probe.sh new file mode 100644 index 0000000..e48f467 --- /dev/null +++ b/systems/quantum_db/scripts/qdb_wukong_probe.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Submit one configurable-shot Wukong Grover probe and wait for the returned measurement result. +set -euo pipefail +: "${ORIGINQC_API_KEY:?Set ORIGINQC_API_KEY in the shell before running this script.}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)"; PYTHON="${QDB_PYTHON:-python3}" +SHOTS="${QDB_WUKONG_SHOTS:-64}"; ITERATIONS="${QDB_GROVER_ITERATIONS:-2}" +USE_STORAGE="${QDB_PROBE_STORAGE:-0}" +COMPONENTS="${QDB_PROBE_COMPONENTS:-quantum}" +USE_QUTE="${QDB_PROBE_QUTE:-0}" +[[ "$SHOTS" =~ ^[1-9][0-9]*$ ]] || { echo "QDB_WUKONG_SHOTS must be a positive integer" >&2; exit 2; } +[[ "$ITERATIONS" == "1" || "$ITERATIONS" == "2" ]] || { echo "QDB_GROVER_ITERATIONS must be 1 or 2" >&2; exit 2; } +[[ "$USE_STORAGE" == "0" || "$USE_STORAGE" == "1" ]] || { echo "QDB_PROBE_STORAGE must be 0 or 1" >&2; exit 2; } +[[ "$COMPONENTS" == "quantum" || "$COMPONENTS" == "hybrid" ]] || { echo "QDB_PROBE_COMPONENTS must be quantum or hybrid" >&2; exit 2; } +[[ "$USE_QUTE" == "0" || "$USE_QUTE" == "1" ]] || { echo "QDB_PROBE_QUTE must be 0 or 1" >&2; exit 2; } +WORKDIR="$(mktemp -d /tmp/qdb-wukong-probe.XXXXXX)"; SOCKET="$WORKDIR/qdb.sock"; PID="" +cleanup() { [[ -n "$PID" ]] && kill "$PID" 2>/dev/null || true; [[ -n "$PID" ]] && wait "$PID" 2>/dev/null || true; rm -rf "$WORKDIR"; } +trap cleanup EXIT +( + cd "$ROOT" + PYTHONDONTWRITEBYTECODE=1 QDB_QCLOUD_BACKEND="${QDB_QCLOUD_BACKEND:-WK_C180}" "$PYTHON" -m qdb.server --socket "$SOCKET" --state "$WORKDIR/jobs.sqlite3" +) >"$WORKDIR/sidecar.log" 2>&1 & PID="$!" +for _ in $(seq 1 100); do [[ -S "$SOCKET" ]] && break; sleep 0.02; done +[[ -S "$SOCKET" ]] || { sed -E 's/[A-Za-z0-9]{32,}/[REDACTED]/g' "$WORKDIR/sidecar.log" >&2; exit 1; } +QDB_SOCKET="$SOCKET" QDB_WUKONG_SHOTS="$SHOTS" QDB_GROVER_ITERATIONS="$ITERATIONS" QDB_PROBE_STORAGE="$USE_STORAGE" QDB_PROBE_COMPONENTS="$COMPONENTS" QDB_PROBE_QUTE="$USE_QUTE" PYTHONPATH="$ROOT" "$PYTHON" -c ' +import hashlib +import json +import os +from qdb.client import call +hybrid = os.environ["QDB_PROBE_COMPONENTS"] == "hybrid" +rows = [{"key": n, "values": {"flag": n == 0, "bucket": "target" if n == 0 else "other"}} for n in range(8)] +sql = ("/*+ QDB_HYBRID */ SELECT * FROM qdb_probe WHERE flag = true AND bucket = " + chr(39) + "target" + chr(39) + if hybrid else "/*+ QDB_QUANTUM */ SELECT * FROM qdb_probe WHERE flag = true") +params = { + "sql": sql, + "shots": int(os.environ["QDB_WUKONG_SHOTS"]), "hardware_grover_iterations": int(os.environ["QDB_GROVER_ITERATIONS"]), + "run_qpanda_probe": True, "wait_for_hardware_result": True, + "allow_hardware": True, + "wait": True, "timeout_ms": 600000, +} +if os.environ["QDB_PROBE_QUTE"] == "1": + params["run_qute_validation"] = True +if os.environ["QDB_PROBE_STORAGE"] == "1": + checksum = hashlib.md5(json.dumps(rows, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + call(os.environ["QDB_SOCKET"], "dataset.register", { + "relation": "qdb_probe", "snapshot_version": "probe-v1", "key_column": "id", + "artifact_checksum": checksum, "rows": rows, + }) + params["snapshot_version"] = "probe-v1" +else: + params["rows"] = rows +submitted = call(os.environ["QDB_SOCKET"], "submit", params, timeout=660.0) +result = call(os.environ["QDB_SOCKET"], "result", {"job_id": submitted["job_id"]}, timeout=60.0) +probe = result["result"]["accelerator"]["qpanda3"] +counts, metadata = probe["counts"], probe["metadata"] +target = str(metadata.get("expected_marked_state", "")) +target_count = int(counts.get(target, 0)) if target else 0 +observed_shots = sum(int(value) for value in counts.values()) +analysis = {} +if target and observed_shots: + probability = target_count / observed_shots + # Wilson 95% interval is well behaved for the small-shot probe. + z = 1.959963984540054 + denominator = 1 + z * z / observed_shots + centre = (probability + z * z / (2 * observed_shots)) / denominator + margin = z * ((probability * (1 - probability) / observed_shots + z * z / (4 * observed_shots * observed_shots)) ** 0.5) / denominator + uniform_probability = 1 / len(counts) if counts else 0.0 + analysis = { + "target_state": target, "target_count": target_count, "target_probability": probability, + "target_wilson_95": [max(0.0, centre - margin), min(1.0, centre + margin)], + "uniform_probability": uniform_probability, + "enrichment_vs_uniform": probability / uniform_probability if uniform_probability else None, + } +print({"job_id": submitted["job_id"], "shots": probe["shots"], "counts": counts, + "target_analysis": analysis, "metadata": metadata}) +' diff --git a/systems/quantum_db/scripts/qute_benchmark.py b/systems/quantum_db/scripts/qute_benchmark.py new file mode 100644 index 0000000..d736bea --- /dev/null +++ b/systems/quantum_db/scripts/qute_benchmark.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from qute.benchmark.runner import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/systems/quantum_db/scripts/tn_binary_native_vldb_plots.py b/systems/quantum_db/scripts/tn_binary_native_vldb_plots.py new file mode 100644 index 0000000..833ec82 --- /dev/null +++ b/systems/quantum_db/scripts/tn_binary_native_vldb_plots.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +import math +from pathlib import Path +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import FancyArrowPatch, Rectangle +from matplotlib.ticker import FormatStrFormatter, MaxNLocator + + +FACTORS = ("distribution_type", "distinct", "corr") +FACTOR_TITLES = { + "distribution_type": "Distribution Type", + "distinct": "Distinct Values", + "corr": "Correlation", +} +MODE_ORDER = ("mps_linear", "tree", "native_mapping") +MODE_LABELS = { + "mps_linear": "MPS-linear", + "tree": "Tree", + "native_mapping": "Native-binary", +} +MODE_INFO_COLORS = { + "mps_linear": "#845ec2", + "tree": "#ff6f91", + "native_mapping": "#ffc75f", +} +MODE_AUX_COLORS = { + "mps_linear": "#d65db1", + "tree": "#ff9671", + "native_mapping": "#f9f871", +} + + +def setup_style() -> None: + plt.rcParams.update( + { + "font.family": "DejaVu Sans", + "font.size": 9, + "axes.titlesize": 10, + "axes.labelsize": 9, + "legend.fontsize": 8, + "xtick.labelsize": 8, + "ytick.labelsize": 8, + "axes.spines.top": False, + "axes.spines.right": False, + "axes.linewidth": 0.8, + "grid.linewidth": 0.45, + "grid.alpha": 0.35, + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.bbox": "tight", + "savefig.pad_inches": 0.04, + } + ) + + +def variant_value(record: dict[str, Any], factor: str) -> Any: + return record["dataset"].get(factor, record["dataset"].get("path", "")) + + +def sort_values(values: set[Any]) -> list[Any]: + return sorted(values, key=lambda x: (str(type(x)), x)) + + +def mean(values: list[float]) -> float: + return sum(values) / max(1, len(values)) + + +def key_of(record: dict[str, Any]) -> tuple[str, str, str]: + return (record["factor"], record["dataset"]["path"], record["tn_mode"]) + + +def dataset_label(record: dict[str, Any]) -> str: + dataset = record["dataset"] + dist = dataset.get("distribution_type", "data") + header = "nh" if dataset.get("header_mode") == "nohead" else "h" + return ( + f"{dist}|cols{dataset.get('cols')}|d{dataset.get('distinct')}|" + f"c{dataset.get('corr')}|s{dataset.get('skew')}|{header}" + ) + + +def compact_count(value: Any) -> str: + try: + numeric = int(value) + except (TypeError, ValueError): + return str(value) + if numeric >= 1000 and numeric % 1000 == 0: + return f"{numeric // 1000}k" + return str(numeric) + + +def dataset_axis_label(label: str) -> str: + tokens = label.split("|") + if len(tokens) != 6: + return label + dist, cols, distinct, corr, skew, header = tokens + dist_short = {"uniform": "Uni", "skewed": "Skew"}.get(dist, dist) + header_short = header.upper() + cols_short = cols.removeprefix("cols") + distinct_short = compact_count(distinct.removeprefix("d")) + corr_short = corr.removeprefix("c") + skew_short = skew.removeprefix("s") + return f"{dist_short} / {header_short}\nCol{cols_short} D{distinct_short} C{corr_short} S{skew_short}" + + +def include_in_figures(record: dict[str, Any]) -> bool: + dataset = record["dataset"] + return int(dataset.get("distinct", -1)) != 100 and int(dataset.get("cols", -1)) != 4 + + +def slugify(text: str) -> str: + safe = [] + for ch in text.lower(): + safe.append(ch if ch.isalnum() else "_") + return "_".join("".join(safe).split("_")).strip("_") + + +def infer_load_model(records: list[dict[str, Any]]) -> dict[str, Any]: + first_by_mode = {mode: next((r for r in records if r["tn_mode"] == mode), None) for mode in MODE_ORDER} + mps_qubits = int(first_by_mode["mps_linear"]["prepare"]["qubits"]) if first_by_mode["mps_linear"] else 0 + tree_qubits = int(first_by_mode["tree"]["prepare"]["qubits"]) if first_by_mode["tree"] else 0 + native_qubits = int(first_by_mode["native_mapping"]["prepare"]["qubits"]) if first_by_mode["native_mapping"] else 0 + qinfo = dict(first_by_mode["native_mapping"].get("qubit_load_info", {})) if first_by_mode["native_mapping"] else {} + native_register_qubits = int(qinfo.get("native_register_qubits", 16)) + effective_scalar_count = int(qinfo.get("effective_scalar_count", max(1, native_qubits - native_register_qubits))) + index_qubits = max(1, int(math.ceil(math.log2(max(2, effective_scalar_count))))) + value_precision_qubits = int(qinfo.get("value_precision_qubits", max(0, native_register_qubits - index_qubits))) + return { + "effective_scalar_count": effective_scalar_count, + "index_qubits": index_qubits, + "value_precision_qubits": value_precision_qubits, + "mps_streaming_qubits": mps_qubits, + "tree_streaming_qubits": tree_qubits, + "native_register_qubits": native_register_qubits, + "native_qubits": native_qubits, + "mode_depths": { + mode: int(first_by_mode[mode]["prepare"]["gate_depth"]) if first_by_mode[mode] else None + for mode in MODE_ORDER + }, + } + + +def merge_completed_with_recalc(completed: dict[str, Any], recalc: dict[str, Any]) -> dict[str, Any]: + by_key = {key_of(r): r for r in recalc["records"]} + load_model = infer_load_model(recalc["records"]) + merged = json.loads(json.dumps(completed)) + for record in merged["records"]: + other = by_key.get(key_of(record)) + if not other: + continue + mode = record["tn_mode"] + materialized = load_model["effective_scalar_count"] if mode == "native_mapping" else 0 + record["metrics"] = other["metrics"] + record["prepare"] = other["prepare"] + record["hardware_cost"] = other["hardware_cost"] + record["qubit_load_info"] = other.get("qubit_load_info", {}) + qinfo = dict(other.get("qubit_load_info", {})) + record["compressed_table_load"] = { + "encoding": "binary", + "definition": ( + "mps_linear streams one tensor with site/physical/bond/value registers; " + "tree adds frontier control for parallel merge; native_mapping materializes every compressed parameter" + ), + "effective_scalar_count": load_model["effective_scalar_count"], + "index_qubits": load_model["index_qubits"], + "value_precision_qubits": load_model["value_precision_qubits"], + "mps_streaming_qubits": load_model["mps_streaming_qubits"], + "tree_streaming_qubits": load_model["tree_streaming_qubits"], + "native_register_qubits": load_model["native_register_qubits"], + "materialized_table_qubits": materialized, + "site_qubits": qinfo.get("site_qubits"), + "phys_qubits": qinfo.get("phys_qubits"), + "bond_qubits": qinfo.get("bond_qubits"), + "information_qubits": qinfo.get("information_qubits"), + "logical_auxiliary_qubits": qinfo.get("logical_auxiliary_qubits"), + "core_load_summary": qinfo.get("core_load_summary"), + "qubits": other["prepare"]["qubits"], + "gate_depth": other["prepare"]["gate_depth"], + } + merged["compressed_table_recalculation"] = { + "definition": ( + "MPS-linear streams the compressed TN-page table through site/physical/bond/value registers; " + "tree adds frontier control qubits to support shallower parallel merging; native_mapping directly " + "materializes the full compressed table." + ), + **load_model, + } + return merged + + +def aggregate_page_all(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in records: + if not include_in_figures(r): + continue + grouped[dataset_label(r)].append(r) + rows = [] + for label in sorted(grouped): + rs = grouped[label] + rows.append( + { + "variant": label, + "fidelity": mean([float(r["metrics"]["fidelity"]) for r in rs]), + "dense_compression": mean([float(r["metrics"]["dense_compression_ratio"]) for r in rs]), + } + ) + return rows + + +def aggregate_quantum_by_dataset(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for r in records: + if not include_in_figures(r): + continue + grouped[(dataset_label(r), r["tn_mode"])].append(r) + rows = [] + for v in sort_values({k[0] for k in grouped}): + for mode in MODE_ORDER: + rs = grouped.get((v, mode), []) + if not rs: + continue + info_values = [] + aux_values = [] + total_values = [] + for r in rs: + qinfo = dict(r.get("qubit_load_info", {})) + hardware = dict(r.get("hardware_cost", {})) + prepare_qubits = float(r["prepare"]["qubits"]) + physical_qubits = float(hardware.get("physical_qubits_used", prepare_qubits)) + routing_aux = max(0.0, physical_qubits - prepare_qubits) + info = float(qinfo.get("information_qubits", prepare_qubits)) + logical_aux = float(qinfo.get("logical_auxiliary_qubits", 0.0)) + aux = logical_aux + routing_aux + info_values.append(info) + aux_values.append(aux) + total_values.append(info + aux) + rows.append( + { + "variant": v, + "mode": mode, + "information_qubits": mean(info_values), + "auxiliary_qubits": mean(aux_values), + "qubits": mean(total_values), + "depth": mean([float(r["prepare"]["gate_depth"]) for r in rs]), + } + ) + return rows + + +def plot_page_all(records: list[dict[str, Any]], out_dir: Path) -> Path: + rows = aggregate_page_all(records) + labels = [dataset_axis_label(str(r["variant"])) for r in rows] + x = list(range(len(rows))) + fig, ax1 = plt.subplots(figsize=(7.2, 2.8), dpi=300) + ax2 = ax1.twinx() + fidelity_values = [r["fidelity"] for r in rows] + dense_values = [r["dense_compression"] for r in rows] + bars = ax1.bar(x, fidelity_values, width=0.58, color="#845ec2", label="Fidelity") + ax2.plot( + x, + dense_values, + color="#111111", + marker="s", + linewidth=1.5, + markersize=4.5, + markerfacecolor="white", + markeredgecolor="#111111", + markeredgewidth=0.8, + label="Dense compression", + ) + ax1.set_title("TN Page Build Across Datasets") + ax1.set_ylabel("Fidelity") + ax2.set_ylabel("Dense compression ratio") + ax1.set_ylim(0.8, 1.0) + ax1.set_yticks([round(0.8 + i * 0.04, 2) for i in range(6)]) + ax1.yaxis.set_major_formatter(FormatStrFormatter("%.2f")) + if dense_values: + dense_min = min(dense_values) + dense_max = max(dense_values) + if math.isclose(dense_min, dense_max): + margin = max(abs(dense_min) * 0.06, 1.0) + else: + margin = (dense_max - dense_min) * 0.12 + ax2.set_ylim(max(0.0, dense_min - margin), dense_max + margin) + ax2.yaxis.set_major_locator(MaxNLocator(nbins=5)) + ax2.yaxis.set_major_formatter(FormatStrFormatter("%.1f")) + ax1.set_xticks(x, labels, rotation=-45, ha="left", rotation_mode="anchor") + ax1.grid(axis="y") + for bar, value in zip(bars, fidelity_values): + ax1.text( + bar.get_x() + bar.get_width() / 2, + min(value + 0.006, 0.995), + f"{value:.3f}", + ha="center", + va="bottom", + fontsize=5.8, + color="#333333", + ) + ax1.text( + -0.055, + -0.018, + "//", + transform=ax1.transAxes, + ha="center", + va="center", + fontsize=9, + color="#333333", + clip_on=False, + ) + h1, l1 = ax1.get_legend_handles_labels() + h2, l2 = ax2.get_legend_handles_labels() + ax1.legend(h1 + h2, l1 + l2, frameon=False, loc="upper right", bbox_to_anchor=(1.0, 1.12)) + path = out_dir / "vldb_page_build_all_datasets.png" + fig.savefig(path) + plt.close(fig) + return path + + +def plot_quantum_by_dataset(records: list[dict[str, Any]], out_dir: Path) -> list[Path]: + rows = aggregate_quantum_by_dataset(records) + variants = sort_values({r["variant"] for r in rows}) + lookup = {(r["variant"], r["mode"]): r for r in rows} + paths: list[Path] = [] + for variant in variants: + x = list(range(len(MODE_ORDER))) + depth = [lookup[(variant, mode)]["depth"] for mode in MODE_ORDER] + info_qubits = [lookup[(variant, mode)]["information_qubits"] for mode in MODE_ORDER] + aux_qubits = [lookup[(variant, mode)]["auxiliary_qubits"] for mode in MODE_ORDER] + fig, ax1 = plt.subplots(figsize=(4.25, 2.45), dpi=300) + fig.subplots_adjust(right=0.70) + ax2 = ax1.twinx() + info_bars = ax1.bar(x, info_qubits, width=0.48, color=[MODE_INFO_COLORS[m] for m in MODE_ORDER], label="Information qubits") + aux_bars = ax1.bar(x, aux_qubits, width=0.48, bottom=info_qubits, color=[MODE_AUX_COLORS[m] for m in MODE_ORDER], label="Auxiliary qubits") + line = ax2.plot(x, depth, color="#333333", marker="o", linewidth=1.3, markersize=4, label="Gate depth") + ax1.set_title(f"Quantum Load: {variant}") + ax1.set_ylabel("Qubits") + ax2.set_ylabel("Gate depth") + ax1.set_xticks(x, [MODE_LABELS[m] for m in MODE_ORDER]) + ax1.grid(axis="y") + handles = [] + labels = [] + for mode, info_bar, aux_bar in zip(MODE_ORDER, info_bars, aux_bars): + handles.extend([info_bar, aux_bar]) + labels.extend([f"{MODE_LABELS[mode]} info", f"{MODE_LABELS[mode]} aux"]) + handles += line + labels += ["Gate depth"] + ax1.legend( + handles, + labels, + frameon=False, + framealpha=0.0, + edgecolor="none", + loc="upper left", + bbox_to_anchor=(0.04, 0.98), + ncols=2, + fontsize=5.2, + ) + path = out_dir / f"vldb_quantum_load_{slugify(variant)}.png" + fig.savefig(path) + plt.close(fig) + paths.append(path) + return paths + + +def plot_native_schematic(out_dir: Path, model: dict[str, Any]) -> Path: + effective_scalars = int(model.get("effective_scalar_count", 0)) + native_qubits = int(model.get("native_qubits", 0)) + fig, ax = plt.subplots(figsize=(5.6, 2.4), dpi=300) + ax.axis("off") + boxes = [ + (0.04, 0.58, 0.22, 0.22, f"TN Page\n{effective_scalars} compact params"), + (0.36, 0.63, 0.25, 0.18, "MPS stream\nsite/phys/bond/value"), + (0.36, 0.23, 0.25, 0.18, f"Native materializes\n{effective_scalars} table qubits"), + (0.74, 0.63, 0.20, 0.18, "Tree stream\nMPS + frontier"), + (0.74, 0.23, 0.20, 0.18, f"Native load\n{native_qubits} qubits"), + ] + for x, y, w, h, text in boxes: + ax.add_patch(Rectangle((x, y), w, h, linewidth=0.9, edgecolor="#333333", facecolor="#F7F7F7")) + ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=8) + arrows = [ + ((0.26, 0.69), (0.36, 0.72)), + ((0.26, 0.69), (0.36, 0.32)), + ((0.61, 0.72), (0.74, 0.72)), + ((0.61, 0.32), (0.74, 0.32)), + ] + for a, b in arrows: + ax.add_patch(FancyArrowPatch(a, b, arrowstyle="-|>", mutation_scale=10, linewidth=0.9, color="#333333")) + ax.text(0.5, 0.93, "Streaming TN Load vs. Native Table Materialization", ha="center", va="center", fontsize=10, weight="bold") + ax.text(0.5, 0.08, "MPS and Tree use different streaming widths; native materializes all compact parameters", ha="center", fontsize=8) + path = out_dir / "vldb_stream_vs_native_loading_schematic.png" + fig.savefig(path) + plt.close(fig) + return path + + +def write_explanation(out: Path, merged: dict[str, Any], figures: list[Path]) -> None: + lines = [ + "# Compressed Parameter Table Recalculation and VLDB Figures", + "", + "## Compression Definitions", + "", + "- `compression_ratio` is reported as compression benefit: `Bytes_raw / Bytes_compressed`; values greater than 1 mean the TN page is smaller than raw data.", + "- `Bytes_compressed = 2 * (sum(physical_dims) + sum(retained_bond_ranks)) * 8`. The factor 2 accounts for retained value and compact index/metadata streams.", + "- `dense_compression_ratio` uses the same benefit convention for the compact dense page plotted in the figures.", + "- `dense_materialization_ratio` is kept only as a diagnostic for explicit dense MPS tensor expansion; it can be much larger than 1 and is not used as the compression result.", + "", + "## Why qubits were previously fixed at 48", + "", + "- The previous quantum IR mapped each TN site to `ceil(log2(phys_dim))` qubits. With `num_sites=16` and `phys_dim=8`, all modes used `16 * 3 = 48` data-loading qubits.", + "- That was not a faithful native-mapping cost model: native did not materialize all compressed-table entries, so it was effectively under-loading the data.", + "- The corrected IR gives MPS-linear and tree different streaming widths: MPS keeps site/physical/left-bond/right-bond/value registers; tree adds frontier control for lower-depth merging.", + "- The corrected native mapping materializes the full compressed table: `effective_scalar_count` table qubits plus the same address/value register. In the representative page, this is `564 + 18 = 582` qubits.", + "", + "## Generated Figures", + ] + for p in figures: + lines.append(f"- `{p}`") + lines += [ + "", + "## Recalculated Mode Costs", + "| mode | information qubits | auxiliary qubits | total qubits | prepare depth | gate counts | routed depth |", + "|---|---:|---:|---:|---:|---|---:|", + ] + for mode in MODE_ORDER: + r = next(x for x in merged["records"] if x["tn_mode"] == mode) + qinfo = dict(r.get("qubit_load_info", {})) + prepare_qubits = int(r["prepare"]["qubits"]) + physical_qubits = int(r["hardware_cost"].get("physical_qubits_used", prepare_qubits)) + routing_aux = max(0, physical_qubits - prepare_qubits) + info_qubits = int(qinfo.get("information_qubits", prepare_qubits)) + aux_qubits = int(qinfo.get("logical_auxiliary_qubits", 0)) + routing_aux + lines.append( + f"| {MODE_LABELS[mode]} | {info_qubits} | {aux_qubits} | {info_qubits + aux_qubits} | " + f"{r['prepare']['gate_depth']} | `{json.dumps(r['prepare']['gate_counts'], sort_keys=True)}` | " + f"{r['hardware_cost']['routed_depth']} |" + ) + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--complete", default="artifacts/tn_single_factor_qcloud_48jobs_complete.json") + parser.add_argument("--recalc", default="artifacts/tn_single_factor_stream_vs_native_recalc.json") + parser.add_argument("--out-json", default="artifacts/tn_single_factor_stream_vs_native_merged.json") + parser.add_argument("--out-report", default="artifacts/tn_stream_vs_native_vldb_report.md") + parser.add_argument("--fig-dir", default="artifacts/figures_vldb_stream_native") + args = parser.parse_args() + + setup_style() + completed = json.loads(Path(args.complete).read_text(encoding="utf-8")) + recalc = json.loads(Path(args.recalc).read_text(encoding="utf-8")) + merged = merge_completed_with_recalc(completed, recalc) + Path(args.out_json).write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8") + fig_dir = Path(args.fig_dir) + fig_dir.mkdir(parents=True, exist_ok=True) + figures: list[Path] = [] + figures.append(plot_page_all(merged["records"], fig_dir)) + figures.extend(plot_quantum_by_dataset(merged["records"], fig_dir)) + figures.append(plot_native_schematic(fig_dir, merged["compressed_table_recalculation"])) + write_explanation(Path(args.out_report), merged, figures) + print(json.dumps({"figures": [str(p) for p in figures], "out_json": args.out_json, "out_report": args.out_report}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_closed_test.py b/systems/quantum_db/scripts/tn_closed_test.py new file mode 100644 index 0000000..3904d8a --- /dev/null +++ b/systems/quantum_db/scripts/tn_closed_test.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import argparse +import json +import random +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import sys + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from data_plane.tn.compute import TNCompute +from data_plane.tn.exproter import TNExporter +from data_plane.tn.cache import TNMultiLayerCache +from data_plane.tn.page_manager import TNPageManager + + +@dataclass +class CaseResult: + dataset: str + keep_ratio: float + chi_max: int + trunc_eps: float + status: str + num_tensors: int = 0 + bond_dims: list[int] | None = None + locality: float | None = None + cache_hit_rate: float | None = None + unified_score: float | None = None + plan_cache_hit: bool = False + artifact_cache_hit: bool = False + export_ok: bool = False + detail: str = "" + + +def generate_samples(kind: str, n: int, rows: int, cols: int, seed: int) -> list[list[list[float]]]: + rng = random.Random(seed) + samples: list[list[list[float]]] = [] + for _ in range(n): + grid: list[list[float]] = [[0.0 for _ in range(cols)] for _ in range(rows)] + if kind == "gaussian_weak": + for r in range(rows): + for c in range(cols): + grid[r][c] = rng.gauss(0.0, 1.0) + elif kind == "local_strong": + for r in range(rows): + prev = rng.gauss(0.0, 1.0) + for c in range(cols): + cur = 0.85 * prev + 0.15 * rng.gauss(0.0, 1.0) + grid[r][c] = cur + prev = cur + else: + raise ValueError(f"unknown dataset kind: {kind}") + samples.append(grid) + return samples + + +def _quality_annotation(page_locality: float, keep_ratio: float, chi_max: int) -> dict[str, float]: + # 统一打分输入：utility越大越好，其余成本越小越好。 + return { + "query_utility": max(0.0, min(1.0, 0.4 + 0.6 * page_locality)), + "approximation_error": max(0.0, min(1.0, 1.0 - keep_ratio)), + "reconstruction_cost": max(0.0, min(1.0, chi_max / 16.0)), + "fidelity_loss": max(0.0, min(1.0, 0.6 - 0.4 * keep_ratio)), + "materialization_latency": max(0.0, min(1.0, 0.1 + chi_max / 20.0)), + } + + +def run_case( + dataset: str, + samples: list[Any], + *, + keep_ratio: float, + chi_max: int, + trunc_eps: float, + num_sites: int, + phys_dim: int, + seed: int, +) -> CaseResult: + compute = TNCompute(seed=seed) + manager = TNPageManager(cache=TNMultiLayerCache()) + exporter = TNExporter() + layered_cache = TNMultiLayerCache() + + try: + query_text = f"SELECT tn_plan FROM {dataset} WHERE keep={keep_ratio}" + q_entry = layered_cache.query.put(query_text, parser_version="v1") + + page_id = f"{dataset}-k{keep_ratio}-chi{chi_max}-eps{trunc_eps}" + page = compute.fit_mps_from_samples( + samples, + page_id=page_id, + num_sites=num_sites, + phys_dim=phys_dim, + chi_max=chi_max, + trunc_eps=trunc_eps, + keep_ratio=keep_ratio, + ) + manager.create(page) + _ = manager.load(page.page_id) + _ = manager.load(page.page_id) # 增加 cache hit + + quality = _quality_annotation(float(page.stats.get("locality", 0.0)), keep_ratio, chi_max) + plan_entry = layered_cache.plan.put( + query_fingerprint=q_entry.fingerprint, + tn_graph={"nodes": num_sites, "edges": max(num_sites - 1, 0)}, + rewrite_result={"rule_set": "storel-lite", "rewrite_steps": 2}, + contraction_order=list(range(num_sites)), + slicing_strategy={"slice_dim": 1, "slice_size": 4}, + physical_layout_choice={"backend": "simulator", "layout": "line"}, + execution_path=["query-cache", "plan-cache", "artifact-cache", "execute"], + quality_annotation=quality, + ) + + artifact_entry = layered_cache.artifact.put( + artifact_type="tn_page", + artifact_id=page.page_id, + plan_id=plan_entry.plan_id, + payload=page.summary(), + ) + + plan_hit = layered_cache.plan.get(plan_entry.plan_id) is not None + artifact_hit = layered_cache.artifact.get(artifact_type="tn_page", artifact_id=artifact_entry.artifact_id) is not None + + page.update_stats(unified_score=plan_entry.score) + + packet = exporter.export( + page, + target_backend="simulator", + allowed_encodings=["fft_topk"], + max_two_qubit_depth=128, + ) + exporter.validate(packet) + + cache_stats = manager.cache.snapshot() + return CaseResult( + dataset=dataset, + keep_ratio=keep_ratio, + chi_max=chi_max, + trunc_eps=trunc_eps, + status="ok", + num_tensors=len(page.tensors), + bond_dims=list(page.metadata.bond_dims), + locality=page.stats.get("locality"), + cache_hit_rate=cache_stats.get("hit_rate"), + unified_score=plan_entry.score, + plan_cache_hit=plan_hit, + artifact_cache_hit=artifact_hit, + export_ok=True, + ) + except Exception as exc: # 测试脚本层保留异常信息 + return CaseResult( + dataset=dataset, + keep_ratio=keep_ratio, + chi_max=chi_max, + trunc_eps=trunc_eps, + status="error", + detail=f"{type(exc).__name__}: {exc}", + ) + + +def run_negative_cases(seed: int) -> list[CaseResult]: + compute = TNCompute(seed=seed) + out: list[CaseResult] = [] + + negatives = [ + ("invalid_empty", [], "empty samples"), + ("invalid_shape", [[[1.0]], [[1.0, 2.0]]], "inconsistent flattened size"), + ] + for name, samples, note in negatives: + try: + compute.fit_mps_from_samples(samples, page_id=name, num_sites=4) + out.append(CaseResult(dataset=name, keep_ratio=0.25, chi_max=16, trunc_eps=1e-4, status="unexpected_pass", detail=note)) + except Exception as exc: + out.append( + CaseResult( + dataset=name, + keep_ratio=0.25, + chi_max=16, + trunc_eps=1e-4, + status="expected_error", + detail=f"{type(exc).__name__}: {exc}", + ) + ) + return out + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Closed test harness for TN page modules") + parser.add_argument("--num-samples", type=int, default=64) + parser.add_argument("--rows", type=int, default=4) + parser.add_argument("--cols", type=int, default=4) + parser.add_argument("--num-sites", type=int, default=8) + parser.add_argument("--phys-dim", type=int, default=8) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--output", type=Path, default=None, help="Optional JSONL output path") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + datasets = { + "gaussian_weak": generate_samples("gaussian_weak", args.num_samples, args.rows, args.cols, args.seed), + "local_strong": generate_samples("local_strong", args.num_samples, args.rows, args.cols, args.seed + 1), + } + + keep_ratios = [0.1, 0.25, 0.5] + chi_values = [4, 8, 16] + trunc_values = [1e-2, 1e-4] + + results: list[CaseResult] = [] + for dataset_name, samples in datasets.items(): + for keep_ratio in keep_ratios: + for chi_max in chi_values: + for trunc_eps in trunc_values: + results.append( + run_case( + dataset_name, + samples, + keep_ratio=keep_ratio, + chi_max=chi_max, + trunc_eps=trunc_eps, + num_sites=args.num_sites, + phys_dim=args.phys_dim, + seed=args.seed, + ) + ) + + results.extend(run_negative_cases(args.seed)) + + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as fp: + for row in results: + fp.write(json.dumps(asdict(row), ensure_ascii=False) + "\n") + + print( + "dataset\tstatus\tkeep_ratio\tchi_max\ttrunc_eps\tnum_tensors\tlocality" + "\tcache_hit_rate\tunified_score\tplan_cache_hit\tartifact_cache_hit\tdetail" + ) + for row in results: + print( + f"{row.dataset}\t{row.status}\t{row.keep_ratio}\t{row.chi_max}\t{row.trunc_eps}" + f"\t{row.num_tensors}\t{row.locality}\t{row.cache_hit_rate}\t{row.unified_score}" + f"\t{row.plan_cache_hit}\t{row.artifact_cache_hit}\t{row.detail}" + ) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_contraction_smoke.py b/systems/quantum_db/scripts/tn_contraction_smoke.py new file mode 100644 index 0000000..a409b82 --- /dev/null +++ b/systems/quantum_db/scripts/tn_contraction_smoke.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from data_plane.tn.compute import TNCompute +from data_plane.tn.contraction import PhysicalTopologyGraph + + +def main() -> None: + topo = PhysicalTopologyGraph( + nodes=[ + {"id": "q00", "coord": (0, 0), "degree": 2, "neighbors": [(0, 1), (1, 0)], "partition_label": "p0"}, + {"id": "q01", "coord": (0, 1), "degree": 2, "neighbors": [(0, 0), (1, 1)], "partition_label": "p0"}, + {"id": "q10", "coord": (1, 0), "degree": 2, "neighbors": [(0, 0), (1, 1)], "partition_label": "p1"}, + {"id": "q11", "coord": (1, 1), "degree": 2, "neighbors": [(0, 1), (1, 0)], "partition_label": "p1"}, + ], + edges=[ + {"qubits": ("q00", "q01"), "quality": 0.99, "cross_partition": False}, + {"qubits": ("q00", "q10"), "quality": 0.97, "cross_partition": True}, + {"qubits": ("q01", "q11"), "quality": 0.96, "cross_partition": True}, + {"qubits": ("q10", "q11"), "quality": 0.98, "cross_partition": False}, + ], + coord_map={"q00": (0, 0), "q01": (0, 1), "q10": (1, 0), "q11": (1, 1)}, + quality_map={"q00": 0.99, "q01": 0.98, "q10": 0.97, "q11": 0.96}, + ) + partition = {"q00": "p0", "q01": "p0", "q10": "p1", "q11": "p1"} + c = TNCompute(seed=7) + bundle_main = c.plan_contraction_order(topology=topo, partition=partition, mode="main") + bundle_compat = c.plan_contraction_order(topology=topo, partition=partition, mode="compat", window_size=2) + + print("main", bundle_main.strategy, len(bundle_main.local_orders), len(bundle_main.merge_plan.merge_levels), len(bundle_main.global_trace)) + print("compat", bundle_compat.strategy, len(bundle_compat.local_orders), len(bundle_compat.merge_plan.merge_levels), len(bundle_compat.global_trace)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_page_experiment.py b/systems/quantum_db/scripts/tn_page_experiment.py new file mode 100644 index 0000000..cda9a40 --- /dev/null +++ b/systems/quantum_db/scripts/tn_page_experiment.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +import argparse +import csv +import json +import random +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig, suggest_workload_config + +DEFAULT_DATASET = "cols_8_distinct_10000_corr_8_skew_15" +DEFAULT_FILE = "train-test-data/SyntheticDataset/cols_8_distinct_10000_corr_8_skew_15.csv" + + +@dataclass(frozen=True) +class Action: + num_sites: int + phys_dim: int + chi_max: int + keep_ratio: float + + +def load_samples(csv_path: Path, *, limit: int | None = None) -> list[list[float]]: + rows: list[list[float]] = [] + with csv_path.open("r", encoding="utf-8") as f: + reader = csv.reader(f) + for row in reader: + if not row: + continue + try: + rows.append([float(cell) for cell in row]) + except ValueError: + continue + if limit is not None and len(rows) >= limit: + break + if not rows: + raise ValueError(f"no numeric sample rows found in {csv_path}") + return rows + + +def estimate_storage_usage(page: Any) -> dict[str, int]: + tensor_count = 0 + dense_scalar_count = 0 + for tensor in page.tensors: + tensor_count += 1 + for left in tensor: + for phys in left: + dense_scalar_count += len(phys) + + physical_dims = tuple(getattr(page.metadata, "physical_dims", ())) + bond_dims = tuple(getattr(page.metadata, "bond_dims", ())) + # Dense storage counts the explicit tensor slots. Effective storage counts + # local marginals plus retained bond spectra, so it must scale with bond rank. + effective_scalar_count = sum(int(dim) for dim in physical_dims) + sum(int(dim) for dim in bond_dims) + if effective_scalar_count <= 0: + effective_scalar_count = dense_scalar_count + compact_scalar_count = 2 * effective_scalar_count + + return { + "tensor_count": tensor_count, + "dense_scalar_count": dense_scalar_count, + "dense_estimated_bytes": dense_scalar_count * 8, + "effective_scalar_count": effective_scalar_count, + "effective_estimated_bytes": effective_scalar_count * 8, + "compact_scalar_count": compact_scalar_count, + "compact_estimated_bytes": compact_scalar_count * 8, + "estimated_bytes": dense_scalar_count * 8, + "scalar_count": dense_scalar_count, + } + + +def calc_metrics(service: TNWorkloadService, page: Any, samples: list[list[float]], sample_count: int) -> dict[str, Any]: + sampled = service.sampling(page, num_samples=sample_count) + fidelity = service.fidelity(samples, sampled) + reconstruction_error = service.reconstruction_error_l1(samples, sampled) + storage_usage = estimate_storage_usage(page) + raw_bytes = sum(len(s) for s in samples) * 8 + dense_materialization_ratio = storage_usage["dense_estimated_bytes"] / max(raw_bytes, 1) + dense_materialization_compression_ratio = raw_bytes / max(storage_usage["dense_estimated_bytes"], 1) + effective_compression_ratio = raw_bytes / max(storage_usage["compact_estimated_bytes"], 1) + return { + "fidelity": fidelity, + "construction_error": reconstruction_error, + "reconstruction_error_l1": reconstruction_error, + "compression_ratio": effective_compression_ratio, + "dense_compression_ratio": effective_compression_ratio, + "effective_compression_ratio": effective_compression_ratio, + "dense_materialization_ratio": dense_materialization_ratio, + "dense_materialization_compression_ratio": dense_materialization_compression_ratio, + "storage_usage": storage_usage, + "raw_input_bytes": raw_bytes, + } + + +def run_once(dataset: str, samples: list[list[float]], cfg: WorkloadConfig, sample_count: int) -> dict[str, Any]: + service = TNWorkloadService(config=cfg) + t0 = time.perf_counter() + page = service.create_page(dataset=dataset, samples=samples) + build_times_ms = (time.perf_counter() - t0) * 1000.0 + metrics = calc_metrics(service, page, samples, sample_count) + version_info = service.page_manager.version_of(page.page_id) + return { + "config": { + "num_sites": cfg.num_sites, + "phys_dim": cfg.phys_dim, + "chi_max": cfg.chi_max, + "keep_ratio": cfg.keep_ratio, + "alu_components": list(cfg.alu_components), + }, + "metrics": {**metrics, "build_times_ms": build_times_ms}, + "page": service.page_manager.describe(page.page_id), + "page_version": {"version": version_info.version, "updated_at_ms": version_info.updated_at_ms}, + } + + +def score(metrics: dict[str, Any]) -> float: + # 越大越好：高 fidelity + 高压缩收益 - 低 L1 reconstruction error + return float(metrics["fidelity"]) + 0.2 * float(metrics["compression_ratio"]) - 0.5 * float(metrics["construction_error"]) + + +def rl_search(dataset: str, samples: list[list[float]], sample_count: int, episodes: int, seed: int) -> dict[str, Any]: + rng = random.Random(seed) + actions = [ + Action(n, p, c, k) + for n in (4, 8, 12, 16) + for p in (4, 8, 12) + for c in (8, 16, 24, 32) + for k in (0.2, 0.25, 0.33, 0.5) + ] + q: dict[Action, float] = {a: 0.0 for a in actions} + visits: dict[Action, int] = {a: 0 for a in actions} + epsilon, alpha = 0.2, 0.3 + history: list[dict[str, Any]] = [] + best_run: dict[str, Any] | None = None + + for ep in range(1, episodes + 1): + if rng.random() < epsilon: + action = rng.choice(actions) + else: + action = max(actions, key=lambda a: q[a]) + + run = run_once( + dataset, + samples, + WorkloadConfig(num_sites=action.num_sites, phys_dim=action.phys_dim, chi_max=action.chi_max, keep_ratio=action.keep_ratio), + sample_count, + ) + reward = score(run["metrics"]) + visits[action] += 1 + q[action] = q[action] + alpha * (reward - q[action]) + + record = { + "episode": ep, + "action": action.__dict__, + "reward": reward, + "fidelity": run["metrics"]["fidelity"], + "construction_error": run["metrics"]["construction_error"], + "compression_ratio": run["metrics"]["compression_ratio"], + "build_times_ms": run["metrics"]["build_times_ms"], + } + history.append(record) + if best_run is None or reward > score(best_run["metrics"]): + best_run = run + + assert best_run is not None + return {"episodes": episodes, "best": best_run, "history": history} + + +def save_learning_curve_svg(history: list[dict[str, Any]], out_path: Path) -> None: + w, h, pad = 900, 320, 30 + rewards = [float(x["reward"]) for x in history] + rmin, rmax = min(rewards), max(rewards) + span = (rmax - rmin) or 1.0 + points = [] + for i, r in enumerate(rewards): + x = pad + (w - 2 * pad) * (i / max(1, len(rewards) - 1)) + y = h - pad - (h - 2 * pad) * ((r - rmin) / span) + points.append(f"{x:.2f},{y:.2f}") + svg = f""" + + + + +RL reward curve +episodes={len(rewards)} rmin={rmin:.4f} rmax={rmax:.4f} +""" + out_path.write_text(svg, encoding="utf-8") + + +def run_experiment(**kwargs: Any) -> dict[str, Any]: + dataset = str(kwargs["dataset"]) + csv_path = Path(kwargs["csv_path"]) + sample_limit = int(kwargs["sample_limit"]) + sample_count = int(kwargs["sample_count"]) + learning_mode = str(kwargs["learning_mode"]) + + process_log: list[dict[str, Any]] = [] + + def log(stage: str, **payload: Any) -> None: + process_log.append({"timestamp_ms": int(time.time() * 1000), "stage": stage, **payload}) + + log("load_samples.start", file=str(csv_path), limit=sample_limit) + samples = load_samples(csv_path, limit=sample_limit) + log("load_samples.done", rows=len(samples), cols=len(samples[0])) + + if learning_mode == "auto": + cfg = suggest_workload_config(samples) + log("config.auto_suggested", config=cfg.__dict__ if hasattr(cfg, "__dict__") else { + "num_sites": cfg.num_sites, "phys_dim": cfg.phys_dim, "chi_max": cfg.chi_max, "keep_ratio": cfg.keep_ratio + }) + run = run_once(dataset, samples, cfg, sample_count) + return { + "dataset": dataset, + "data_file": str(csv_path), + "metrics_definition": { + "fidelity": "1 - normalized marginal histogram distance proxy", + "construction_error": "L1 reconstruction error", + "reconstruction_error_l1": "L1 reconstruction error between source marginal distribution and sampled reconstruction", + "compression_ratio": "raw_input_bytes / compact TN page bytes; compact bytes = 2 * (sum physical dimensions + sum retained bond ranks) * 8", + "dense_compression_ratio": "same compression-benefit convention for the compact dense page used in figures", + "effective_compression_ratio": "raw_input_bytes / compact TN page bytes", + "dense_materialization_ratio": "explicit dense MPS tensor bytes / raw_input_bytes; diagnostic only", + "build_times_ms": "TN page construction wall-clock time in milliseconds", + }, + "result": run, + "process_log": process_log, + } + + if learning_mode == "manual": + cfg = WorkloadConfig( + num_sites=int(kwargs["num_sites"]), + phys_dim=int(kwargs["phys_dim"]), + chi_max=int(kwargs["chi_max"]), + keep_ratio=float(kwargs["keep_ratio"]), + ) + run = run_once(dataset, samples, cfg, sample_count) + return { + "dataset": dataset, + "data_file": str(csv_path), + "metrics_definition": { + "fidelity": "1 - normalized marginal histogram distance proxy", + "construction_error": "L1 reconstruction error", + "reconstruction_error_l1": "L1 reconstruction error between source marginal distribution and sampled reconstruction", + "compression_ratio": "raw_input_bytes / compact TN page bytes; compact bytes = 2 * (sum physical dimensions + sum retained bond ranks) * 8", + "dense_compression_ratio": "same compression-benefit convention for the compact dense page used in figures", + "effective_compression_ratio": "raw_input_bytes / compact TN page bytes", + "dense_materialization_ratio": "explicit dense MPS tensor bytes / raw_input_bytes; diagnostic only", + "build_times_ms": "TN page construction wall-clock time in milliseconds", + }, + "result": run, + "process_log": process_log, + } + + # learning_mode == rl + episodes = int(kwargs["episodes"]) + seed = int(kwargs["seed"]) + rl = rl_search(dataset, samples, sample_count, episodes=episodes, seed=seed) + log("rl.finished", episodes=episodes, best_config=rl["best"]["config"], best_metrics=rl["best"]["metrics"]) + + chart_path = kwargs.get("chart_out") + if chart_path: + save_learning_curve_svg(rl["history"], Path(chart_path)) + log("rl.chart_written", path=str(chart_path)) + + return { + "dataset": dataset, + "data_file": str(csv_path), + "metrics_definition": { + "fidelity": "1 - normalized marginal histogram distance proxy", + "construction_error": "L1 reconstruction error", + "reconstruction_error_l1": "L1 reconstruction error between source marginal distribution and sampled reconstruction", + "compression_ratio": "raw_input_bytes / compact TN page bytes; compact bytes = 2 * (sum physical dimensions + sum retained bond ranks) * 8", + "dense_compression_ratio": "same compression-benefit convention for the compact dense page used in figures", + "effective_compression_ratio": "raw_input_bytes / compact TN page bytes", + "dense_materialization_ratio": "explicit dense MPS tensor bytes / raw_input_bytes; diagnostic only", + "build_times_ms": "TN page construction wall-clock time in milliseconds", + "reward": "fidelity + 0.2*compression_ratio - 0.5*construction_error", + }, + "learning": { + "mode": "rl", + "episodes": episodes, + "history": rl["history"], + "best": rl["best"], + }, + "process_log": process_log, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="TN-page 实验：构建并评估 fidelity / construction_error / storage usage / compression_ratio") + parser.add_argument("--dataset", default=DEFAULT_DATASET) + parser.add_argument("--csv", default=DEFAULT_FILE) + parser.add_argument("--num-sites", type=int, default=16) + parser.add_argument("--phys-dim", type=int, default=8) + parser.add_argument("--chi-max", type=int, default=16) + parser.add_argument("--keep-ratio", type=float, default=0.25) + parser.add_argument("--sample-limit", type=int, default=1024) + parser.add_argument("--sample-count", type=int, default=128) + parser.add_argument("--learning-mode", choices=("manual", "auto", "rl"), default="manual") + parser.add_argument("--episodes", type=int, default=100, help="RL 学习轮数，默认100") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--chart-out", default="", help="RL模式可选：学习过程曲线图输出路径（svg）") + parser.add_argument("--log-out", default="", help="可选：将完整结果JSON写入文件") + args = parser.parse_args() + + result = run_experiment( + dataset=args.dataset, + csv_path=Path(args.csv), + num_sites=args.num_sites, + phys_dim=args.phys_dim, + chi_max=args.chi_max, + keep_ratio=args.keep_ratio, + sample_limit=args.sample_limit, + sample_count=args.sample_count, + learning_mode=args.learning_mode, + episodes=args.episodes, + seed=args.seed, + chart_out=args.chart_out, + ) + + print(json.dumps(result, ensure_ascii=False, indent=2)) + if args.log_out: + Path(args.log_out).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_qcloud_collect_and_plot.py b/systems/quantum_db/scripts/tn_qcloud_collect_and_plot.py new file mode 100644 index 0000000..2eef8d7 --- /dev/null +++ b/systems/quantum_db/scripts/tn_qcloud_collect_and_plot.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +import json +import os +from pathlib import Path +import sys +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from interface.cdk_service import CdkTNQuantumService +from pyqpanda3.qcloud import DataBase, QCloudJob, QCloudService + + +FACTOR_LABELS = { + "distribution_type": "Distribution", + "distinct": "Distinct", + "corr": "Correlation", +} + + +def variant_value(record: dict[str, Any], factor: str) -> Any: + dataset = record["dataset"] + return dataset.get(factor, dataset.get("path", "")) + + +def sort_values(values: set[Any]) -> list[Any]: + return sorted(values, key=lambda x: (str(type(x)), x)) + + +def mean(values: list[float]) -> float: + return sum(values) / max(len(values), 1) + + +def refresh_qcloud_results(data: dict[str, Any], config_path: str) -> None: + service_cfg = CdkTNQuantumService(config_path=config_path).quantum.adapter.config + os.environ["QPANDA3_API_KEY"] = service_cfg.api_token + # Initialize the cloud service once; QCloudJob(job_id) relies on this default context. + QCloudService(service_cfg.api_token) + + refreshed = 0 + for record in data["records"]: + quantum = record.setdefault("quantum", {}) + metadata = quantum.setdefault("metadata", {}) + job_id = metadata.get("job_id", "") + if not job_id: + continue + try: + job = QCloudJob(str(job_id)) + status = str(job.status()) + metadata["status"] = status + metadata.pop("status_query_error", None) + pending = "FINISH" not in status.upper() + metadata["result_pending"] = pending + if not pending: + result = job.result() + try: + quantum["counts"] = dict(result.get_counts(base=DataBase.Binary)) + except Exception: + quantum["counts"] = dict(result.get_counts()) + try: + quantum["probs"] = dict(result.get_probs(base=DataBase.Binary)) + except Exception: + quantum["probs"] = {} + refreshed += 1 + except Exception as exc: + metadata["status_query_error"] = f"{type(exc).__name__}: {exc}" + data["status_refresh"] = {"queried_jobs": refreshed} + + +def aggregate_page_metrics(records: list[dict[str, Any]], factor: str) -> list[dict[str, Any]]: + grouped: dict[Any, list[dict[str, Any]]] = defaultdict(list) + for record in records: + if record["factor"] == factor: + grouped[variant_value(record, factor)].append(record) + out = [] + for value in sort_values(set(grouped)): + rows = grouped[value] + out.append( + { + "variant": value, + "fidelity": mean([float(r["metrics"]["fidelity"]) for r in rows]), + "compression_ratio": mean([float(r["metrics"]["compression_ratio"]) for r in rows]), + } + ) + return out + + +def aggregate_quantum_metrics(records: list[dict[str, Any]], factor: str) -> list[dict[str, Any]]: + out = [] + grouped: dict[tuple[Any, str], list[dict[str, Any]]] = defaultdict(list) + for record in records: + if record["factor"] == factor: + grouped[(variant_value(record, factor), record["tn_mode"])].append(record) + for value in sort_values({key[0] for key in grouped}): + for mode in ("mps_linear", "tree", "native_mapping"): + rows = grouped.get((value, mode), []) + if not rows: + continue + out.append( + { + "variant": value, + "mode": mode, + "qubits": mean([float(r["prepare"]["qubits"]) for r in rows]), + "gate_depth": mean([float(r["prepare"]["gate_depth"]) for r in rows]), + } + ) + return out + + +def plot_page_metric(records: list[dict[str, Any]], factor: str, out_dir: Path) -> Path: + rows = aggregate_page_metrics(records, factor) + labels = [str(r["variant"]) for r in rows] + x = list(range(len(rows))) + width = 0.36 + + fig, ax1 = plt.subplots(figsize=(8.5, 5.0), dpi=180) + ax2 = ax1.twinx() + ax1.bar([i - width / 2 for i in x], [r["fidelity"] for r in rows], width=width, color="#4C78A8", label="Fidelity") + ax2.bar([i + width / 2 for i in x], [r["compression_ratio"] for r in rows], width=width, color="#F58518", label="Compression ratio") + + ax1.set_ylabel("Fidelity") + ax2.set_ylabel("Compression ratio") + ax1.set_ylim(0, 1.05) + ax1.set_xticks(x, labels) + ax1.set_xlabel(FACTOR_LABELS[factor]) + ax1.set_title(f"TN Page Build: {FACTOR_LABELS[factor]}") + ax1.grid(axis="y", alpha=0.25) + lines1, labels1 = ax1.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax1.legend(lines1 + lines2, labels1 + labels2, loc="best") + fig.tight_layout() + path = out_dir / f"page_build_{factor}.png" + fig.savefig(path) + plt.close(fig) + return path + + +def plot_quantum_metric(records: list[dict[str, Any]], factor: str, out_dir: Path) -> Path: + rows = aggregate_quantum_metrics(records, factor) + variants = sort_values({r["variant"] for r in rows}) + modes = ["mps_linear", "tree", "native_mapping"] + colors = {"mps_linear": "#4C78A8", "tree": "#54A24B", "native_mapping": "#E45756"} + mode_offsets = {"mps_linear": -0.24, "tree": 0.0, "native_mapping": 0.24} + width = 0.22 + + lookup = {(r["variant"], r["mode"]): r for r in rows} + x = list(range(len(variants))) + fig, ax1 = plt.subplots(figsize=(9.0, 5.2), dpi=180) + ax2 = ax1.twinx() + for mode in modes: + depth = [lookup[(v, mode)]["gate_depth"] for v in variants] + qubits = [lookup[(v, mode)]["qubits"] for v in variants] + xpos = [i + mode_offsets[mode] for i in x] + ax1.bar(xpos, depth, width=width, color=colors[mode], label=f"{mode} depth") + ax2.plot(xpos, qubits, color=colors[mode], marker="o", linewidth=1.4, linestyle="--", label=f"{mode} qubits") + + ax1.set_ylabel("Gate depth") + ax2.set_ylabel("Qubits") + ax1.set_xticks(x, [str(v) for v in variants]) + ax1.set_xlabel(FACTOR_LABELS[factor]) + ax1.set_title(f"Quantum Load Cost: {FACTOR_LABELS[factor]}") + ax1.grid(axis="y", alpha=0.25) + lines1, labels1 = ax1.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax1.legend(lines1 + lines2, labels1 + labels2, loc="upper center", ncols=2, fontsize=8) + fig.tight_layout() + path = out_dir / f"quantum_load_{factor}.png" + fig.savefig(path) + plt.close(fig) + return path + + +def write_report(data: dict[str, Any], paths: list[Path], out_path: Path) -> None: + records = data["records"] + statuses = Counter(r.get("quantum", {}).get("metadata", {}).get("status", "") for r in records) + modes = Counter(r.get("quantum", {}).get("execution_mode", "") for r in records) + finished = sum(1 for r in records if r.get("quantum", {}).get("counts")) + + lines = [ + "# TN QCloud Complete Run Results", + "", + f"- Jobs: {len(records)}", + f"- Backend: {records[0]['quantum']['backend'] if records else ''}", + f"- Rows fixed: {data['rows_fixed']}", + f"- Shots per job: {data['shots']}", + f"- Execution modes: {dict(modes)}", + f"- Finished jobs with counts: {finished}", + f"- Status distribution: {dict(statuses)}", + "", + "## Generated Figures", + ] + for path in paths: + lines.append(f"- `{path}`") + lines += [ + "", + "## Mode Cost Summary", + "| mode | prepare qubits | prepare depth | gate counts | routed depth | CZ-eq ops | swaps |", + "|---|---:|---:|---|---:|---:|---:|", + ] + for mode in ("mps_linear", "tree", "native_mapping"): + row = next(r for r in records if r["tn_mode"] == mode) + h = row["hardware_cost"] + p = row["prepare"] + lines.append( + f"| {mode} | {p['qubits']} | {p['gate_depth']} | `{json.dumps(p['gate_counts'], sort_keys=True)}` | " + f"{h['routed_depth']} | {h['routed_cz_equivalent_ops']} | {h['routing_swap_count']} |" + ) + out_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", default="artifacts/tn_single_factor_qcloud_48jobs_status.json") + parser.add_argument("--config", default="config/tn_quantum_experiment.json") + parser.add_argument("--out-json", default="artifacts/tn_single_factor_qcloud_48jobs_complete.json") + parser.add_argument("--out-report", default="artifacts/tn_single_factor_qcloud_48jobs_complete_report.md") + parser.add_argument("--fig-dir", default="artifacts/figures") + args = parser.parse_args() + + data = json.loads(Path(args.input).read_text(encoding="utf-8")) + refresh_qcloud_results(data, args.config) + out_json = Path(args.out_json) + out_json.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + fig_dir = Path(args.fig_dir) + fig_dir.mkdir(parents=True, exist_ok=True) + figures: list[Path] = [] + for factor in ("distribution_type", "distinct", "corr"): + figures.append(plot_page_metric(data["records"], factor, fig_dir)) + for factor in ("distribution_type", "distinct", "corr"): + figures.append(plot_quantum_metric(data["records"], factor, fig_dir)) + write_report(data, figures, Path(args.out_report)) + + statuses = Counter(r.get("quantum", {}).get("metadata", {}).get("status", "") for r in data["records"]) + print(json.dumps({"records": len(data["records"]), "statuses": dict(statuses), "figures": [str(p) for p in figures]}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_qpanda3_full_experiment.py b/systems/quantum_db/scripts/tn_qpanda3_full_experiment.py new file mode 100644 index 0000000..0213286 --- /dev/null +++ b/systems/quantum_db/scripts/tn_qpanda3_full_experiment.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +from interface.cdk_service import CdkTNQuantumService, TNBuildRequest + + +def load_rows(path: Path, limit: int) -> list[list[float]]: + out: list[list[float]] = [] + with path.open("r", encoding="utf-8") as f: + r = csv.reader(f) + for row in r: + try: + out.append([float(v) for v in row]) + except ValueError: + continue + if len(out) >= limit: + break + return out + + +def run_mode(service: CdkTNQuantumService, mode: str, dataset: str, samples: list[list[float]], shots: int) -> dict: + req = TNBuildRequest(dataset=f"{dataset}:{mode}", samples=samples, tn_mode=mode, shots=shots) + return service.invoke(req) + + +def main() -> None: + ap = argparse.ArgumentParser(description="Run full TN->Qpanda3 noise-sim experiment for 3 modes") + ap.add_argument("--csv", default="train-test-data/SyntheticDataset/cols_8_distinct_10000_corr_8_skew_15.csv") + ap.add_argument("--limit", type=int, default=1024) + ap.add_argument("--shots", type=int, default=0, help="0 means use config file default") + ap.add_argument("--config", default="config/tn_quantum_experiment.json") + ap.add_argument("--out", default="artifacts/tn_qpanda3_full_experiment.json") + args = ap.parse_args() + + samples = load_rows(Path(args.csv), args.limit) + service = CdkTNQuantumService(config_path=args.config) + result = { + "dataset": args.csv, + "rows": len(samples), + "modes": { + "mps_linear": run_mode(service, "mps_linear", "exp", samples, args.shots or None), + "tree": run_mode(service, "tree", "exp", samples, args.shots or None), + "native_mapping": run_mode(service, "native_mapping", "exp", samples, args.shots or None), + }, + } + Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_quantum_smoke.py b/systems/quantum_db/scripts/tn_quantum_smoke.py new file mode 100644 index 0000000..76fdcd1 --- /dev/null +++ b/systems/quantum_db/scripts/tn_quantum_smoke.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +from data_plane.quantum.executor import QuantumExecutor +from data_plane.tn.compute import TNCompute +from data_plane.tn.exproter import TNExporter + + +def main() -> None: + samples = [ + [[0.1, -0.2], [0.4, 0.3]], + [[0.2, -0.1], [0.35, 0.25]], + [[-0.1, 0.05], [0.45, 0.1]], + [[0.0, -0.05], [0.5, 0.2]], + ] + page = TNCompute(seed=7).fit_mps_from_samples(samples, page_id="q-smoke", num_sites=4, phys_dim=4, chi_max=4) + exporter = TNExporter() + packet = exporter.export(page, target_backend="qpanda3-sim", allowed_encodings=["fft_topk"]) + ir = exporter.to_quantum_ir(packet) + + qx = QuantumExecutor() + preflight = qx.execute({"action": "preflight", "payload": {"ir": ir}}) + compiled = qx.execute({"action": "compile", "payload": {"ir": ir}}) + estimate = qx.execute({"action": "estimate", "payload": {"ir": ir}}) + result = qx.execute({"action": "run", "payload": {"compiled": compiled, "shots": 256}}) + + print("preflight_ok", preflight.ok, "route_edges", len(preflight.route_edges)) + print("compiled_ops", len(compiled.operations), "depth_est", estimate["depth_est"]) + print("shots", result.shots, "counts", result.counts) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/systems/quantum_db/scripts/tn_single_factor_hardware_experiment.py b/systems/quantum_db/scripts/tn_single_factor_hardware_experiment.py new file mode 100644 index 0000000..ed148c6 --- /dev/null +++ b/systems/quantum_db/scripts/tn_single_factor_hardware_experiment.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from data_plane.config_loader import load_runtime_config +from interface.cdk_service import CdkTNQuantumService, TNBuildRequest +from scripts.tn_page_experiment import calc_metrics +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig + + +DATASET_RE = re.compile( + r"cols_(?P\d+)_distinct_(?P\d+)_corr_(?P\d+)_skew_(?P\d+)(?P_nohead)?\.csv$" +) + + +def parse_dataset(path: Path) -> dict[str, Any] | None: + match = DATASET_RE.match(path.name) + if not match: + return None + out: dict[str, Any] = {k: int(v) for k, v in match.groupdict().items() if k != "nohead" and v is not None} + out["header_mode"] = "nohead" if match.group("nohead") else "header" + out["distribution_type"] = "skewed" if out["skew"] else "uniform" + out["path"] = str(path) + return out + + +def load_rows(path: Path, limit: int) -> list[list[float]]: + rows: list[list[float]] = [] + with path.open("r", encoding="utf-8") as f: + for row in csv.reader(f): + try: + rows.append([float(v) for v in row]) + except ValueError: + continue + if len(rows) >= limit: + break + return rows + + +def check_environment(config_path: Path) -> dict[str, Any]: + runtime = load_runtime_config(config_path) + quantum = dict(runtime.get("quantum", {})) + raw_api_base = str(quantum.get("api_base", "")).strip() + parsed_api_base = urlparse(raw_api_base) + api_base_is_url = bool(parsed_api_base.scheme and parsed_api_base.hostname) + api_token = str(quantum.get("api_token", "")).strip() + qcloud_token_from_api_base = bool(raw_api_base and not api_base_is_url and not api_token) + hardware_profile_path = Path(str(quantum.get("hardware_profile", ""))) + hardware_api_key_configured = False + if hardware_profile_path.exists(): + try: + hardware_profile = json.loads(hardware_profile_path.read_text(encoding="utf-8")) + hardware_api_key_configured = bool(str(hardware_profile.get("API_KEY", "")).strip()) + except (OSError, json.JSONDecodeError): + hardware_api_key_configured = False + bundled_spec = importlib.util.find_spec("pyqpanda3") + system_probe = subprocess.run( + [ + "python3", + "-c", + ( + "import importlib.util, json; " + "spec=importlib.util.find_spec('pyqpanda3'); " + "status={'spec': bool(spec), 'origin': getattr(spec, 'origin', '') if spec else ''}; " + "\ntry:\n import pyqpanda3\n status['import_ok']=True\n status['version']=getattr(pyqpanda3,'__version__','unknown')\n" + "except Exception as e:\n status['import_ok']=False\n status['error']=type(e).__name__ + ': ' + str(e)\n" + "print(json.dumps(status, ensure_ascii=False))" + ), + ], + capture_output=True, + text=True, + ) + try: + system_status = json.loads(system_probe.stdout.strip() or "{}") + except json.JSONDecodeError: + system_status = {"import_ok": False, "error": system_probe.stderr.strip()} + return { + "config_path": str(config_path), + "api_base_configured": api_base_is_url, + "api_base_value_looks_like_token": qcloud_token_from_api_base, + "api_token_configured": bool(api_token or qcloud_token_from_api_base), + "hardware_profile_api_key_configured": hardware_api_key_configured, + "hardware_profile": quantum.get("hardware_profile", ""), + "bundled_python_pyqpanda3_installed": bundled_spec is not None, + "system_python_pyqpanda3": system_status, + "api_note": "REST uses quantum.api_base when it is an https URL. QPanda3 qcloud SDK uses the API key with its default cloud URL when api_base is empty or looks like a token.", + } + + +def select_cases(dataset_dir: Path) -> list[dict[str, Any]]: + all_cases = [case for p in dataset_dir.glob("*.csv") if (case := parse_dataset(p)) is not None] + existing = {(c["cols"], c["distinct"], c["corr"], c["skew"], c["header_mode"]): c for c in all_cases} + baseline = {"cols": 8, "distinct": 10000, "corr": 8, "skew": 15, "header_mode": "header"} + + specs = [ + ("distribution_type", [{**baseline, "skew": 0}, baseline]), + ("cols", [{**baseline, "cols": 4}, baseline]), + ("distinct", [{**baseline, "distinct": 100}, {**baseline, "distinct": 1000}, baseline]), + ("corr", [{**baseline, "corr": 0}, {**baseline, "corr": 4}, baseline]), + ("skew", [{**baseline, "skew": 0}, {**baseline, "skew": 5}, {**baseline, "skew": 10}, baseline]), + ("header_mode", [baseline, {**baseline, "header_mode": "nohead"}]), + ] + + selected: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for factor, variants in specs: + for spec in variants: + key = (spec["cols"], spec["distinct"], spec["corr"], spec["skew"], spec["header_mode"]) + case = existing.get(key) + if case is None: + continue + row = dict(case) + row["factor"] = factor + row["baseline"] = spec == baseline + unique = (factor, row["path"]) + if unique not in seen: + selected.append(row) + seen.add(unique) + return selected + + +def run_tn_metrics(samples: list[list[float]], dataset_id: str, mode: str, runtime: dict[str, Any], sample_count: int) -> dict[str, Any]: + cfg = WorkloadConfig( + num_sites=int(runtime["num_sites"]), + phys_dim=int(runtime["phys_dim"]), + chi_max=int(runtime["chi_max"]), + keep_ratio=float(runtime["keep_ratio"]), + tn_mode=mode, + ) + service = TNWorkloadService(config=cfg) + page = service.create_page(dataset=dataset_id, samples=samples) + return calc_metrics(service, page, samples, sample_count) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fixed-row single-factor TN hardware-cost experiment") + parser.add_argument("--dataset-dir", default="train-test-data/SyntheticDataset") + parser.add_argument("--config", default="config/tn_quantum_experiment.json") + parser.add_argument("--rows", type=int, default=256) + parser.add_argument("--shots", type=int, default=128) + parser.add_argument("--out", default="artifacts/tn_single_factor_hardware_experiment.json") + args = parser.parse_args() + + runtime = load_runtime_config(args.config) + service = CdkTNQuantumService(config_path=args.config) + cases = select_cases(Path(args.dataset_dir)) + sample_count = int(runtime.get("sample_num_for_fidelity", 64)) + + records: list[dict[str, Any]] = [] + for case in cases: + samples = load_rows(Path(case["path"]), args.rows) + if len(samples) != args.rows: + continue + for mode in ("mps_linear", "tree", "native_mapping"): + dataset_id = f"{Path(case['path']).stem}:{mode}" + hardware = service.invoke(TNBuildRequest(dataset=dataset_id, samples=samples, tn_mode=mode, shots=args.shots)) + metrics = run_tn_metrics(samples, dataset_id, mode, runtime, sample_count) + records.append( + { + "factor": case["factor"], + "dataset": case, + "rows": len(samples), + "tn_mode": mode, + "metrics": { + "fidelity": metrics["fidelity"], + "reconstruction_error_l1": metrics["reconstruction_error_l1"], + "compression_ratio": metrics["compression_ratio"], + "dense_compression_ratio": metrics["dense_compression_ratio"], + "effective_compression_ratio": metrics["effective_compression_ratio"], + "dense_materialization_ratio": metrics["dense_materialization_ratio"], + "dense_materialization_compression_ratio": metrics["dense_materialization_compression_ratio"], + "storage_usage": metrics["storage_usage"], + "raw_input_bytes": metrics["raw_input_bytes"], + }, + "hardware_cost": hardware["hardware_cost"], + "qubit_load_info": hardware.get("qubit_load_info", {}), + "prepare": { + "qubits": hardware["prepare_qubits"], + "gate_depth": hardware["prepare_gate_depth"], + "gate_counts": hardware["prepare_gate_counts"], + }, + "quantum": { + "backend": hardware["quantum_backend"], + "shots": hardware["quantum_shots"], + "counts": hardware["quantum_counts"], + "metadata": hardware.get("quantum_metadata", {}), + "execution_mode": hardware.get("cloud_execution_mode", ""), + }, + "cloud_api_enabled": hardware["cloud_api_enabled"], + } + ) + + result = { + "experiment": "single_factor_fixed_rows_hardware_cost", + "rows_fixed": args.rows, + "shots": args.shots, + "factors": sorted({r["factor"] for r in records}), + "environment": check_environment(Path(args.config)), + "records": records, + } + Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/scripts/tn_train_data_experiment.py b/systems/quantum_db/scripts/tn_train_data_experiment.py new file mode 100644 index 0000000..78fb54d --- /dev/null +++ b/systems/quantum_db/scripts/tn_train_data_experiment.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import csv +import json +import os +import random +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from control_plane.semantic.workload_semantic import SemanticExperimentConfig, TNSemanticOrchestrator +from data_plane.tn.workloads import TNWorkloadService, WorkloadConfig, suggest_workload_config + +DATASET_NAMES = ("Forest", "IMDB", "XueTang", "Synthetic Dataset") +SUPPORTED_SUFFIX = {".csv", ".tsv", ".json", ".jsonl", ".txt"} + + +def _flatten_numeric(value: Any) -> list[float]: + if isinstance(value, bool): + return [1.0 if value else 0.0] + if isinstance(value, (int, float)): + return [float(value)] + if isinstance(value, str): + text = value.strip() + if not text: + return [] + try: + return [float(text)] + except ValueError: + return [] + if isinstance(value, dict): + out: list[float] = [] + for v in value.values(): + out.extend(_flatten_numeric(v)) + return out + if isinstance(value, (list, tuple)): + out: list[float] = [] + for v in value: + out.extend(_flatten_numeric(v)) + return out + return [] + + +def _read_csv(path: Path, delim: str = ",", limit: int = 4096) -> list[list[float]]: + rows: list[list[float]] = [] + with path.open("r", encoding="utf-8") as f: + reader = csv.DictReader(f, delimiter=delim) + for rec in reader: + vals: list[float] = [] + for v in rec.values(): + vals.extend(_flatten_numeric(v)) + if vals: + rows.append(vals) + if len(rows) >= limit: + break + return rows + + +def _read_json(path: Path, limit: int = 4096) -> list[list[float]]: + rows: list[list[float]] = [] + if path.suffix == ".jsonl": + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + vals = _flatten_numeric(json.loads(line)) + if vals: + rows.append(vals) + if len(rows) >= limit: + break + return rows + + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, list): + for obj in payload: + vals = _flatten_numeric(obj) + if vals: + rows.append(vals) + if len(rows) >= limit: + break + elif isinstance(payload, dict): + vals = _flatten_numeric(payload) + if vals: + rows.append(vals) + return rows + + +def _read_txt(path: Path, limit: int = 4096) -> list[list[float]]: + rows: list[list[float]] = [] + with path.open("r", encoding="utf-8") as f: + for line in f: + nums: list[float] = [] + for token in line.strip().replace(",", " ").split(): + try: + nums.append(float(token)) + except ValueError: + pass + if nums: + rows.append(nums) + if len(rows) >= limit: + break + return rows + + +def _load_dataset_file(path: Path, limit: int) -> list[list[float]]: + if path.suffix == ".csv": + return _read_csv(path, ",", limit) + if path.suffix == ".tsv": + return _read_csv(path, "\t", limit) + if path.suffix in {".json", ".jsonl"}: + return _read_json(path, limit) + if path.suffix == ".txt": + return _read_txt(path, limit) + return [] + + +def _find_dataset_files(base_dir: Path, dataset_name: str) -> list[Path]: + key = dataset_name.lower().replace(" ", "") + files: list[Path] = [] + for p in base_dir.rglob("*"): + if not p.is_file() or p.suffix.lower() not in SUPPORTED_SUFFIX: + continue + stem = p.stem.lower().replace(" ", "") + full = str(p).lower().replace(" ", "") + if key in stem or key in full: + files.append(p) + files.sort() + return files + + +def _synthetic_samples(limit: int, width: int = 8) -> list[list[float]]: + rng = random.Random(7) + out: list[list[float]] = [] + for _ in range(limit): + base = [rng.gauss(0.0, 1.0) for _ in range(width)] + out.append(base) + return out + + +def run_experiments(base_dir: Path, limit: int, base_workload: WorkloadConfig, semantic_config: SemanticExperimentConfig, *, allow_synthetic_fallback: bool) -> dict[str, Any]: + if not base_dir.exists(): + raise FileNotFoundError(f"TN_DATA_DIR does not exist: {base_dir}") + + results: list[dict[str, Any]] = [] + discovered: dict[str, list[str]] = {} + failures: list[dict[str, str]] = [] + + for dataset in DATASET_NAMES: + files = _find_dataset_files(base_dir, dataset) + discovered[dataset] = [str(f.relative_to(base_dir)) for f in files] + samples: list[list[float]] = [] + + for f in files: + rows = _load_dataset_file(f, limit=limit) + if rows: + samples.extend(rows) + if len(samples) >= limit: + break + + source = "file" + if not samples: + if allow_synthetic_fallback: + samples = _synthetic_samples(limit) + source = "synthetic" + else: + failures.append({"dataset": dataset, "reason": "匹配的文件中没有可解析的数值行"}) + continue + + min_len = min(len(row) for row in samples) + samples = [row[:min_len] for row in samples if len(row) >= min_len][:limit] + dynamic_cfg = suggest_workload_config( + samples, + max_num_sites=base_workload.num_sites, + max_phys_dim=base_workload.phys_dim, + max_chi=base_workload.chi_max, + ) + service = TNWorkloadService(config=dynamic_cfg, seed=None) + orchestrator = TNSemanticOrchestrator(service=service, config=semantic_config) + case = orchestrator.run_dataset(dataset, samples, discovered[dataset], source) + case["dynamic_config"] = { + "num_sites": dynamic_cfg.num_sites, + "phys_dim": dynamic_cfg.phys_dim, + "chi_max": dynamic_cfg.chi_max, + "keep_ratio": dynamic_cfg.keep_ratio, + } + results.append(case) + + if failures and not allow_synthetic_fallback: + raise RuntimeError(f"dataset loading failures: {failures}") + + return {"base_dir": str(base_dir), "datasets": results, "discovered_files": discovered} + + +def main() -> None: + base_dir = Path(os.getenv("TN_DATA_DIR", "/storage/train-test-data")) + limit = int(os.getenv("TN_LIMIT", "512")) + allow_fallback = os.getenv("TN_ALLOW_SYNTHETIC_FALLBACK", "0") in {"1", "true", "TRUE"} + + workload_config = WorkloadConfig( + num_sites=int(os.getenv("TN_NUM_SITES", "16")), + phys_dim=int(os.getenv("TN_PHYS_DIM", "8")), + chi_max=int(os.getenv("TN_CHI_MAX", "16")), + keep_ratio=float(os.getenv("TN_KEEP_RATIO", "0.25")), + ) + semantic_config = SemanticExperimentConfig(data_dir=base_dir, limit=limit, allow_synthetic_fallback=allow_fallback) + + out = run_experiments(base_dir, limit, workload_config, semantic_config, allow_synthetic_fallback=allow_fallback) + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/systems/quantum_db/tests/test_control_plane_primitives.py b/systems/quantum_db/tests/test_control_plane_primitives.py new file mode 100644 index 0000000..8acb938 --- /dev/null +++ b/systems/quantum_db/tests/test_control_plane_primitives.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace + +from control_plane.monitor.alert import AlertEvaluator, AlertRule +from control_plane.monitor.monitor import Monitor +from control_plane.scheduler.policy import AdaptiveSchedulingPolicy +from control_plane.scheduler.priority_queue import PriorityQueue +from data_plane.bus.bus import Bus +from data_plane.bus.command_bus import CommandBus +from data_plane.bus.data_bus import DataBus +from data_plane.quantum.fallback import QuantumFallbackPolicy +from protocol.events import AlertEvent, TaskCompletedEvent, TaskStateEvent +from protocol.status import TaskStatus + + +class ControlPlanePrimitiveTests(unittest.TestCase): + def test_priority_queue_is_stable_and_supports_front_requeue(self) -> None: + queue = PriorityQueue() + queue.push("low", priority=1) + queue.push("first", priority=4) + queue.push("second", priority=4) + queue.requeue("first", front=True) + self.assertEqual(queue.pop(), ("first", None)) + self.assertEqual(queue.pop(), ("second", None)) + self.assertEqual(queue.pop(), ("low", None)) + self.assertIsNone(queue.pop()) + + def test_adaptive_policy_reduces_dispatch_for_unhealthy_feedback(self) -> None: + policy = AdaptiveSchedulingPolicy() + self.assertEqual(policy.advise(SimpleNamespace(global_health="healthy"), requested_dispatch=6).max_dispatch, 6) + critical = policy.advise(SimpleNamespace(global_health="critical", anomaly_summary=("queue",)), requested_dispatch=6) + self.assertEqual(critical.max_dispatch, 1) + self.assertTrue(critical.prefer_classical_fallback) + + def test_bus_lane_facades_share_transport(self) -> None: + transport = Bus() + commands, data = CommandBus(transport), DataBus(transport, lane="event") + commands.start() + commands.register("echo", lambda payload, metadata: {"payload": payload, "trace": metadata["trace_id"]}) + self.assertEqual(commands.send("echo", 3, metadata={"trace_id": "t"}), {"payload": 3, "trace": "t"}) + seen: list[str] = [] + subscription = data.subscribe("finished", lambda payload, metadata: seen.append(f"{payload}:{metadata['source']}")) + self.assertEqual(data.publish("finished", "job", metadata={"source": "qdb"}), 1) + self.assertEqual(seen, ["job:qdb"]) + self.assertTrue(data.unsubscribe("finished", subscription)) + + def test_alerts_use_monitor_aggregates(self) -> None: + monitor = Monitor() + monitor.report_metric("sidecar", "performance", "queue_depth", 350) + alerts = AlertEvaluator([AlertRule("queue_depth", 300, severity="warning")]).evaluate(monitor.snapshot()) + self.assertEqual(alerts[0].component, "sidecar") + self.assertEqual(alerts[0].severity, "warning") + + def test_fallback_is_explicit_and_events_are_immutable_models(self) -> None: + decision = QuantumFallbackPolicy().decide(action="run", error=TimeoutError("wait"), allow_local_fallback=True) + self.assertTrue(decision.permitted) + self.assertTrue(decision.retryable) + state = TaskStateEvent("e1", "task_state", 1, "sidecar", task_id="j", status=TaskStatus.RUNNING) + completed = TaskCompletedEvent("e2", "task_completed", 2, "sidecar", task_id="j") + alert = AlertEvent("e3", "alert", 3, "monitor", component="sidecar", severity="critical") + self.assertEqual((state.status, completed.status, alert.severity), (TaskStatus.RUNNING, TaskStatus.SUCCEEDED, "critical")) + + +if __name__ == "__main__": + unittest.main() diff --git a/systems/quantum_db/tests/test_large_scale.py b/systems/quantum_db/tests/test_large_scale.py new file mode 100644 index 0000000..ec5af1a --- /dev/null +++ b/systems/quantum_db/tests/test_large_scale.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from qdb.large_scale import BenchmarkConfig, run_large_scale_benchmark, write_report + + +class LargeScaleBenchmarkTests(unittest.TestCase): + def test_streaming_benchmark_reports_exact_and_approximate_operators(self) -> None: + report = run_large_scale_benchmark(BenchmarkConfig(sizes=(2_000,), reservoir_size=256, tn_train_samples=128, tn_draws=64)) + item = report["results"][0] + self.assertEqual(item["traversal"]["rows"], 2_000) + self.assertGreater(item["traversal"]["predicate_matches"], 0) + self.assertEqual(item["cardinality"]["qute_c0_sample_count"], item["sampling"]["predicate_sample_hits"]) + self.assertEqual(item["sampling"]["tn_draw_count"], 64 * 4) + self.assertGreater(item["cardinality"]["distinct_exact"], 0) + with tempfile.TemporaryDirectory() as directory: + json_path, markdown_path = write_report(report, Path(directory)) + self.assertTrue(json_path.exists()) + self.assertIn("Scan rows/s", markdown_path.read_text(encoding="utf-8")) diff --git a/systems/quantum_db/tests/test_pg_extension_static.py b/systems/quantum_db/tests/test_pg_extension_static.py new file mode 100644 index 0000000..5be6732 --- /dev/null +++ b/systems/quantum_db/tests/test_pg_extension_static.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] / "pg_extension" + + +class PGExtensionStaticTests(unittest.TestCase): + def test_pgxs_regression_has_sql_and_expected_pair(self) -> None: + makefile = (ROOT / "Makefile").read_text(encoding="utf-8") + regress = re.search(r"^REGRESS\s*=\s*(\w+)", makefile, re.MULTILINE) + self.assertIsNotNone(regress) + name = regress.group(1) + self.assertTrue((ROOT / "sql" / f"{name}.sql").exists()) + self.assertTrue((ROOT / "expected" / f"{name}.out").exists()) + + def test_extension_has_an_upgrade_path(self) -> None: + control = (ROOT / "quantum_db.control").read_text(encoding="utf-8") + self.assertIn("default_version = '0.1.1'", control) + self.assertTrue((ROOT / "sql" / "quantum_db--0.1.1.sql").exists()) + upgrade = (ROOT / "sql" / "quantum_db--0.1.0--0.1.1.sql").read_text(encoding="utf-8") + self.assertIn("ALTER TABLE qdb.datasets ADD COLUMN IF NOT EXISTS artifact_checksum", upgrade) + self.assertIn("CREATE OR REPLACE FUNCTION qdb_submit", upgrade) + + def test_custom_scan_uses_table_access_method_and_postgres_recheck(self) -> None: + source = (ROOT / "quantum_db.c").read_text(encoding="utf-8") + self.assertIn("table_beginscan(node->ss.ss_currentRelation", source) + self.assertIn("table_scan_getnextslot", source) + self.assertIn("ExecScan(&node->ss, qdb_next, qdb_recheck)", source) + self.assertIn("extract_actual_clauses(clauses, false)", source) + self.assertIn('qdb_call_sidecar("candidates", params)', source) + self.assertIn("Quantum DB candidate source", source) + self.assertIn("PostgreSQL fallback", source) + self.assertIn("candidate_contract,complete", source) + self.assertIn("classical_snapshot_filter", source) + self.assertIn("complete_superset", source) + self.assertIn("pg_builtin_scalar_equality_v1", source) + self.assertIn("INT8OID", source) + self.assertIn("artifact_checksum", source) + + def test_rpc_has_timeout_and_size_limit(self) -> None: + source = (ROOT / "quantum_db.c").read_text(encoding="utf-8") + self.assertIn("SO_RCVTIMEO", source) + self.assertIn("response.len > 1048576", source) + self.assertIn("Quantum DB socket path is too long", source) + + def test_registration_functions_do_not_escalate_table_privileges(self) -> None: + sql = (ROOT / "sql" / "quantum_db--0.1.0.sql").read_text(encoding="utf-8") + register = sql[sql.index("CREATE FUNCTION qdb_register_dataset"):sql.index("CREATE FUNCTION qdb_unregister_dataset")] + unregister = sql[sql.index("CREATE FUNCTION qdb_unregister_dataset"):sql.index("CREATE FUNCTION qdb_explain")] + self.assertIn("SECURITY INVOKER", register) + self.assertIn("SECURITY INVOKER", unregister) + self.assertIn("REVOKE ALL ON FUNCTION qdb._rpc_text", sql) + self.assertIn("response->'result'", sql) + self.assertIn("Quantum DB sidecar error", sql) + self.assertIn("qdb._sync_dataset", sql) + self.assertIn("dataset.begin", sql) + self.assertIn("dataset.append", sql) + self.assertIn("dataset.commit", sql) + self.assertIn("qdb._remove_sidecar_dataset", sql) + self.assertIn("'quantum_index'", sql) + self.assertIn("INSERT INTO qdb.jobs", sql) + self.assertIn("UPDATE qdb.jobs SET status=response->>'status'", sql) + + def test_current_registration_streams_pages_without_full_jsonb_aggregation(self) -> None: + sql = (ROOT / "sql" / "quantum_db--0.1.1.sql").read_text(encoding="utf-8") + register = sql[sql.index("CREATE FUNCTION qdb_register_dataset"):sql.index("CREATE FUNCTION qdb_unregister_dataset")] + self.assertIn("FOR row_payload IN EXECUTE", register) + self.assertIn("qdb._begin_dataset_upload", register) + self.assertIn("qdb._append_dataset_page", register) + self.assertIn("qdb._commit_dataset_upload", register) + self.assertNotIn("jsonb_agg(jsonb_build_object", register) + self.assertIn("length(%I::text) >= 1024", register) + self.assertIn("unsafe_key_count", register) + + +if __name__ == "__main__": + unittest.main() diff --git a/systems/quantum_db/tests/test_qdb.py b/systems/quantum_db/tests/test_qdb.py new file mode 100644 index 0000000..e31b769 --- /dev/null +++ b/systems/quantum_db/tests/test_qdb.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import tempfile +import time +import unittest +import cmath +import math +from pathlib import Path + +from qdb.protocol import JobStatus, QDBError +from data_plane.quantum.backend import BackendConfig +from data_plane.quantum.adapter import Qpanda3Adapter +from qdb.relational_ir import build_relational_ir, parse_hint +from qdb.router import QueryRouter +from qdb.service import QuantumDatabaseService +from qdb.candidate_executor import CandidateExecutor + + +class RelationalIRTests(unittest.TestCase): + def test_hints(self) -> None: + self.assertEqual(parse_hint("/*+ QDB_QUANTUM */ SELECT 1"), ("quantum", None)) + self.assertEqual(parse_hint("/*+ QDB_TN */ /*+ QDB_SYNC(timeout_ms=25) */ SELECT 1"), ("tn", 25)) + + def test_route_limits_override_forced_quantum(self) -> None: + ir = build_relational_ir({"sql": "/*+ QDB_QUANTUM */ select * from t", "predicates": [{"column": "x", "value": 1}]}) + self.assertEqual(QueryRouter(max_qubits=4).decide(ir, qubits=5).backend, "classical") + + def test_hybrid_route(self) -> None: + ir = build_relational_ir({"sql": "select * from t", "estimated_selectivity": 0.3, + "predicates": [{"column": "x", "value": 1}, {"column": "y", "value": 2}]}) + self.assertEqual(QueryRouter().decide(ir).backend, "hybrid") + + def test_sql_metadata_is_inferred_for_simple_conjunctive_filter(self) -> None: + ir = build_relational_ir({"sql": "/*+ QDB_TN */ SELECT count(*) FROM public.orders WHERE region = 'east' AND amount >= 10 GROUP BY region"}) + self.assertEqual(ir.relation, "public.orders") + self.assertEqual([(p.column, p.operator, p.value) for p in ir.predicates], [("region", "=", "east"), ("amount", ">=", 10)]) + self.assertEqual(ir.hint, "tn") + self.assertEqual(ir.aggregates, ["count", "group_by"]) + + def test_unsupported_or_clause_is_not_partially_inferred(self) -> None: + ir = build_relational_ir({"sql": "SELECT * FROM t WHERE value = 1 OR value = 2"}) + self.assertEqual(ir.predicates, []) + + +class ServiceTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.service = QuantumDatabaseService(state_path=Path(self.temp.name) / "jobs.sqlite3") + + def tearDown(self) -> None: + self.service.close() + self.temp.cleanup() + + def test_candidates_require_recheck(self) -> None: + result = self.service.candidates({ + "sql": "select * from t where value = 10", "relation": "t", "snapshot_version": "1", + "estimated_selectivity": 0.1, "predicates": [{"column": "value", "value": 10}], + "rows": [{"key": 1, "values": {"value": 10}}, {"key": 2, "values": {"value": 20}}], + }) + self.assertEqual(result["candidate_keys"], [1]) + self.assertTrue(result["requires_recheck"]) + self.assertFalse(result["candidate_contract"]["complete"]) + + def test_candidates_estimate_selectivity_from_registered_snapshot_rows(self) -> None: + result = self.service.candidates({ + "sql": "SELECT * FROM t WHERE value = 10", "rows": [ + {"key": key, "values": {"value": 10 if key == 1 else 20}} for key in range(1, 11) + ], + }) + self.assertEqual(result["candidate_keys"], [1]) + self.assertEqual(result["route"]["backend"], "quantum") + + def test_explain_and_submit_share_automatic_route(self) -> None: + params = { + "sql": "SELECT * FROM t WHERE value = 10", + "rows": [{"key": key, "values": {"value": 10 if key == 1 else 20}} for key in range(1, 11)], + } + self.assertEqual(self.service.explain(params)["route"]["backend"], "quantum") + submitted = self.service.submit(params) + self.assertIn(submitted["status"], {JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.SUCCEEDED}) + + def test_registered_snapshot_can_supply_candidate_rows(self) -> None: + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", + "artifact_checksum": "a" * 32, + "rows": [{"key": key, "values": {"value": 10 if key == 1 else 20}} for key in range(1, 11)], + }) + result = self.service.candidates({ + "sql": "SELECT * FROM public.t WHERE value = 10", "snapshot_version": "v1", "artifact_checksum": "a" * 32, + "candidate_semantics": "pg_builtin_scalar_equality_v1", + "predicates": [{"column": "value", "operator": "=", "value": "10", "scalar_type": "int4"}], + }) + self.assertEqual(result["candidate_keys"], [1]) + self.assertEqual(result["route"]["backend"], "quantum") + self.assertTrue(result["candidate_contract"]["complete"]) + self.assertEqual(result["candidate_contract"]["candidate_set"], "complete_superset") + + def test_unqualified_sql_relation_resolves_one_registered_schema_snapshot(self) -> None: + self.service.register_dataset({ + "relation": "public.orders", "snapshot_version": "v1", "key_column": "id", + "artifact_checksum": "0123456789abcdef0123456789abcdef", + "rows": [{"key": 1, "values": {"amount": 10}}, {"key": 2, "values": {"amount": 20}}], + }) + result = self.service.candidates({ + "sql": "SELECT * FROM orders WHERE amount = 10", "snapshot_version": "v1", + }) + self.assertEqual(result["candidate_keys"], [1]) + + def test_unqualified_sql_relation_does_not_guess_between_schemas(self) -> None: + for relation in ("public.orders", "audit.orders"): + self.service.register_dataset({ + "relation": relation, "snapshot_version": "v1", "key_column": "id", + "artifact_checksum": "0123456789abcdef0123456789abcdef", + "rows": [{"key": 1, "values": {"amount": 10}}], + }) + result = self.service.candidates({"sql": "SELECT * FROM orders WHERE amount = 10", "snapshot_version": "v1"}) + self.assertEqual(result["candidate_keys"], []) + + def test_snapshot_mismatch_is_rejected_before_candidate_use(self) -> None: + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", "artifact_checksum": "a" * 32, "rows": [], + }) + with self.assertRaisesRegex(QDBError, "does not match"): + self.service.candidates({"sql": "SELECT * FROM public.t WHERE value = 10", "snapshot_version": "v2"}) + + def test_checksum_mismatch_never_claims_complete_candidate_set(self) -> None: + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", "artifact_checksum": "a" * 32, + "rows": [{"key": 1, "values": {"value": 10}}], + }) + result = self.service.candidates({ + "sql": "SELECT * FROM public.t WHERE value = 10", "snapshot_version": "v1", "artifact_checksum": "b" * 32, + }) + self.assertEqual(result["candidate_keys"], [1]) + self.assertFalse(result["candidate_contract"]["complete"]) + + def test_dataset_registration_requires_a_checksum(self) -> None: + with self.assertRaisesRegex(QDBError, "artifact_checksum"): + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", "rows": [], + }) + + def test_dataset_registration_rejects_noncanonical_checksum(self) -> None: + with self.assertRaisesRegex(QDBError, "MD5"): + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", + "artifact_checksum": "not-a-digest", "rows": [], + }) + + def test_dataset_pages_are_atomic_and_ordered(self) -> None: + begin = self.service.dispatch("dataset.begin", { + "relation": "public.paged", "snapshot_version": "v1", "key_column": "id", + "artifact_checksum": "c" * 32, "expected_rows": 3, + }) + upload_id = begin["upload_id"] + self.service.dispatch("dataset.append", {"upload_id": upload_id, "page": 0, + "rows": [{"key": 2, "values": {"value": 20}}]}) + self.assertIsNone(self.service.jobs.get_dataset("public.paged")) + with self.assertRaisesRegex(QDBError, "expected page 1"): + self.service.dispatch("dataset.append", {"upload_id": upload_id, "page": 2, "rows": []}) + self.service.dispatch("dataset.append", {"upload_id": upload_id, "page": 1, "rows": [ + {"key": 1, "values": {"value": 10}}, {"key": 3, "values": {"value": 30}}, + ]}) + self.service.dispatch("dataset.commit", {"upload_id": upload_id}) + dataset = self.service.jobs.get_dataset("public.paged", snapshot_version="v1") + self.assertEqual([row["key"] for row in dataset["rows"]], [2, 1, 3]) + + def test_only_pg_scalar_equality_can_activate_complete_contract(self) -> None: + self.service.dispatch("dataset.register", { + "relation": "public.t", "snapshot_version": "v1", "key_column": "id", "artifact_checksum": "a" * 32, + "rows": [{"key": 1, "values": {"value": "east"}}], + }) + result = self.service.candidates({ + "sql": "SELECT * FROM public.t WHERE value = 'east'", "snapshot_version": "v1", "artifact_checksum": "a" * 32, + "candidate_semantics": "pg_builtin_scalar_equality_v1", + "predicates": [{"column": "value", "operator": "=", "value": "east", "scalar_type": "text"}], + }) + self.assertFalse(result["candidate_contract"]["complete"]) + + def test_postgresql_textual_constants_match_typed_snapshot_values(self) -> None: + result = self.service.candidates({ + "sql": "SELECT * FROM t", "predicates": [{"column": "value", "operator": ">=", "value": "10"}], + "rows": [{"key": 1, "values": {"value": 10}}, {"key": 2, "values": {"value": 5}}], + }) + self.assertEqual(result["candidate_keys"], [1]) + + def test_async_job(self) -> None: + submitted = self.service.submit({"sql": "select * from t", "rows": []}) + for _ in range(100): + state = self.service.status({"job_id": submitted["job_id"]}) + if state["status"] in {JobStatus.SUCCEEDED, JobStatus.FAILED}: + break + time.sleep(0.01) + self.assertEqual(state["status"], JobStatus.SUCCEEDED) + self.assertTrue(self.service.result({"job_id": submitted["job_id"]})["requires_recheck"]) + + def test_async_quantum_and_tn_probes_are_advisory(self) -> None: + rows = [{"key": key, "values": {"value": 10 if key == 1 else 20}} for key in range(1, 9)] + quantum = self.service.submit({"sql": "SELECT * FROM t WHERE value = 10", "rows": rows, + "run_qpanda_probe": True, "wait": True, "timeout_ms": 2_000}) + quantum_result = self.service.result({"job_id": quantum["job_id"]})["result"] + self.assertTrue(quantum_result["accelerator"]["advisory_only"]) + self.assertTrue(quantum_result["accelerator"]["quantum"]["executed"]) + self.assertEqual(quantum_result["accelerator"]["quantum"]["operator"], "QMEASURE") + self.assertTrue(quantum_result["accelerator"]["qpanda3"]["executed"]) + tn = self.service.submit({"sql": "/*+ QDB_TN */ SELECT * FROM t WHERE value = 10", "rows": rows, "wait": True, "timeout_ms": 2_000}) + tn_result = self.service.result({"job_id": tn["job_id"]})["result"] + self.assertTrue(tn_result["accelerator"]["tn"]["executed"]) + self.assertEqual(tn_result["accelerator"]["tn"]["engine"], "TNCompute.fit_mps_from_samples") + + def test_hybrid_hint_runs_tn_quantum_and_optional_qute_validation(self) -> None: + rows = [{"key": key, "values": {"flag": key == 0, "bucket": "target" if key == 0 else "other"}} + for key in range(8)] + submitted = self.service.submit({ + "sql": "/*+ QDB_HYBRID */ SELECT * FROM t WHERE flag = true AND bucket = 'target'", + "rows": rows, "run_qute_validation": True, "wait": True, "timeout_ms": 2_000, + }) + result = self.service.result({"job_id": submitted["job_id"]})["result"] + self.assertEqual(result["route"]["backend"], "hybrid") + self.assertTrue(result["accelerator"]["tn"]["executed"]) + self.assertTrue(result["accelerator"]["quantum"]["executed"]) + self.assertTrue(result["qute_validation"]["executed"]) + self.assertTrue(result["qute_validation"]["exact"]) + self.assertEqual(result["qute_validation"]["value"], 1) + trace = self.service._probe_component_trace({"rows": rows, "execution_origin": "postgres_extension"}, result) + self.assertEqual(trace["tn_page"], "used") + self.assertEqual(trace["qute_engine"], "used") + self.assertEqual(trace["postgres_extension"], "used") + + def test_three_qubit_hardware_probe_implements_grover_amplification(self) -> None: + ir = CandidateExecutor._qpanda3_ir(width=3, amplification=2, marked_addresses=[0]) + self.assertIsNotNone(ir) + assert ir is not None + self.assertEqual(ir["probe_contract"], "grover-oracle-diffuser-v1") + self.assertEqual({op["gate"] for op in ir["operations"]}, {"u3", "cz", "measure"}) + state = [1 + 0j] + [0j] * 7 + for op in ir["operations"]: + if op["gate"] == "measure": + continue + if op["gate"] == "cz": + first, second = op["qubits"] + for index in range(8): + if ((index >> first) & 1) and ((index >> second) & 1): + state[index] *= -1 + continue + qubit = op["qubits"][0] + theta, phi, lam = op["params"] + matrix = ((math.cos(theta / 2), -cmath.exp(1j * lam) * math.sin(theta / 2)), + (cmath.exp(1j * phi) * math.sin(theta / 2), cmath.exp(1j * (phi + lam)) * math.cos(theta / 2))) + for base in range(8): + if (base >> qubit) & 1: + continue + pair = base | (1 << qubit) + zero, one = state[base], state[pair] + state[base] = matrix[0][0] * zero + matrix[0][1] * one + state[pair] = matrix[1][0] * zero + matrix[1][1] * one + self.assertGreater(abs(state[0]) ** 2, 0.94) + self.assertAlmostEqual(sum(abs(value) ** 2 for value in state), 1.0, places=10) + + def test_grover_probe_iteration_controls_ideal_distribution_and_gate_budget(self) -> None: + rows = [{"key": index, "values": {"flag": index == 0}} for index in range(8)] + one = CandidateExecutor(max_qubits=3).execute(route="quantum", rows=rows, candidate_keys=[0], shots=64, + hardware_grover_iterations=1)["quantum"] + two = CandidateExecutor(max_qubits=3).execute(route="quantum", rows=rows, candidate_keys=[0], shots=64, + hardware_grover_iterations=2)["quantum"] + self.assertAlmostEqual(one["ideal_marked_probability"], 0.78125) + self.assertAlmostEqual(two["ideal_marked_probability"], 0.9453125) + self.assertEqual(one["qpanda3_ir"]["two_qubit_gate_count"], 12) + self.assertEqual(two["qpanda3_ir"]["two_qubit_gate_count"], 24) + + def test_probe_component_trace_identifies_inline_quantum_only_path(self) -> None: + trace = self.service._probe_component_trace({"rows": [{"key": 0, "values": {}}]}, { + "accelerator": {"quantum": {"executed": True}}, + }) + self.assertEqual(trace["candidate_source"], "inline_rows") + self.assertEqual(trace["ideal_qmeasure"], "used") + self.assertEqual(trace["tn_page"], "not_used") + self.assertEqual(trace["postgres_extension"], "not_used") + + def test_registered_snapshot_can_drive_the_hardware_probe_shape(self) -> None: + rows = [{"key": index, "values": {"flag": index == 0}} for index in range(8)] + self.service.register_dataset({ + "relation": "qdb_probe", "snapshot_version": "probe-v1", "key_column": "id", + "artifact_checksum": "0123456789abcdef0123456789abcdef", "rows": rows, + }) + submitted = self.service.submit({ + "sql": "/*+ QDB_QUANTUM */ SELECT * FROM qdb_probe WHERE flag = true", "snapshot_version": "probe-v1", + "run_qpanda_probe": True, "wait": True, "timeout_ms": 2_000, + }) + result = self.service.result({"job_id": submitted["job_id"]})["result"] + self.assertEqual(result["candidate_keys"], [0]) + self.assertEqual(result["accelerator"]["quantum"]["qpanda3_ir"]["expected_marked_state"], "0x0") + self.assertEqual(result["accelerator"]["qpanda3"]["metadata"]["execution_components"]["candidate_source"], "registered_dataset") + + def test_invalid_hardware_grover_iterations_fail_before_execution(self) -> None: + submitted = self.service.submit({ + "sql": "/*+ QDB_QUANTUM */ SELECT * FROM t WHERE value = 10", + "rows": [{"key": index, "values": {"value": 10 if index == 0 else 20}} for index in range(8)], + "run_qpanda_probe": True, "hardware_grover_iterations": 3, "wait": True, "timeout_ms": 2_000, + }) + state = self.service.status({"job_id": submitted["job_id"]}) + self.assertEqual(state["status"], JobStatus.FAILED) + self.assertEqual(state["error_code"], "INVALID_GROVER_ITERATIONS") + + def test_real_hardware_probe_requires_explicit_authorization(self) -> None: + hardware = QuantumDatabaseService(state_path=Path(self.temp.name) / "hardware-jobs.sqlite3", + backend_config=BackendConfig(api_token="configured-token")) + try: + submitted = hardware.submit({ + "sql": "/*+ QDB_QUANTUM */ SELECT * FROM t WHERE value = 10", + "rows": [{"key": key, "values": {"value": 10 if key == 0 else 20}} for key in range(8)], + "run_qpanda_probe": True, + "wait": True, "timeout_ms": 2_000, + }) + result = hardware.result({"job_id": submitted["job_id"]})["result"] + self.assertFalse(result["accelerator"]["qpanda3"]["executed"]) + self.assertIn("HARDWARE_NOT_AUTHORIZED", result["accelerator"]["qpanda3"]["reason"]) + finally: + hardware.close() + + def test_explicit_quantum_ir_requires_hardware_authorization(self) -> None: + hardware = QuantumDatabaseService(state_path=Path(self.temp.name) / "hardware-ir-jobs.sqlite3", + backend_config=BackendConfig(api_token="configured-token")) + try: + submitted = hardware.submit({ + "sql": "/*+ QDB_QUANTUM */ SELECT * FROM t WHERE value = 10", + "rows": [{"key": 1, "values": {"value": 10}}], + "quantum_ir": {"model_id": "probe", "operations": []}, + "wait": True, "timeout_ms": 2_000, + }) + result = hardware.result({"job_id": submitted["job_id"]})["result"] + self.assertFalse(result["quantum"]["executed"]) + self.assertIn("HARDWARE_NOT_AUTHORIZED", result["quantum"]["reason"]) + finally: + hardware.close() + + def test_cancelled_job_is_not_overwritten_by_late_execution_failure(self) -> None: + job_id = self.service.jobs.create({"sql": "SELECT 1"}, backend="quantum", trace_id="trace") + original_candidates = self.service.candidates + + def cancel_then_fail(_: dict) -> dict: + self.service.jobs.cancel(job_id) + raise QDBError("BACKEND_TIMEOUT", "late provider failure", retryable=True) + + self.service.candidates = cancel_then_fail # type: ignore[method-assign] + try: + self.service._run_job(job_id, {"sql": "SELECT 1"}) + finally: + self.service.candidates = original_candidates # type: ignore[method-assign] + self.assertEqual(self.service.jobs.get(job_id)["status"], JobStatus.CANCELLED) + + def test_conditional_job_update_cannot_overwrite_cancelled_state(self) -> None: + job_id = self.service.jobs.create({"sql": "SELECT 1"}, backend="classical", trace_id="trace") + self.assertTrue(self.service.jobs.cancel(job_id)) + self.assertFalse(self.service.jobs.update(job_id, JobStatus.FAILED, error_code="LATE", preserve_cancelled=True)) + self.assertEqual(self.service.jobs.get(job_id)["status"], JobStatus.CANCELLED) + + def test_unknown_job(self) -> None: + with self.assertRaises(QDBError): + self.service.status({"job_id": "missing"}) + + def test_hardware_preflight_requires_explicit_authorization_and_configured_token(self) -> None: + with self.assertRaisesRegex(QDBError, "allow_hardware"): + self.service.dispatch("hardware.preflight", {}) + with self.assertRaisesRegex(QDBError, "ORIGINQC_API_KEY"): + self.service.dispatch("hardware.preflight", {"allow_hardware": True}) + health = self.service.health({}) + self.assertFalse(health["hardware_configured"]) + self.assertFalse(health["hardware_sdk_available"]) + configured = QuantumDatabaseService(state_path=Path(self.temp.name) / "hardware.sqlite3", + backend_config=BackendConfig(api_token="test-token")) + try: + with self.assertRaisesRegex(QDBError, "pyqpanda3"): + configured.dispatch("hardware.preflight", {"allow_hardware": True}) + finally: + configured.close() + + def test_qcloud_count_parser_uses_fallback_accessors(self) -> None: + class Result: + def get_counts(self, **kwargs): return {} + def get_counts_list(self): return [{"00": 2}, {"00": 1, "11": 3}] + def origin_data(self): return "raw-provider-payload" + counts, source = Qpanda3Adapter._extract_qcloud_counts(Result()) + self.assertEqual(counts, {"00": 3, "11": 3}) + self.assertEqual(source, "counts_list") + + def test_qcloud_count_parser_reads_documented_raw_prob_count(self) -> None: + class Result: + def get_counts(self, **kwargs): return {} + def get_counts_list(self): return [] + def prob_count_raw(self): return '{"probCount":"{\\\"0\\\": 61, \\\"1\\\": 3}"}' + def origin_data(self): return "raw-provider-payload" + counts, source = Qpanda3Adapter._extract_qcloud_counts(Result()) + self.assertEqual(counts, {"0": 61, "1": 3}) + self.assertEqual(source, "prob_count_raw") + + def test_qcloud_raw_count_parser_rejects_probability_maps(self) -> None: + self.assertEqual(Qpanda3Adapter._parse_qcloud_prob_count('{"0": 0.5, "1": 0.5}'), {}) + + def test_qcloud_result_diagnostics_are_bounded(self) -> None: + class Result: + def error_message(self): return "provider-error" + def measure_qubits(self): return "q[0],q[1]" + self.assertEqual(Qpanda3Adapter._qcloud_result_diagnostics(Result()), + {"provider_error": "provider-error", "measure_qubits": "q[0],q[1]"}) + + def test_qcloud_result_requests_optional_count_fields(self) -> None: + class Job: + def __init__(self): self.keys = None + def result(self, *, keys): + self.keys = keys + return "result" + job = Job() + self.assertEqual(Qpanda3Adapter._qcloud_result_with_counts(job), "result") + self.assertEqual(job.keys, ["probCount", "measureQubits"]) + + def test_qcloud_result_falls_back_only_for_unsupported_optional_keys(self) -> None: + class LegacyJob: + def __init__(self): self.calls = 0 + def result(self, **kwargs): + self.calls += 1 + if kwargs: + raise RuntimeError("unknown result key probCount") + return "default-result" + job = LegacyJob() + self.assertEqual(Qpanda3Adapter._qcloud_result_with_counts(job), "default-result") + self.assertEqual(job.calls, 2) + + def test_query_execute_uses_qute_contract(self) -> None: + result = self.service.dispatch("query.execute", { + "query_id": "rpc-q", "source": "t", "rows": [{"id": 1, "value": 3}, {"id": 2, "value": 4}], + "filters": [{"column": "value", "op": ">", "value": 3}], "aggregate": {"type": "count"}, "policy": "C0", + }) + self.assertTrue(result["exact"]) + self.assertEqual(result["value"], 1) + self.assertIn("timing_breakdown", result["trace"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/systems/quantum_db/tests/test_qute_framework.py b/systems/quantum_db/tests/test_qute_framework.py new file mode 100644 index 0000000..9a0ad58 --- /dev/null +++ b/systems/quantum_db/tests/test_qute_framework.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path + +from qute.api.query_request import ExecutionPolicy, OptimizationStrategy, QueryRequest +from qute.artifacts.artifact_store import ArtifactStore +from qute.benchmark.runner import run_benchmark +from qute.execution.engine import QuteEngine +from qute.quantum.backend.ideal_simulator import IdealSimulatorBackend +from qute.quantum.backend.noisy_simulator import NoisySimulatorBackend +from qute.quantum.task_model import make_task +from qute.quantum.batching import BatchScheduler +from qute.quantum.tn_adapter import TNPageTaskAdapter +from data_plane.tn.page import TNPage, TNPageMetadata + + +def request(policy: ExecutionPolicy, **overrides) -> QueryRequest: + base = QueryRequest( + "q", "orders", [{"id": i, "amount": float(i + 1), "region": "east" if i % 2 == 0 else "west"} for i in range(64)], + filters=[{"column": "region", "op": "=", "value": "east"}], + aggregate={"type": "qexpect", "column": "amount"}, policy=policy, shots=4096, + quality_budget={"max_error": 0.2, "min_confidence": 0.95, "min_success_probability": 0.7}, + ) + return replace(base, **overrides) + + +class BackendTests(unittest.TestCase): + def test_ideal_operator_contract(self) -> None: + task = make_task("QEXPECT", {"probabilities": {"0": 0.5, "1": 0.5}, "state_values": {"0": -1, "1": 1}}, + shots=4096, seed=1, quality_budget={"max_error": 0.1}, resource_budget={"max_shots": 5000, "max_qubits": 4, "max_depth": 50}) + result = IdealSimulatorBackend(max_qubits=4).execute(task) + self.assertEqual(result.backend, "ideal_simulator") + self.assertEqual(sum(result.histogram.values()), 4096) + self.assertIsNotNone(result.quality.confidence_interval) + + def test_all_initial_quantum_operators(self) -> None: + backend = IdealSimulatorBackend(max_qubits=8) + common = dict(shots=2048, seed=4, quality_budget={"max_error": 0.2}, + resource_budget={"max_shots": 4096, "max_qubits": 8, "max_depth": 100}) + measured = backend.execute(make_task("QMEASURE", {"probabilities": {"0": 0.75, "1": 0.25}}, **common)) + self.assertAlmostEqual(sum(measured.value.values()), 1.0) + similar = backend.execute(make_task("QSIMILARITY", {"left": [1.0, 0.0], "right": [1.0, 0.0]}, **common)) + self.assertAlmostEqual(similar.value, 1.0) + estimated = backend.execute(make_task("QESTIMATE", {"probabilities": {"0": 0.5, "1": 0.5}, + "state_values": {"0": 1.0, "1": 3.0}, "result_scale": 2}, **common)) + self.assertAlmostEqual(estimated.value, 4.0, delta=0.2) + + def test_noisy_feasibility_failure(self) -> None: + task = make_task("QMEASURE", {"probabilities": {"000": 1.0}, "circuit_depth": 20}, shots=10, seed=1, + quality_budget={}, resource_budget={"max_shots": 10, "max_qubits": 1, "max_depth": 5}) + estimate = NoisySimulatorBackend(max_qubits=2, max_depth=10).estimate(task) + self.assertFalse(estimate.backend_feasible) + self.assertTrue(estimate.feasibility_reasons) + + def test_compatible_tasks_are_batched_and_amortized(self) -> None: + backend = IdealSimulatorBackend() + tasks = [make_task("QEXPECT", {"state_ref": "s", "probabilities": {"0": 0.5, "1": 0.5}, + "state_values": {"0": -1, "1": 1}, "observable": f"Z{i}"}, + shots=100, seed=i, quality_budget={"max_error": 1.0}, + resource_budget={"max_shots": 100, "max_qubits": 4, "max_depth": 50}) for i in range(3)] + output = BatchScheduler(backend).execute(tasks) + self.assertEqual(len(output), 1) + results, metrics = output[0] + self.assertEqual(len(results), 3) + self.assertEqual(metrics.batch_size, 3) + self.assertGreaterEqual(metrics.amortized_prepare_time_ms, 0.0) + self.assertGreater(results[0].timings_ms["compile"], 0.0) + self.assertEqual(results[1].timings_ms["compile"], 0.0) + self.assertEqual(results[1].timings_ms["prepare"], 0.0) + + def test_tn_page_adapter_binds_data_version(self) -> None: + page = TNPage(TNPageMetadata(page_id="p", version="data-v1", layout="mps", physical_dims=(2,), bond_dims=(), encoding="fft_topk"), tensors=[]) + adapter = TNPageTaskAdapter() + task = adapter.to_task(page, data_version="data-v1", target_backend="ideal_simulator", operator="QEXPECT", + observable="Z0", shots=100, seed=1, quality_budget={"max_error": 0.2}, + resource_budget={"max_shots": 100, "max_qubits": 32, "max_depth": 500}, + probabilities={"0": 0.5, "1": 0.5}, state_values={"0": -1, "1": 1}) + self.assertEqual(task.payload["data_version"], "data-v1") + with self.assertRaises(ValueError): + adapter.to_task(page, data_version="data-v2", target_backend="ideal_simulator", operator="QEXPECT", + observable="Z0", shots=100, seed=1, quality_budget={}, + resource_budget={"max_shots": 100}) + + +class EngineTests(unittest.TestCase): + def test_c0_is_exact_oracle_and_trace_serializes(self) -> None: + result = QuteEngine(IdealSimulatorBackend()).execute(request(ExecutionPolicy.C0_CLASSICAL)) + payload = result.to_dict() + json.dumps(payload) + self.assertTrue(result.exact) + self.assertEqual(result.trace.selected_path, "classical_exact") + self.assertIn("e2e_ms", payload["trace"]["timing_breakdown"]) + + def test_c1_hybrid_returns_validated_exact(self) -> None: + result = QuteEngine(IdealSimulatorBackend()).execute(request(ExecutionPolicy.C1_MANUAL_HYBRID)) + self.assertTrue(result.exact) + self.assertEqual(result.result_type, "validated_exact") + self.assertFalse(result.trace.fallback_triggered) + + def test_c2_depth_fallback_is_explicit(self) -> None: + constrained = request(ExecutionPolicy.C2_STATIC, resource_budget={"max_depth": 1, "max_qubits": 32, "max_shots": 10000, "max_latency_ms": 5000, "max_two_qubit_gates": 10000}) + result = QuteEngine(IdealSimulatorBackend()).execute(constrained) + self.assertTrue(result.exact) + self.assertTrue(result.trace.fallback_triggered) + self.assertIn("depth", result.trace.fallback_reason) + + def test_backend_unavailable_never_returns_silent_result(self) -> None: + result = QuteEngine(NoisySimulatorBackend(available=False)).execute(request(ExecutionPolicy.C1_MANUAL_HYBRID)) + self.assertTrue(result.exact) + self.assertTrue(result.trace.fallback_triggered) + self.assertIsNotNone(result.trace.structured_error) + + def test_classical_tn_path_uses_same_result_contract(self) -> None: + result = QuteEngine(IdealSimulatorBackend()).execute(request(ExecutionPolicy.C3_ADAPTIVE, force_backend="tn")) + self.assertIn(result.result_type, {"tn_estimate", "fallback_exact"}) + self.assertIn(result.trace.backend, {"classical_tn", "classical_local"}) + self.assertIsNotNone(result.trace.quality_contract.estimated_error) + + def test_c0_c1_c2_c3_agree_within_budget_or_fallback_exact(self) -> None: + engine = QuteEngine(IdealSimulatorBackend()) + oracle = engine.execute(request(ExecutionPolicy.C0_CLASSICAL)).value + for policy in ExecutionPolicy: + result = engine.execute(request(policy)) + if result.exact: + self.assertEqual(result.value, oracle) + else: + error = abs(float(result.value) - float(oracle)) / abs(float(oracle)) + self.assertLessEqual(error, result.quality.error_budget) + + def test_trace_contains_complete_timing_breakdown(self) -> None: + trace = QuteEngine(IdealSimulatorBackend()).execute(request(ExecutionPolicy.C1_MANUAL_HYBRID)).to_dict()["trace"] + expected = {"plan_ms", "prepare_ms", "compile_ms", "submit_ms", "queue_ms", "execute_ms", + "measure_ms", "postprocess_ms", "validation_ms", "fallback_ms", "e2e_ms"} + self.assertEqual(set(trace["timing_breakdown"]), expected) + + def test_all_optimizer_strategies_are_explicit_and_safe(self) -> None: + engine = QuteEngine(IdealSimulatorBackend()) + selected = {} + for strategy in OptimizationStrategy: + result = engine.execute(request(ExecutionPolicy.C3_ADAPTIVE, optimization_strategy=strategy)) + selected[strategy] = result.trace.selected_path + self.assertEqual(result.trace.metadata["optimization_strategy"], strategy.value) + self.assertEqual(selected[OptimizationStrategy.CLASSICAL_FIRST], "classical_exact") + self.assertIn(selected[OptimizationStrategy.QUANTUM_FIRST], {"quantum", "classical_exact"}) + self.assertIn(selected[OptimizationStrategy.FALLBACK_SAFE], {"hybrid", "classical_exact"}) + + +class ArtifactAndBenchmarkTests(unittest.TestCase): + def test_version_change_stales_artifact(self) -> None: + store = ArtifactStore() + key = dict(data_source="orders", data_version="v1", encoding_version="e", circuit_version="c", observable="z", + backend="ideal", noise_model="none", transpilation_config="default", shots=100) + artifact, hit = store.get_or_create(**key) + self.assertFalse(hit) + self.assertEqual(store.invalidate_data_version("orders", "v2"), 1) + self.assertEqual(artifact.state, "stale") + _, hit = store.get_or_create(**key) + self.assertFalse(hit) + + def test_invalidation_is_scoped_to_authoritative_source(self) -> None: + store = ArtifactStore() + common = dict(data_version="v1", encoding_version="e", circuit_version="c", observable="z", + backend="ideal", noise_model="none", transpilation_config="default", shots=100) + orders, _ = store.get_or_create(data_source="orders", **common) + customers, _ = store.get_or_create(data_source="customers", **common) + self.assertEqual(store.invalidate_data_version("orders", "v2"), 1) + self.assertEqual(orders.state, "stale") + self.assertIn(customers.state, {"cached", "reused"}) + + def test_engine_automatically_invalidates_previous_data_version(self) -> None: + store = ArtifactStore() + engine = QuteEngine(IdealSimulatorBackend(), artifact_store=store) + engine.execute(request(ExecutionPolicy.C1_MANUAL_HYBRID, source="orders", data_version="v1")) + result = engine.execute(request(ExecutionPolicy.C0_CLASSICAL, source="orders", data_version="v2")) + self.assertGreaterEqual(result.trace.metadata["invalidated_artifacts"], 1) + + def test_smoke_benchmark_persists_required_outputs(self) -> None: + with tempfile.TemporaryDirectory() as directory: + metrics = run_benchmark({"seed": 3, "dataset_size": 64, "warm_repetitions": 2, "repetitions": 2, + "backend": {"type": "ideal", "max_qubits": 32, "max_depth": 500}}, directory) + self.assertTrue(metrics) + for name in ("config.json", "dataset_metadata.json", "artifacts.json", "execution.log.jsonl", "raw_traces.jsonl", "metrics.csv", "summary.json", "calibration.json", "batch_metrics.json", "tn_ablation_assessment.json", "report.md"): + self.assertTrue((Path(directory) / name).exists(), name) + self.assertEqual(len(list((Path(directory) / "plots").glob("*.svg"))), 8) + self.assertTrue(any(row["fallback_triggered"] for row in metrics)) + self.assertTrue(all("e2e_ms" in row for row in metrics)) + summary = json.loads((Path(directory) / "summary.json").read_text(encoding="utf-8")) + self.assertTrue(all("ci95_e2e_ms" in group for group in summary.values())) + + +if __name__ == "__main__": + unittest.main()