Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/us-ss-spm-release.changed
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Update the bundled US release to policyengine-us 1.691.3 and policyengine-us-data 1.113.1.
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ uk = [
]
us = [
"policyengine_core>=3.26.1",
"policyengine-us==1.690.7",
"policyengine-us==1.691.3",
]
dev = [
"pytest",
Expand All @@ -61,7 +61,7 @@ dev = [
"ruff>=0.9.0",
"policyengine_core>=3.26.1",
"policyengine-uk==2.88.14",
"policyengine-us==1.690.7",
"policyengine-us==1.691.3",
"towncrier>=24.8.0",
"mypy>=1.11.0",
"pytest-cov>=5.0.0",
Expand Down
28 changes: 14 additions & 14 deletions src/policyengine/data/release_manifests/us.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,34 +5,34 @@
"policyengine_version": "4.4.3",
"model_package": {
"name": "policyengine-us",
"version": "1.690.7",
"sha256": "5a7a541efabac98fa069d6845902cf5924c81db67383234b55dcd2b8bfcfc3ca",
"wheel_url": "https://files.pythonhosted.org/packages/2a/02/52109bae5f4767237b43bd72ce0bc4edf7925650a788053b2bc168caa5ae/policyengine_us-1.690.7-py3-none-any.whl"
"version": "1.691.3",
"sha256": "c5d37aa4442f23d48bd5d587a02876c89d83c6135809f12988cc39bd3a47e8b2",
"wheel_url": "https://files.pythonhosted.org/packages/2a/03/e21c872664f90dcc99f1fcf29d1da71409c50cf8a7798ff0596ad10d9400/policyengine_us-1.691.3-py3-none-any.whl"
},
"data_package": {
"name": "policyengine-us-data",
"version": "1.110.12",
"version": "1.113.1",
"repo_id": "policyengine/policyengine-us-data",
"release_manifest_path": "releases/1.110.12/release_manifest.json",
"release_manifest_revision": "3aac4505ec10d31efc1b3799a1e6458a15853ecc"
"release_manifest_path": "releases/1.113.1/release_manifest.json",
"release_manifest_revision": "99e0ec7e784cdba43dd21ff1d80a081599a7a537"
},
"certified_data_artifact": {
"data_package": {
"name": "policyengine-us-data",
"version": "1.110.12"
"version": "1.113.1"
},
"build_id": "policyengine-us-data-1.110.12",
"build_id": "policyengine-us-data-1.113.1",
"dataset": "enhanced_cps_2024",
"uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.110.12",
"sha256": "58a6639f7511b8d804701417e2647f0c3a77f51a3d90441037eaf004b1f00761"
"uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@99e0ec7e784cdba43dd21ff1d80a081599a7a537",
"sha256": "0ce549af18753287c097718362b8cd5cdccfc47953acf7f282709d604cf314d2"
},
"certification": {
"compatibility_basis": "exact_build_model_version",
"data_build_id": "policyengine-us-data-1.110.12",
"built_with_model_version": "1.690.7",
"certified_for_model_version": "1.690.7",
"data_build_id": "policyengine-us-data-1.113.1",
"built_with_model_version": "1.691.3",
"certified_for_model_version": "1.691.3",
"certified_by": "policyengine.py bundled manifest",
"data_build_fingerprint": "sha256:9961ed1c5d00943a360724da560eee425eb9f99f91896f053dca74724c46e96e"
"data_build_fingerprint": "sha256:d891044ece8ec3338904771c879b98ec11a12f1090c074e5b8cee846825d8056"
},
"default_dataset": "enhanced_cps_2024",
"datasets": {
Expand Down
39 changes: 18 additions & 21 deletions src/policyengine/data/release_manifests/us.trace.tro.jsonld
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"schema:name": "PolicyEngine",
"schema:url": "https://policyengine.org"
},
"schema:dateCreated": "2026-05-11T18:53:05.508006Z",
"schema:dateCreated": "2026-05-13T03:56:37.150215Z",
"schema:description": "TRACE TRO for certified runtime bundle us-4.4.3 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.",
"schema:name": "policyengine us certified bundle TRO",
"trov:createdWith": {
Expand Down Expand Up @@ -45,23 +45,23 @@
"trov:hasArtifact": {
"@id": "composition/1/artifact/data_release_manifest"
},
"trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/3aac4505ec10d31efc1b3799a1e6458a15853ecc/releases/1.110.12/release_manifest.json"
"trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/99e0ec7e784cdba43dd21ff1d80a081599a7a537/releases/1.113.1/release_manifest.json"
},
{
"@id": "arrangement/1/location/dataset",
"@type": "trov:ArtifactLocation",
"trov:hasArtifact": {
"@id": "composition/1/artifact/dataset"
},
"trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/1.110.12/enhanced_cps_2024.h5"
"trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/99e0ec7e784cdba43dd21ff1d80a081599a7a537/enhanced_cps_2024.h5"
},
{
"@id": "arrangement/1/location/model_wheel",
"@type": "trov:ArtifactLocation",
"trov:hasArtifact": {
"@id": "composition/1/artifact/model_wheel"
},
"trov:hasLocation": "https://files.pythonhosted.org/packages/2a/02/52109bae5f4767237b43bd72ce0bc4edf7925650a788053b2bc168caa5ae/policyengine_us-1.690.7-py3-none-any.whl"
"trov:hasLocation": "https://files.pythonhosted.org/packages/2a/03/e21c872664f90dcc99f1fcf29d1da71409c50cf8a7798ff0596ad10d9400/policyengine_us-1.691.3-py3-none-any.whl"
}
]
}
Expand All @@ -75,54 +75,51 @@
"@type": "trov:ResearchArtifact",
"schema:name": "policyengine.py bundle manifest for us",
"trov:mimeType": "application/json",
"trov:sha256": "41e196a6263b8168d403058029c52ebab795e17024ac9ebef11ff876e36959e2"
"trov:sha256": "67d4d6505bed4af9bf2ec575d8b037e36be71b2f9a5afa9bb8cc695ec7a1e913"
},
{
"@id": "composition/1/artifact/data_release_manifest",
"@type": "trov:ResearchArtifact",
"schema:name": "policyengine-us-data release manifest 1.110.12",
"schema:name": "policyengine-us-data release manifest 1.113.1",
"trov:mimeType": "application/json",
"trov:sha256": "17cfd2fbb31064834ed82c0fd7d8ae5c272fe7f24b1e48b226a4acf97ff4c5dd"
"trov:sha256": "d6b29ceff0cbf6a5cff4de94362ebc533dc5044c6a4155a46da7143140a8cb5f"
},
{
"@id": "composition/1/artifact/dataset",
"@type": "trov:ResearchArtifact",
"schema:name": "enhanced_cps_2024",
"trov:mimeType": "application/x-hdf5",
"trov:sha256": "58a6639f7511b8d804701417e2647f0c3a77f51a3d90441037eaf004b1f00761"
"trov:sha256": "0ce549af18753287c097718362b8cd5cdccfc47953acf7f282709d604cf314d2"
},
{
"@id": "composition/1/artifact/model_wheel",
"@type": "trov:ResearchArtifact",
"schema:name": "policyengine-us==1.690.7 wheel",
"schema:name": "policyengine-us==1.691.3 wheel",
"trov:mimeType": "application/zip",
"trov:sha256": "5a7a541efabac98fa069d6845902cf5924c81db67383234b55dcd2b8bfcfc3ca"
"trov:sha256": "c5d37aa4442f23d48bd5d587a02876c89d83c6135809f12988cc39bd3a47e8b2"
}
],
"trov:hasFingerprint": {
"@id": "composition/1/fingerprint",
"@type": "trov:CompositionFingerprint",
"trov:sha256": "b84e895b3f19ffee5ec299b94ae2155448a12af2b3dc61d00f3d17003ecdf14a"
"trov:sha256": "316e373ed13360efa12037200719c9621ce7bced6d80acc4dfa7bbb72962892f"
}
},
"trov:hasPerformance": {
"@id": "trp/1",
"@type": "trov:TransparentResearchPerformance",
"pe:builtWithModelVersion": "1.690.7",
"pe:builtWithModelVersion": "1.691.3",
"pe:certifiedBy": "policyengine.py bundled manifest",
"pe:certifiedForModelVersion": "1.690.7",
"pe:ciGitRef": "refs/heads/main",
"pe:ciGitSha": "1718b493e4749faf62f0ffdff480205abdd20011",
"pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/25692889721",
"pe:certifiedForModelVersion": "1.691.3",
"pe:compatibilityBasis": "exact_build_model_version",
"pe:dataBuildFingerprint": "sha256:9961ed1c5d00943a360724da560eee425eb9f99f91896f053dca74724c46e96e",
"pe:dataBuildId": "policyengine-us-data-1.110.12",
"pe:emittedIn": "github-actions",
"rdfs:comment": "Certification of build policyengine-us-data-1.110.12 for policyengine-us 1.690.7.",
"pe:dataBuildFingerprint": "sha256:d891044ece8ec3338904771c879b98ec11a12f1090c074e5b8cee846825d8056",
"pe:dataBuildId": "policyengine-us-data-1.113.1",
"pe:emittedIn": "local",
"rdfs:comment": "Certification of build policyengine-us-data-1.113.1 for policyengine-us 1.691.3.",
"trov:accessedArrangement": {
"@id": "arrangement/1"
},
"trov:startedAtTime": "2026-05-11T18:53:05.508006Z",
"trov:startedAtTime": "2026-05-13T03:56:37.150215Z",
"trov:wasConductedBy": {
"@id": "trs"
}
Expand Down
7 changes: 7 additions & 0 deletions src/policyengine/provenance/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,13 @@ def refresh_release_bundle(
dataset_repo_id = data_artifact_json.get("repo_id", repo_id)
dataset_path = data_artifact_json.get("path", dataset_path)
dataset_revision = data_artifact_json.get("revision", new_data)
if (
release_manifest_json is not None
and new_release_manifest_revision is not None
and dataset_repo_id == repo_id
and dataset_revision == new_data
):
dataset_revision = new_release_manifest_revision

# Only hit HF if the data version actually changed.
if new_data != old_data:
Expand Down
28 changes: 24 additions & 4 deletions src/policyengine/provenance/manifest.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import hashlib
import os
from functools import lru_cache
from importlib import import_module
Expand Down Expand Up @@ -127,6 +128,8 @@ class DataReleaseManifest(BaseModel):
can enclose the full set of artifacts published together). Distinct
from per-artifact DOIs on ``DataReleaseArtifact.preservation_mirrors``.
Populated when the release pipeline mirrors to a DOI-minting host."""
source_sha256: Optional[str] = Field(default=None, exclude=True)
"""Byte sha256 of the fetched manifest before runtime URI rewrites."""


class DataCertification(BaseModel):
Expand Down Expand Up @@ -180,9 +183,13 @@ def https_dataset_uri(repo_id: str, path_in_repo: str, revision: str) -> str:
return f"https://huggingface.co/{repo_id}/resolve/{revision}/{path_in_repo}"


def _artifact_revision(data_package: "DataPackageVersion") -> str:
return data_package.release_manifest_revision or data_package.version


def https_release_manifest_uri(data_package: "DataPackageVersion") -> str:
"""Return a dereferenceable HTTPS URI for a data release manifest."""
revision = data_package.release_manifest_revision or data_package.version
revision = _artifact_revision(data_package)
return (
f"https://huggingface.co/{data_package.repo_id}/resolve/"
f"{revision}/{data_package.release_manifest_path}"
Expand Down Expand Up @@ -267,7 +274,20 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest:
raise DataReleaseManifestUnavailableError(
"Could not fetch the data release manifest from Hugging Face."
) from exc
return DataReleaseManifest.model_validate_json(response.text)
data_release_manifest = DataReleaseManifest.model_validate_json(response.text)
source_bytes = response.content
if not isinstance(source_bytes, bytes):
source_bytes = response.text.encode("utf-8")
data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest()
release_revision = country_manifest.data_package.release_manifest_revision
if release_revision is not None:
for artifact in data_release_manifest.artifacts.values():
if (
artifact.repo_id == country_manifest.data_package.repo_id
and artifact.revision == country_manifest.data_package.version
):
artifact.revision = release_revision
return data_release_manifest


def _specifier_matches(version: str, specifier: str) -> bool:
Expand Down Expand Up @@ -404,7 +424,7 @@ def resolve_dataset_reference(country_id: str, dataset: str) -> str:
return build_hf_uri(
repo_id=manifest.data_package.repo_id,
path_in_repo=path_reference.path,
revision=manifest.data_package.version,
revision=_artifact_revision(manifest.data_package),
)

data_release_manifest = get_data_release_manifest(country_id)
Expand Down Expand Up @@ -525,5 +545,5 @@ def resolve_region_dataset_path(
return build_hf_uri(
repo_id=manifest.data_package.repo_id,
path_in_repo=resolved_path,
revision=manifest.data_package.version,
revision=_artifact_revision(manifest.data_package),
)
13 changes: 3 additions & 10 deletions src/policyengine/provenance/trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,21 +353,14 @@ def build_trace_tro_from_release_bundle(
if data_release_manifest is not None
else None
)
dataset_location = (
https_dataset_uri(
repo_id=dataset_artifact.repo_id,
path_in_repo=dataset_artifact.path,
revision=dataset_artifact.revision,
)
if dataset_artifact is not None
else _dataset_location_from_uri(certified_artifact.uri)
)
dataset_location = _dataset_location_from_uri(certified_artifact.uri)

bundle_manifest_hash = hashlib.sha256(
canonical_json_bytes(country_manifest.model_dump(mode="json"))
).hexdigest()
data_release_manifest_hash = (
hashlib.sha256(
data_release_manifest.source_sha256
or hashlib.sha256(
canonical_json_bytes(data_release_manifest.model_dump(mode="json"))
).hexdigest()
if data_release_manifest is not None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"has_income_tax": true,
"has_region_registry": true,
"model_package_name": "policyengine-us",
"num_parameters_bucketed_100s": 851,
"num_parameters_bucketed_100s": 852,
"num_variables_bucketed_100s": 48,
"region_registry_country": "us"
}
8 changes: 8 additions & 0 deletions tests/test_bundle_refresh.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,10 @@ def fake_urlopen(request, *args, **kwargs):
written["data_package"]["release_manifest_revision"]
== "release-manifest-commit-sha"
)
assert (
written["certified_data_artifact"]["uri"]
== "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@release-manifest-commit-sha"
)


def test__bump_data_only_falls_back_to_main_for_release_manifest(
Expand Down Expand Up @@ -337,6 +341,10 @@ def fake_urlopen(request, *args, **kwargs):
written["data_package"]["release_manifest_revision"]
== "release-manifest-commit-sha"
)
assert (
written["certified_data_artifact"]["uri"]
== "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@release-manifest-commit-sha"
)


def test__release_manifest_version_mismatch_raises(sandbox) -> None:
Expand Down
6 changes: 3 additions & 3 deletions tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,12 @@ def test_has_release_manifest_metadata(self):
assert us_latest.release_manifest is not None
assert us_latest.release_manifest.country_id == "us"
assert us_latest.model_package.name == "policyengine-us"
assert us_latest.model_package.version == "1.690.7"
assert us_latest.model_package.version == "1.691.3"
assert us_latest.data_package.name == "policyengine-us-data"
assert us_latest.data_package.version == "1.110.12"
assert us_latest.data_package.version == "1.113.1"
assert (
us_latest.default_dataset_uri
== "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.110.12"
== "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@99e0ec7e784cdba43dd21ff1d80a081599a7a537"
)

def test_has_hundreds_of_parameters(self):
Expand Down
Loading