From ef321229869b373e393bf2a9cf5792febab3167b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 26 May 2026 15:33:05 +0200 Subject: [PATCH 01/13] =?UTF-8?q?test(resource-docs):=20pin=20doc-listing?= =?UTF-8?q?=20cascade=20surface=20=E2=80=94=20v2.0.0-origin=20endpoints=20?= =?UTF-8?q?visible=20under=20newer-version=20prefixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to commit 9c9b5fee3 (the NMB fix that restored cascade reachability at runtime when `api_disabled_versions` skips a middle version). That commit pins the *runtime* side of the cascade contract via ResourceDocMiddlewareEnableDisablePropsTest. This commit pins the *documentation* side: when an operator skips v2.0.0 in `api_enabled_versions`, the v2.0.0-origin endpoints stay reachable via newer prefixes — they must therefore also still appear in the corresponding `/resource-docs/.../obp` responses, or callers can't discover what's reachable. The NMB-reported endpoint `Add Entitlement for User` was exactly this case. Two scenarios added to SwaggerDocsTest: 1. `GET /obp/v6.0.0/resource-docs/v6.0.0/obp` with the NMB-style `api_enabled_versions` (skips v2.0.0) — the response must still include the v2.0.0-origin `addEntitlement` doc. 2. The same response surfaces a non-trivial number (>5) of v2.0.0-origin endpoints — a regression that drops most of the cascade fails even if one or two stay. If `getResourceDocsList` ever starts filtering by `versionIsAllowed(rd.implementedInApiVersion)`, or the `OBPAPI{version}.allResourceDocs` cascade chain breaks, these tests fail and force a re-think before the change ships. SwaggerDocsTest now has 14 scenarios; was 12. --- .../ResourceDocs1_4_0/SwaggerDocsTest.scala | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/SwaggerDocsTest.scala b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/SwaggerDocsTest.scala index d015851682..ad29e08990 100644 --- a/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/SwaggerDocsTest.scala +++ b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/SwaggerDocsTest.scala @@ -252,4 +252,58 @@ class SwaggerDocsTest extends ResourceDocsV140ServerSetup with PropsReset with D } + // ─── Doc-listing surface mirrors the runtime cascade ──────────────────────── + // + // The runtime side is pinned by ResourceDocMiddlewareEnableDisablePropsTest's + // "cascade reachability survives api_disabled_versions on the middle version" + // feature (commit 9c9b5fee3 — the NMB fix where `Add Entitlement for User` + // disappeared because the operator skipped v2.0.0 in api_enabled_versions). + // This feature pins the *documentation* side: a newer version's + // /resource-docs/.../obp response must include the v2.0.0-origin endpoints + // that cascade into it. Without this, an operator who disables v2.0.0 sees + // those endpoints disappear from the docs even though they're still + // reachable, which is the kind of doc/runtime drift that confused us once. + // + // If `getResourceDocsList` ever starts filtering by + // versionIsAllowed(rd.implementedInApiVersion), or the OBPAPI{version} + // cascade chain breaks, these scenarios will fail and force a re-think + // before the change ships. + feature("ResourceDoc listing — v2.0.0-origin endpoints cascade into newer versions' docs") { + + scenario("GET /obp/v6.0.0/resource-docs/v6.0.0/obp lists the v2.0.0-origin addEntitlement endpoint", ApiEndpoint1, VersionOfApi) { + Given("api_enabled_versions skips v2.0.0 (matching the NMB-reported config)") + setPropsValues( + "resource_docs_requires_role" -> "false", + "api_enabled_versions" -> "[OBPv7.0.0,OBPv2.1.0,OBPv6.0.0,OBPv5.1.0,OBPv5.0.0,OBPv4.0.0,OBPv3.0.0,OBPv3.1.0]" + ) + When("requesting v6.0.0's resource-docs by API version v6.0.0") + val resp = makeGetRequest((ResourceDocsV6_0Request / "resource-docs" / "v6.0.0" / "obp").GET) + Then("the response is 200") + resp.code should equal(200) + And("addEntitlement (introduced in v2.0.0) is in the returned doc list") + // The response body is a JObject {resource_docs: [...]} — pluck operation_id + val opIds = (resp.body \ "resource_docs" \\ "operation_id").children.collect { + case JString(s) => s + } + opIds.exists(_.contains("addEntitlement")) shouldBe true + } + + scenario("requested API version v6.0.0 surfaces a non-trivial number of v2.0.0-origin endpoints — proves the whole cascade, not one doc", ApiEndpoint1, VersionOfApi) { + setPropsValues("resource_docs_requires_role" -> "false") + val resp = makeGetRequest((ResourceDocsV6_0Request / "resource-docs" / "v6.0.0" / "obp").GET) + resp.code should equal(200) + // implemented_by.version tags every doc with its origin (fullyQualifiedVersion, + // e.g. "OBPv2.0.0"). The v2.0.0 group should be sizable (>5) so a regression + // that drops most of the cascade fails this even if a couple stay. + val docs = (resp.body \ "resource_docs").children + val v2_0_0Count = docs.count { d => + (d \ "implemented_by" \ "version") match { + case JString(v) => v == "OBPv2.0.0" + case _ => false + } + } + v2_0_0Count should be > 5 + } + } + } From c728ba78acdd1aa27352e6c123de6cc07cfaa503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 26 May 2026 16:31:31 +0200 Subject: [PATCH 02/13] docs+test(retired-standards): mark BahrainOBF/AU/STET/Polish/MxOF retired in migration doc + regression-guard test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to PR #2814 (commit d19af2b92) which commented out 5 non-OBP Lift API standards. The disable is structural — `ClassScanUtils.getSubTypeObjects` finds no `ScannedApis` in those packages, so Lift's dispatch never sees them and `Boot.scala` needs no change. The flip side is that a partial uncomment of any retired-standard file would silently re-register that standard at the next startup. Two changes: 1. **`LIFT_HTTP4S_MIGRATION.md`** — replace the five "Lift" rows in the Open-banking standards table with "Retired" (strike-through) entries, add a paragraph explaining the comment-out pattern + the deletion candidate timeline, and forward-reference the new regression-guard test. Section heading drops "(large, deferred indefinitely)" — only the BG v1.3 / UKOpenBanking / sandbox rows remain in-scope. 2. **`RetiredApiStandardsTest`** — scenario `"ScannedApis registry must not contain any object from a retired-standard package"`. Walks the live `ScannedApis.versionMapScannedApis` map and asserts no entry's fully-qualified class name starts with one of: code.api.BahrainOBF. code.api.AUOpenBanking. code.api.STET. code.api.Polish. code.api.MxOF. If anyone partially uncomments a retired standard, this test fails with the exact class name they brought back, and forces a re-think before the change ships. Uses `V400ServerSetup` because the scan needs a real classpath view (same hierarchy as `ApiVersionUtilsTest`, which counts scanned versions 26 → 20). Currently passes — confirms the empirical "zero scanned classes in the retired packages" invariant after PR #2814. --- LIFT_HTTP4S_MIGRATION.md | 22 ++++--- .../util/http4s/RetiredApiStandardsTest.scala | 65 +++++++++++++++++++ 2 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/http4s/RetiredApiStandardsTest.scala diff --git a/LIFT_HTTP4S_MIGRATION.md b/LIFT_HTTP4S_MIGRATION.md index cf4653b990..95a68946d5 100644 --- a/LIFT_HTTP4S_MIGRATION.md +++ b/LIFT_HTTP4S_MIGRATION.md @@ -432,13 +432,13 @@ Already partly described in the next major section, but counted here for complet | `ImporterAPI` | (deleted) | **Retired.** The legacy `POST /obp_transactions_saver/api/transactions` shared-secret bulk-insert endpoint, its `TransactionInserter` LiftActor, and the connector helpers it relied on (`createImportedTransaction`, `getMatchingTransactionCount`, `updateAccountBalance`, `setBankAccountLastUpdated`) have been removed entirely. Modern callers use connector-driven flows or the `/obp/vX.X.X/transaction-requests/...` endpoints. | | `OpenIdConnect` | (auth-stack table above) | OIDC callback, registered separately from OAuth2. | -### Open-banking standards (large, deferred indefinitely) +### Open-banking standards -Lift implementations of 3rd-party regulatory standards. All currently pass through `Http4sLiftWebBridge` and continue to work; they are *not* OBP API per se but optional regulatory shims. Migrating them is out of scope for the "remove Lift Web" milestone if you accept keeping the bridge for these stacks only. If total Lift removal is the goal, each needs its own workstream. +Lift implementations of 3rd-party regulatory standards. Each is *not* OBP API per se but an optional regulatory shim. Reflection via `ClassScanUtils.getSubTypeObjects` is what registers them with Lift's dispatch — so commenting out the source removes them from the registry without touching `Boot.scala`. -Three forks for how this workstream resolves: +Three forks for how the still-active workstream resolves: -- **(a) Migrate each to http4s.** Weeks per standard × 7 standards. Highest cost; cleanest end state. +- **(a) Migrate each to http4s.** Weeks per standard. Highest cost; cleanest end state. - **(b) "Regulatory mode" feature-flagged Lift.** Keep `Http4sLiftWebBridge` wired in only when an `obf-*` / standards prop is set; otherwise the bridge is unregistered at boot. Lets "Lift Web removed from the OBP API path" ship, but Lift Web stays in the codebase as an opt-in shim. Defeats the milestone technically; ships the headline. - **(c) Extract as plugin projects.** Move each standard out of this repo into its own project that depends on OBP API. Probably right long-term — these are optional, externally-governed standards on different release cadences — but socially expensive and reshapes the build. @@ -447,13 +447,17 @@ Three forks for how this workstream resolves: | Berlin Group v1.3 | `code/api/berlin/group/v1_3/*` — 7 files (AIS / PIS / PIIS / signing baskets / common) | Lift | | **Berlin Group v2** | `code/api/berlin/group/v2/Http4sBGv2.scala` | ✅ already on http4s | | UK Open Banking v2.0.0 + v3.1.0 | `code/api/UKOpenBanking/*` — ~20 files | Lift | -| Bahrain OBF v1.0.0 | `code/api/BahrainOBF/*` — ~20 files | Lift | -| AU OpenBanking v1.0.0 | `code/api/AUOpenBanking/*` — ~10 files | Lift | -| STET v1.4 | `code/api/STET/v1_4/*` — 4 files | Lift | -| MxOF v1.0.0 | `code/api/MxOF/*` — 2 files | Lift | -| Polish v2.1.1.1 | `code/api/Polish/v2_1_1_1/*` — 4 files | Lift | +| ~~Bahrain OBF v1.0.0~~ | `code/api/BahrainOBF/*` | ✅ **Retired** — commented out in PR #2814 (`d19af2b92`, 2026-05-22). ScannedApis reflection no longer finds it; URLs 404 with no bridge handling. | +| ~~AU OpenBanking v1.0.0~~ | `code/api/AUOpenBanking/*` | ✅ **Retired** — same PR / commit. | +| ~~STET v1.4~~ | `code/api/STET/v1_4/*` | ✅ **Retired** — same PR / commit. | +| ~~MxOF / CNBV9 v1.0.0~~ | `code/api/MxOF/*` | ✅ **Retired** — same PR / commit. | +| ~~Polish v2.1.1.1~~ | `code/api/Polish/v2_1_1_1/*` | ✅ **Retired** — same PR / commit. | | Sandbox / `SandboxApiCalls.scala` | `code/api/sandbox/*` | Lift | +The retired five total ~22,000 lines of Scala kept in-tree as line-comments. The same `// ` stub pattern used for the per-version OBPAPI files; uncomment to bring them back. After ~1 month with no operator complaints and the bridge audit showing zero hits for their URLs, candidates for outright deletion. + +A regression-guard test (`code.api.util.http4s.RetiredApiStandardsTest`) asserts that `ClassScanUtils.getSubTypeObjects` does **not** return any object whose package matches the retired standards — so a partial uncomment that re-registers them with Lift trips CI. + ### `Boot.scala` scaffolding Currently runs on startup and goes away once the Lift bridge is removable: diff --git a/obp-api/src/test/scala/code/api/util/http4s/RetiredApiStandardsTest.scala b/obp-api/src/test/scala/code/api/util/http4s/RetiredApiStandardsTest.scala new file mode 100644 index 0000000000..2ae0462c3c --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/http4s/RetiredApiStandardsTest.scala @@ -0,0 +1,65 @@ +package code.api.util.http4s + +import code.api.util.ScannedApis +import code.api.v4_0_0.V400ServerSetup +import org.scalatest.Tag + +/** + * Regression-guard for the "retire by commenting-out" pattern used to take + * non-OBP Lift API standards (BahrainOBF, AUOpenBanking, STET, Polish, MxOF/CNBV9) + * off the bridge in PR #2814 (commit `d19af2b92`, 2026-05-22). + * + * Background: each standard is registered with Lift via reflection in + * `ScannedApis.versionMapScannedApis`, which calls + * `ClassScanUtils.getSubTypeObjects[ScannedApis]`. Commenting out the source + * removes the `OBPAPIxxx` objects from the classpath, so the reflection finds + * nothing and the routes stop being served. The disable is "structural" — no + * `Boot.scala` change — which makes it cheap but also reversible by accident: + * a partial uncomment that brings back even one object would silently + * re-register that standard at startup. + * + * This test asserts the inverse: at the moment of the scan, none of the + * `ScannedApis` instances live in a package matching the five retired + * standards. If any does, someone has put a piece of one of those standards + * back in business; review what they uncommented and decide whether that's + * intentional. + * + * The test sits in the v4 server-setup hierarchy (same hierarchy as + * `ApiVersionUtilsTest`) because the scan only sees compiled classes — a + * fresh test JVM with the test classpath has the right view. + */ +class RetiredApiStandardsTest extends V400ServerSetup { + + object RetiredStandardsTag extends Tag("RetiredStandards") + + // Packages whose every concrete object/class extends `ScannedApis`. Any + // entry showing up under one of these prefixes means a once-retired + // standard has been (partially) brought back to life. + private val retiredPackagePrefixes: Set[String] = Set( + "code.api.BahrainOBF.", + "code.api.AUOpenBanking.", + "code.api.STET.", + "code.api.Polish.", + "code.api.MxOF." + ) + + feature("Retired API standards stay retired") { + + scenario("ScannedApis registry must not contain any object from a retired-standard package", RetiredStandardsTag) { + Given("`ScannedApis.versionMapScannedApis` is built via `ClassScanUtils.getSubTypeObjects`") + val scanned = ScannedApis.versionMapScannedApis.values.toList + + When("we inspect each scanned object's fully-qualified class name") + val resurrected: List[(String, String)] = + scanned.flatMap { obj => + val cls = obj.getClass.getName + retiredPackagePrefixes.collectFirst { + case pkg if cls.startsWith(pkg) => (pkg.stripSuffix("."), cls) + } + } + + Then(s"no scanned object should live in a retired-standard package, but found: $resurrected") + resurrected shouldBe empty + } + } +} From 8dfdae9ae989783d3eb23398287b5d2cf73add02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 27 May 2026 16:04:08 +0200 Subject: [PATCH 03/13] refactor(directlogin): remove dead Lift dlServe block + vestigial RestHelper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bare `POST /my/logins/direct` route has been served by `code.api.DirectLoginRoutes` (wired into `Http4sApp.baseServices`) since `LiftRules.statelessDispatch.append(DirectLogin)` was removed from `Boot.scala`. That left the `dlServe` self-registration block inside `DirectLogin` as dead code — `super.serve(...)` only does anything when the object is registered with `LiftRules`, which it no longer is. Removed: - the `dlServe` helper (its only job was wrapping a handler in `RestHelper.serve`), - the `dlServe { case Req("my" :: "logins" :: "direct" :: Nil, _, PostRequest) }` registration block, - `extends RestHelper` (now only `MdcLoggable`) — nothing outside the dead block used a RestHelper member; there were no `.extract`/`Formats` call sites, so no local `Formats` replacement was needed (unlike the OAuth2/GatewayLogin/DAuth de-RestHelper cleanups), - the now-unused `RestHelper` and `NewStyle.HttpCode` imports. Kept (still have live callers): `createTokenCommonPart`, `getAllParameters`, `validator`, `validatorFuture`, `validatorFutureWithParams`, `getUserFromDirectLoginHeaderFuture`, `getUser`, `getConsumer`, `grantEntitlementsToUseDynamicEndpointsInSpacesInDirectLogin`, and the token/consumer/user lookup helpers used by the auth flows. No behavioural change: the removed block was never reachable after the Boot dispatch removal. obp-api compiles; no test referenced the removed members. --- .../src/main/scala/code/api/directlogin.scala | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/obp-api/src/main/scala/code/api/directlogin.scala b/obp-api/src/main/scala/code/api/directlogin.scala index fa2f8ca31a..8f08b00b8e 100644 --- a/obp-api/src/main/scala/code/api/directlogin.scala +++ b/obp-api/src/main/scala/code/api/directlogin.scala @@ -30,7 +30,6 @@ import java.util.Date import code.api.util.APIUtil._ import code.api.util.ErrorMessages.{InvalidDirectLoginParameters, attemptedToOpenAnEmptyBox} -import code.api.util.NewStyle.HttpCode import code.api.util._ import code.consumer.Consumers._ import code.model.dataAccess.AuthUser @@ -44,7 +43,6 @@ import com.openbankproject.commons.ExecutionContext.Implicits.global import com.openbankproject.commons.model.User import net.liftweb.common._ import net.liftweb.http._ -import net.liftweb.http.rest.RestHelper import net.liftweb.mapper.{By, By_>, Descending, OrderBy} import net.liftweb.util.Helpers import net.liftweb.util.Helpers.tryo @@ -79,39 +77,8 @@ object JSONFactory { } } -object DirectLogin extends RestHelper with MdcLoggable { +object DirectLogin extends MdcLoggable { - // Our version of serve - def dlServe(handler : PartialFunction[Req, JsonResponse]) : Unit = { - val obpHandler : PartialFunction[Req, () => Box[LiftResponse]] = { - new PartialFunction[Req, () => Box[LiftResponse]] { - def apply(r : Req) = { - handler(r) - } - def isDefinedAt(r : Req) = handler.isDefinedAt(r) - } - } - super.serve(obpHandler) - } - - dlServe - { - //Handling get request for a token - case Req("my" :: "logins" :: "direct" :: Nil,_ , PostRequest) => { - for{ - (httpCode: Int, message: String, userId:Long) <- createTokenFuture(getAllParameters) - _ <- Future{grantEntitlementsToUseDynamicEndpointsInSpacesInDirectLogin(userId)} - } yield { - if (httpCode == 200) { - (JSONFactory.createTokenJSON(message), HttpCode.`201`(CallContext())) - } else { - unboxFullOrFail(Empty, None, message, httpCode) - } - } - } - } - - def grantEntitlementsToUseDynamicEndpointsInSpacesInDirectLogin(userId:Long) = { try { val resourceUser = UserX.findByResourceUserId(userId).openOrThrowException(s"$InvalidDirectLoginParameters can not find the resourceUser!") From ee05e701a766aff9f3ff9f6a664a57562dfcd27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 27 May 2026 18:51:04 +0200 Subject: [PATCH 04/13] feat(dynamic-entity): native http4s data-plane routes (off the Lift bridge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the dynamic-* migration. The operator-created dynamic-entity data plane (`/obp/dynamic-entity/{entityName | my/.. | public/.. | community/..}` and their `banks/BANK_ID/..` variants) is now served by a native `code.api.dynamic.entity.Http4sDynamicEntity` route instead of falling through to `Http4sLiftWebBridge` → `OBPAPIDynamicEntity`. What it does: - Reuses the framework-agnostic `EntityName` / `PublicEntityName` / `CommunityEntityName` extractors (they already take `List[String]` and consult `DynamicEntityHelper.definitionsMap`, rebuilt per request). - Mirrors the Lift `genericEndpoint` / `publicEndpoint` / `communityEndpoint` partial functions verbatim — same `authenticatedAccess` / `anonymousAccess`, `getBank`, role checks (`hasEntitlement` with the same canGet/Create/Update/ Delete roles + the `personalRequiresRole` shortcut), before/after auth interceptors, and `NewStyle.invokeDynamicConnector` calls. Only the Lift plumbing changes: matching on the http4s path, query params from the URI, and the `(JValue, HttpCode)` return replaced by `EndpointHelpers` (200 for GET/PUT/DELETE, 201 for POST). Public/Community are matched before the generic extractor to preserve Lift's registration precedence. Wiring: - `gate(ApiVersion.dynamic-entity, Http4sDynamicEntity.routes)` added to `Http4sApp.baseServices` just before the Lift bridge — gated by the same api_disabled_versions / api_enabled_versions machinery as the versioned routes. A non-match yields OptionT.none so unrelated paths fall through unchanged. - `OBPAPIDynamicEntity` stays registered on Lift as a dormant fallback (this route wins by ordering); its Lift registration is removed in the bridge-removal PR. Out of scope: dynamic-*endpoint* (runtime Scala codegen → Lift `OBPEndpoint`) remains on the bridge — a separate workstream. Admin CRUD (createDynamicEntity, …) was already native http4s in the versioned files. Verified: v6 DynamicEntityAccessFlagsTest + v4 DynamicEntityTest (38 tests) pass against the in-process http4s server, covering personal/public/community access flags, full CRUD, and the 404 fall-through for a system-level entity's absent `my` endpoint. v4 DynamicIntegrationTest / DynamicResourceDocTest / DynamicendPointsTest also green (dynamic-endpoint still served by the bridge). --- .../dynamic/entity/Http4sDynamicEntity.scala | 488 ++++++++++++++++++ .../code/api/util/http4s/Http4sApp.scala | 5 + 2 files changed, 493 insertions(+) create mode 100644 obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala new file mode 100644 index 0000000000..c4b0b41d20 --- /dev/null +++ b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala @@ -0,0 +1,488 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2019, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.dynamic.entity + +import cats.data.{Kleisli, OptionT} +import cats.effect.IO +import code.DynamicData.{DynamicData, DynamicDataProvider} +import code.api.Constant.PARAM_LOCALE +import code.api.dynamic.entity.helper.{CommunityEntityName, DynamicEntityHelper, DynamicEntityInfo, EntityName, PublicEntityName} +import code.api.util.APIUtil._ +import code.api.util.ErrorMessages._ +import code.api.util.http4s.Http4sRequestAttributes.EndpointHelpers +import code.api.util.http4s.{Http4sCallContextBuilder, Http4sRequestAttributes} +import code.api.util.{CallContext, CustomJsonFormats, NewStyle} +import code.util.Helper +import code.util.Helper.MdcLoggable +import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.model._ +import com.openbankproject.commons.model.enums.DynamicEntityOperation +import com.openbankproject.commons.model.enums.DynamicEntityOperation._ +import com.openbankproject.commons.util.{ApiVersion, JsonUtils} +import net.liftweb.common._ +import net.liftweb.http.{InMemoryResponse, JsonResponse} +import net.liftweb.json.JsonAST.JValue +import net.liftweb.json.JsonDSL._ +import net.liftweb.json._ +import net.liftweb.util.StringHelpers +import org.apache.commons.lang3.StringUtils +import org.http4s._ +import org.typelevel.ci.CIString + +import scala.concurrent.Future + +/** + * Native http4s routes for the dynamic-entity data plane. + * + * Serves the runtime, operator-created entity URLs under `/obp/dynamic-entity/...` + * (`{entityName}`, `my/{entityName}`, `public/{entityName}`, `community/{entityName}`, + * and their `banks/BANK_ID/...` variants). This replaces the Lift dispatch through + * `OBPAPIDynamicEntity` → `APIMethodsDynamicEntity` for the entity data plane. + * + * The URL→registry matching reuses the framework-agnostic `EntityName` / + * `PublicEntityName` / `CommunityEntityName` extractors (they take `List[String]` + * and consult `DynamicEntityHelper.definitionsMap`, rebuilt per request from the DB). + * The business logic mirrors the Lift `genericEndpoint` / `publicEndpoint` / + * `communityEndpoint` partial functions verbatim — same auth (`authenticatedAccess` / + * `anonymousAccess`), role checks, interceptors and `NewStyle.invokeDynamicConnector` + * calls — only the Lift `Req` / `JsonResponse` plumbing is replaced: matching is done + * on the http4s path, query params come from the URI, and the `(JValue, HttpCode)` + * return is replaced by `EndpointHelpers` (200 for GET/PUT/DELETE, 201 for POST). + * + * Admin CRUD (`createDynamicEntity`, `getDynamicEntities`, …) is unaffected — it is + * already native http4s in the versioned files (e.g. `Http4s600`). + * + * Note: `OBPAPIDynamicEntity` remains registered on the Lift bridge as a dormant + * fallback (this route wins by ordering in `Http4sApp.baseServices`); the Lift + * registration is removed in the bridge-removal PR. dynamic-*endpoint* (runtime + * Scala codegen) is a separate workstream and still served by the bridge. + */ +object Http4sDynamicEntity extends MdcLoggable { + + private type HttpF[A] = OptionT[IO, A] + + private implicit val formats: Formats = CustomJsonFormats.formats + + // "dynamic-entity" — passed as the CallContext apiVersion so getUserAndSessionContextFuture's + // S.request fallback (for implementedInVersion/verb/url) is never reached (it throws when not + // under a Lift dispatch). Same trick as Http4sResourceDocs / DirectLoginRoutes. + private val dynamicEntityVersion: String = ApiVersion.`dynamic-entity`.toString + + // ── Shared helpers (ported from ImplementationsDynamicEntity) ────────────── + + private def unboxResult[T: Manifest](box: Box[T], entityName: String): T = { + if (box.isInstanceOf[Failure]) { + val failure = box.asInstanceOf[Failure] + // change the internal db column name 'dynamicdataid' to entity's id name + val msg = failure.msg.replace(DynamicData.DynamicDataId.dbColumnName, StringUtils.uncapitalize(entityName) + "Id") + val changedMsgFailure = failure.copy(msg = s"$InternalServerError $msg") + fullBoxOrException[T](changedMsgFailure) + } + box.openOrThrowException("impossible error") + } + + // Filter a result list by `field=value` query params (skip the locale param), mirroring + // the Lift `filterDynamicObjects(resultList, req)` which read `req.params`. + private def filterDynamicObjects(resultList: JArray, params: Map[String, List[String]]): JArray = { + val effective = params.filter(_._1 != PARAM_LOCALE) + if (effective.isEmpty) resultList + else JArray(resultList.arr.filter { jValue => + effective.forall { case (path, values) => + values.exists(JsonUtils.isFieldEquals(jValue, path, _)) + } + }) + } + + // The before-authenticate interceptor short-circuits with a fully-formed Lift JsonResponse + // (rarely configured). Render it directly to http4s — ErrorResponseConverter only knows how + // to turn thrown APIFailures into responses, not an arbitrary JsonResponse. + private def liftJsonResponseToHttp4s(jr: JsonResponse): IO[Response[IO]] = jr.toResponse match { + case InMemoryResponse(data, headers, _, code) => + val status = org.http4s.Status.fromInt(code).getOrElse(org.http4s.Status.InternalServerError) + val h = Headers(headers.map { case (k, v) => Header.Raw(CIString(k), v) }) + IO.pure(Response[IO](status).withEntity(data).withHeaders(h)) + case other => + IO.pure(Response[IO](org.http4s.Status.fromInt(other.code).getOrElse(org.http4s.Status.InternalServerError))) + } + + /** + * Build the CallContext, run the before-authenticate interceptor, then execute the + * handler body through the standard EndpointHelpers (200 or 201). The augmented + * CallContext (operationId + resourceDocument set, mirroring the Lift `cc.copy(...)`) is + * stashed so auth/role checks and rate-limiting see it. + */ + private def respond( + req: Request[IO], + resourceDoc: Option[ResourceDoc], + operationId: String, + created: Boolean + )(body: CallContext => Future[JValue]): IO[Response[IO]] = + Http4sCallContextBuilder.fromRequest(req, apiVersion = dynamicEntityVersion).flatMap { baseCc => + val cc = baseCc.copy(operationId = Some(operationId), resourceDocument = resourceDoc) + beforeAuthenticateInterceptResult(Some(cc), operationId) match { + case Full(jr) => liftJsonResponseToHttp4s(jr) + case _ => + val reqWithCC = req.withAttribute(Http4sRequestAttributes.callContextKey, cc) + if (created) EndpointHelpers.executeFutureCreated[JValue](reqWithCC)(body(cc)) + else EndpointHelpers.executeAndRespond[JValue](reqWithCC)(body) + } + } + + private def queryParams(req: Request[IO]): Map[String, List[String]] = + req.uri.query.multiParams.map { case (k, vs) => k -> vs.toList } + + // ── Generic endpoint (authenticated, role-gated, full CRUD) ──────────────── + + private def genericGet(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = { + val listName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "_list") + val singleName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "") + val isGetAll = StringUtils.isBlank(id) + val operation: DynamicEntityOperation = if (isGetAll) GET_ALL else GET_ONE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val mySplitNameWithBankId = s"My$splitNameWithBankId" + val resourceDoc = + if (isPersonalEntity) DynamicEntityHelper.operationToResourceDoc.get(operation -> mySplitNameWithBankId) + else DynamicEntityHelper.operationToResourceDoc.get(operation -> splitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + val params = queryParams(req) + respond(req, resourceDoc, operationId, created = false) { cc => + for { + (Full(u), callContext) <- authenticatedAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + personalRequiresRole = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).exists(_.personalRequiresRole) + _ <- if (isPersonalEntity && !personalRequiresRole) Future.successful(true) + else NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canGetRole(entityName, bankId), callContext) + jsonResponse: Box[ErrorMessage] = afterAuthenticateInterceptResult(callContext, operationId).collect({ + case JsonResponseExtractor(message, code) => ErrorMessage(code, message) + }) + _ <- Helper.booleanToFuture(failMsg = jsonResponse.map(_.message).orNull, failCode = jsonResponse.map(_.code).openOr(400), cc = callContext) { + jsonResponse.isEmpty + } + (box, _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, None, Option(id).filter(StringUtils.isNotBlank), bankId, None, + Some(u.userId), isPersonalEntity, Some(cc)) + _ <- Helper.booleanToFuture( + s"$EntityNotFoundByEntityId Entity: '$entityName', entityId: '${id}'" + bankId.map(bid => s", bank_id: '$bid'").getOrElse(""), + 404, cc = callContext) { + box.isDefined + } + } yield { + if (isGetAll) { + val resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], entityName) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (listName -> filterDynamicObjects(resultList, params)) + bankIdJobject merge result + } else { + val result: JObject = (listName -> filterDynamicObjects(resultList, params)) + result + } + } else { + val singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (singleName -> singleObject) + bankIdJobject merge result + } else { + val result: JObject = (singleName -> singleObject) + result + } + } + } + } + } + + private def genericCreate(req: Request[IO], bankId: Option[String], entityName: String, isPersonalEntity: Boolean): IO[Response[IO]] = { + val singleName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "") + val operation: DynamicEntityOperation = CREATE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val mySplitNameWithBankId = s"My$splitNameWithBankId" + val resourceDoc = + if (isPersonalEntity) DynamicEntityHelper.operationToResourceDoc.get(operation -> mySplitNameWithBankId) + else DynamicEntityHelper.operationToResourceDoc.get(operation -> splitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + respond(req, resourceDoc, operationId, created = true) { cc => + val json = net.liftweb.json.parse(cc.httpBody.getOrElse("")) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + personalRequiresRole = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).exists(_.personalRequiresRole) + _ <- if (isPersonalEntity && !personalRequiresRole) Future.successful(true) + else NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canCreateRole(entityName, bankId), callContext) + jsonResponse: Box[ErrorMessage] = afterAuthenticateInterceptResult(callContext, operationId).collect({ + case JsonResponseExtractor(message, code) => ErrorMessage(code, message) + }) + _ <- Helper.booleanToFuture(failMsg = jsonResponse.map(_.message).orNull, failCode = jsonResponse.map(_.code).openOr(400), cc = callContext) { + jsonResponse.isEmpty + } + // Pass userId for all authenticated requests - personal records are filtered by userId + (box, _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, Some(json.asInstanceOf[JObject]), None, bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) + singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) + } yield { + val result: JObject = (singleName -> singleObject) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + bankIdJobject merge result + } else { + result + } + } + } + } + + private def genericUpdate(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = { + val singleName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "") + val operation: DynamicEntityOperation = UPDATE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val mySplitNameWithBankId = s"My$splitNameWithBankId" + val resourceDoc = + if (isPersonalEntity) DynamicEntityHelper.operationToResourceDoc.get(operation -> mySplitNameWithBankId) + else DynamicEntityHelper.operationToResourceDoc.get(operation -> splitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + respond(req, resourceDoc, operationId, created = false) { cc => + val json = net.liftweb.json.parse(cc.httpBody.getOrElse("")) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + personalRequiresRole = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).exists(_.personalRequiresRole) + _ <- if (isPersonalEntity && !personalRequiresRole) Future.successful(true) + else NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canUpdateRole(entityName, bankId), callContext) + jsonResponse: Box[ErrorMessage] = afterAuthenticateInterceptResult(callContext, operationId).collect({ + case JsonResponseExtractor(message, code) => ErrorMessage(code, message) + }) + _ <- Helper.booleanToFuture(failMsg = jsonResponse.map(_.message).orNull, failCode = jsonResponse.map(_.code).openOr(400), cc = callContext) { + jsonResponse.isEmpty + } + (box, _) <- NewStyle.function.invokeDynamicConnector(GET_ONE, entityName, None, Some(id), bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) + _ <- Helper.booleanToFuture( + s"$EntityNotFoundByEntityId Entity: '$entityName', entityId: '$id'" + bankId.map(bid => s", bank_id: '$bid'").getOrElse(""), + 404, cc = callContext) { + box.isDefined + } + (box: Box[JValue], _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, Some(json.asInstanceOf[JObject]), Some(id), bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) + singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) + } yield { + val result: JObject = (singleName -> singleObject) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + bankIdJobject merge result + } else { + result + } + } + } + } + + private def genericDelete(req: Request[IO], bankId: Option[String], entityName: String, id: String, isPersonalEntity: Boolean): IO[Response[IO]] = { + val operation: DynamicEntityOperation = DELETE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val mySplitNameWithBankId = s"My$splitNameWithBankId" + val resourceDoc = + if (isPersonalEntity) DynamicEntityHelper.operationToResourceDoc.get(operation -> mySplitNameWithBankId) + else DynamicEntityHelper.operationToResourceDoc.get(operation -> splitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + respond(req, resourceDoc, operationId, created = false) { cc => + for { + (Full(u), callContext) <- authenticatedAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + personalRequiresRole = DynamicEntityHelper.definitionsMap.get((bankId, entityName)).exists(_.personalRequiresRole) + _ <- if (isPersonalEntity && !personalRequiresRole) Future.successful(true) + else NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canDeleteRole(entityName, bankId), callContext) + jsonResponse: Box[ErrorMessage] = afterAuthenticateInterceptResult(callContext, operationId).collect({ + case JsonResponseExtractor(message, code) => ErrorMessage(code, message) + }) + _ <- Helper.booleanToFuture(failMsg = jsonResponse.map(_.message).orNull, failCode = jsonResponse.map(_.code).openOr(400), cc = callContext) { + jsonResponse.isEmpty + } + (box, _) <- NewStyle.function.invokeDynamicConnector(GET_ONE, entityName, None, Some(id), bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) + _ <- Helper.booleanToFuture( + s"$EntityNotFoundByEntityId Entity: '$entityName', entityId: '$id'" + bankId.map(bid => s", bank_id: '$bid'").getOrElse(""), + 404, cc = callContext) { + box.isDefined + } + (box, _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, None, Some(id), bankId, None, Some(u.userId), isPersonalEntity, Some(cc)) + deleteResult: JBool = unboxResult(box.asInstanceOf[Box[JBool]], entityName) + } yield { + deleteResult + } + } + } + + // ── Public endpoint (anonymous, read-only) ───────────────────────────────── + + private def publicGet(req: Request[IO], bankId: Option[String], entityName: String, id: String): IO[Response[IO]] = { + val listName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "_list") + val singleName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "") + val isGetAll = StringUtils.isBlank(id) + val operation: DynamicEntityOperation = if (isGetAll) GET_ALL else GET_ONE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val publicSplitNameWithBankId = s"Public$splitNameWithBankId" + val resourceDoc = DynamicEntityHelper.operationToResourceDoc.get(operation -> publicSplitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + val params = queryParams(req) + respond(req, resourceDoc, operationId, created = false) { cc => + for { + (_, callContext) <- anonymousAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + // No entitlement checks for public endpoints; userId=None, isPersonalEntity=false + (box, _) <- NewStyle.function.invokeDynamicConnector(operation, entityName, None, Option(id).filter(StringUtils.isNotBlank), bankId, None, + None, false, Some(cc)) + _ <- Helper.booleanToFuture( + s"$EntityNotFoundByEntityId Entity: '$entityName', entityId: '${id}'" + bankId.map(bid => s", bank_id: '$bid'").getOrElse(""), + 404, cc = callContext) { + box.isDefined + } + } yield { + if (isGetAll) { + val resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], entityName) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (listName -> filterDynamicObjects(resultList, params)) + bankIdJobject merge result + } else { + val result: JObject = (listName -> filterDynamicObjects(resultList, params)) + result + } + } else { + val singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (singleName -> singleObject) + bankIdJobject merge result + } else { + val result: JObject = (singleName -> singleObject) + result + } + } + } + } + } + + // ── Community endpoint (authenticated, role-gated, returns all users' records) ─ + + private def communityGet(req: Request[IO], bankId: Option[String], entityName: String, id: String): IO[Response[IO]] = { + val listName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "_list") + val singleName = StringHelpers.snakify(entityName).replaceFirst("[-_]*$", "") + val isGetAll = StringUtils.isBlank(id) + val operation: DynamicEntityOperation = if (isGetAll) GET_ALL else GET_ONE + val splitNameWithBankId = if (bankId.isDefined) s"""$entityName(${bankId.getOrElse("")})""" else entityName + val communitySplitNameWithBankId = s"Community$splitNameWithBankId" + val resourceDoc = DynamicEntityHelper.operationToResourceDoc.get(operation -> communitySplitNameWithBankId) + val operationId = resourceDoc.map(_.operationId).orNull + val params = queryParams(req) + respond(req, resourceDoc, operationId, created = false) { cc => + for { + (Full(u), callContext) <- authenticatedAccess(cc) + (_, callContext) <- + if (bankId.isDefined) NewStyle.function.getBank(bankId.map(BankId(_)).orNull, callContext) + else Future.successful(("", callContext)) + _ <- NewStyle.function.hasEntitlement(bankId.getOrElse(""), u.userId, DynamicEntityInfo.canGetRole(entityName, bankId), callContext) + jsonResponse: Box[ErrorMessage] = afterAuthenticateInterceptResult(callContext, operationId).collect({ + case JsonResponseExtractor(message, code) => ErrorMessage(code, message) + }) + _ <- Helper.booleanToFuture(failMsg = jsonResponse.map(_.message).orNull, failCode = jsonResponse.map(_.code).openOr(400), cc = callContext) { + jsonResponse.isEmpty + } + } yield { + if (isGetAll) { + val resultList: List[JObject] = DynamicDataProvider.connectorMethodProvider.vend.getAllDataJsonCommunity(bankId, entityName) + val resultArray = JArray(resultList) + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (listName -> filterDynamicObjects(resultArray, params)) + bankIdJobject merge result + } else { + val result: JObject = (listName -> filterDynamicObjects(resultArray, params)) + result + } + } else { + val singleResult = DynamicDataProvider.connectorMethodProvider.vend.getCommunity(bankId, entityName, id) + val singleObject: JValue = singleResult match { + case Full(data) => net.liftweb.json.parse(data.dataJson) + case _ => throw new RuntimeException(s"$EntityNotFoundByEntityId Entity: '$entityName', entityId: '$id'" + bankId.map(bid => s", bank_id: '$bid'").getOrElse("")) + } + if (bankId.isDefined) { + val bankIdJobject: JObject = ("bank_id" -> bankId.getOrElse("")) + val result: JObject = (singleName -> singleObject) + bankIdJobject merge result + } else { + val result: JObject = (singleName -> singleObject) + result + } + } + } + } + } + + // ── Routing ──────────────────────────────────────────────────────────────── + // Match `/obp/dynamic-entity/...`, strip the prefix to the segment list the extractors + // expect, then dispatch. Public/Community are tried before the generic extractor to + // preserve the Lift registration precedence (publicEndpoint, communityEndpoint, + // genericEndpoint). A non-match yields OptionT.none so the request falls through the + // chain (to the Lift bridge) unchanged. + + private def handle(req: Request[IO], rest: List[String]): IO[Option[Response[IO]]] = + req.method match { + case Method.GET => rest match { + case PublicEntityName(bankId, entityName, id) => publicGet(req, bankId, entityName, id).map(Some(_)) + case CommunityEntityName(bankId, entityName, id) => communityGet(req, bankId, entityName, id).map(Some(_)) + case EntityName(bankId, entityName, id, isP) => genericGet(req, bankId, entityName, id, isP).map(Some(_)) + case _ => IO.pure(None) + } + case Method.POST => rest match { + case EntityName(bankId, entityName, _, isP) => genericCreate(req, bankId, entityName, isP).map(Some(_)) + case _ => IO.pure(None) + } + case Method.PUT => rest match { + case EntityName(bankId, entityName, id, isP) => genericUpdate(req, bankId, entityName, id, isP).map(Some(_)) + case _ => IO.pure(None) + } + case Method.DELETE => rest match { + case EntityName(bankId, entityName, id, isP) => genericDelete(req, bankId, entityName, id, isP).map(Some(_)) + case _ => IO.pure(None) + } + case _ => IO.pure(None) + } + + val routes: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req: Request[IO] => + // Drop empty segments to mirror Lift's Req.path.partPath (e.g. trailing slash). + val segments = req.uri.path.segments.map(_.decoded()).filter(_.nonEmpty).toList + segments match { + case "obp" :: "dynamic-entity" :: rest => OptionT(handle(req, rest)) + case _ => OptionT.none[IO, Response[IO]] + } + } +} diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala index 14092720a8..afa0a4122f 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala @@ -74,6 +74,10 @@ object Http4sApp { private val v510Routes: HttpRoutes[IO] = gate(ApiVersion.v5_1_0, code.api.v5_1_0.Http4s510.wrappedRoutesV510Services) private val v600Routes: HttpRoutes[IO] = gate(ApiVersion.v6_0_0, code.api.v6_0_0.Http4s600.wrappedRoutesV600Services) private val v700Routes: HttpRoutes[IO] = gate(ApiVersion.v7_0_0, code.api.v7_0_0.Http4s700.wrappedRoutesV700Services) + // Dynamic-entity data plane (operator-created entities under /obp/dynamic-entity/*). + // Native http4s replacement for the Lift OBPAPIDynamicEntity dispatch; gated by the + // same api_disabled_versions / api_enabled_versions machinery as the versioned routes. + private val dynamicEntityRoutes: HttpRoutes[IO] = gate(ApiVersion.`dynamic-entity`, code.api.dynamic.entity.Http4sDynamicEntity.routes) /** * Build the base HTTP4S routes with priority-based routing. @@ -129,6 +133,7 @@ object Http4sApp { .orElse(v121Routes.run(req)) .orElse(code.api.DirectLoginRoutes.routes.run(req)) .orElse(code.api.AliveCheckRoutes.routes.run(req)) + .orElse(dynamicEntityRoutes.run(req)) .orElse(Http4sLiftWebBridge.routes.run(req)) } } From 4c262e4208a45cd9be9fd86882ff12148b0b0029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Wed, 27 May 2026 19:10:11 +0200 Subject: [PATCH 05/13] docs(migration): mark dynamic-entity native on http4s; dynamic-endpoint remains Phase 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to ee05e701a. Reflects that the dynamic-entity data plane is now served by `Http4sDynamicEntity` and off the Lift bridge: - Progress table: add "Dynamic-entity data plane" (done) and "Dynamic-endpoint data plane" (todo — Phase 3, runtime codegen still on the bridge) rows. - Bridge-traffic audit section: note that the `/obp/dynamic-entity/...` half of the `real_work` bucket no longer reaches the bridge (test runs show only legitimate 404 fall-throughs remain, e.g. a `my/` request against a system-level entity); the remaining `real_work` is `/obp/dynamic-endpoint/...`. --- LIFT_HTTP4S_MIGRATION.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/LIFT_HTTP4S_MIGRATION.md b/LIFT_HTTP4S_MIGRATION.md index 95a68946d5..29717b21c3 100644 --- a/LIFT_HTTP4S_MIGRATION.md +++ b/LIFT_HTTP4S_MIGRATION.md @@ -401,6 +401,8 @@ First real audit data (shard 1 CI run on 2026-05-25, 515 tests): - 20 `real_work` entries — all the `/obp/dynamic-entity/...` and `/obp/dynamic-endpoint/...` URLs. These are runtime-generated by Lift's dynamic dispatch when an admin creates a dynamic entity / endpoint at runtime; porting them is a workstream of its own (not endpoint-by-endpoint). - 2 `not_found` entries — `DELETE /obp/v4.0.0/banks/{id}/accounts`, asserted as 404 by `DeleteBankCascadeTest` to verify the cascade actually wiped the bank. +**Update (dynamic-entity ported):** the `/obp/dynamic-entity/...` half of that `real_work` bucket is now served natively by `code.api.dynamic.entity.Http4sDynamicEntity` (see Progress table) and no longer reaches the bridge — in the dynamic-entity test runs the only `/obp/dynamic-entity/...` bridge hits left are legitimate 404 fall-throughs (e.g. a `my/` request against a system-level entity that has no personal endpoint). The remaining `real_work` is `/obp/dynamic-endpoint/...` only (runtime Scala codegen → Lift `OBPEndpoint`; Phase 3, still on the bridge). + ### Auth stack — every handler is its own `RestHelper` | Handler | File | Routes | Status | @@ -615,6 +617,8 @@ Binds to `hostname` / `dev.port` from your props file (defaults: `127.0.0.1:8080 | Resource-docs: aggregation bug fix | done | | Resource-docs: `Http4sResourceDocs` service | todo | | Resource-docs: `openapi.yaml` route | todo | +| Dynamic-entity data plane | done — `code.api.dynamic.entity.Http4sDynamicEntity` serves `/obp/dynamic-entity/{entityName \| my/.. \| public/.. \| community/..}` (+ `banks/BANK_ID/..`) natively, wired into `Http4sApp.baseServices` via `gate(ApiVersion.\`dynamic-entity\`, ...)`. Reuses the `EntityName`/`PublicEntityName`/`CommunityEntityName` extractors + `NewStyle.invokeDynamicConnector`; mirrors the Lift `genericEndpoint`/`publicEndpoint`/`communityEndpoint` PFs. `OBPAPIDynamicEntity` stays registered on Lift as a dormant fallback (removed in the bridge-removal PR). | +| Dynamic-endpoint data plane | todo — Phase 3. Runtime Scala codegen (`DynamicEndpoints.scala`) compiles strings to Lift `OBPEndpoint`; still served by the bridge. 3a (keep Lift-typed codegen behind an adapter) recommended for the Lift-removal milestone; 3b (emit `HttpRoutes[IO]`) deferred. | ### Cleanup done From ca4237e737449f09e2f97e37435612e05b1c1981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Thu, 28 May 2026 05:37:33 +0200 Subject: [PATCH 06/13] test+docs(maker-checker): mark TTL race resolved; add stress regression guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md flagged a "~40% flaky" race in MakerCheckerTransactionRequestTest: in the multi-challenge path, the request-scoped proxy connection sometimes failed to propagate (via TransmittableThreadLocal) to a read Future's worker thread, so the read landed on a fresh pool connection and saw 0 uncommitted challenge rows. Two empirical sweeps + an in-JVM stress run all show the race no longer reproduces on current develop: - 8 instrumented runs (logged `currentProxy.get() == null` and row counts at every challenge write/read): all PASS, `proxyNull=false` on every read, across many worker threads, multi-user path included. - 12 clean runs (no instrumentation, separate JVMs): all PASS. - 1 stress run (20-iteration loop of multi-challenge creates in one warm JVM, hammering the exact write→read surface): all 20 iterations PASS, both challenges read back into the 201 response every time. 40 consecutive multi-challenge create→read cycles with zero failures. At the historical ~40% per-cycle rate, P ≈ 1.3×10⁻⁹ — the rate definitively no longer holds. Almost certainly fixed by the RequestScopeConnection hardening already on develop: 1. `currentProxy.childValue(parent) → null` (overridden in `RequestScopeConnection.scala`). Its own comment describes exactly the failing symptom: "workers stuck with a stale proxy then read 0 rows for the current request's freshly-written data, since the underlying real connection was closed by the original request's WBT." 2. The stale-proxy `isClosed()` guard added in `RequestAwareConnectionManager.newConnection`, which now falls back to a fresh vendor connection rather than returning a stale (already-closed) proxy. This commit locks the resolution in and updates the stale TODO: - **MakerCheckerTransactionRequestTest** — add a new scenario, "Stress: repeated multi-challenge creates must always read back both challenges (RequestScopeConnection regression guard)". Reuses the existing multi-user setup (REQUIRED_CHALLENGE_ANSWERS=2, two-user view access, maker-checker enforcement) and fires 20 INITIATED creates in one warm JVM, asserting both per-user challenges round-trip into each 201 response. No money moves (INITIATED only), so it's safe to loop. Iteration is recorded via `withClue` so a regression points at the offending iteration. - **CLAUDE.md** — strike through the "Flaky MakerCheckerTransactionRequestTest" bullet under "Other TODOs", explain it's resolved by the RequestScopeConnection hardening, link the regression guard, and keep the historical diagnosis + original fix directions in case a regression resurfaces. No core connector / payment-path code is changed — the proper fix already landed in `RequestScopeConnection`; this commit adds the safety net. --- CLAUDE.md | 2 +- .../MakerCheckerTransactionRequestTest.scala | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index a8ece6c322..08d4c07501 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -363,4 +363,4 @@ Architectural note from the v6 migration: around the 140-endpoint mark `Implemen - **CI speed-up** (not done): two-tier fast gate + full suite; surefire parallel forks. - **Disabled tests to fix**: `Http4s500RoutesTest` (@Ignore, in-process issue), `RootAndBanksTest` (@Ignore), `V500ContractParityTest` (@Ignore), `CardTest` (fully commented out). `v5_0_0`: 13 skipped tests (setup cost paid, no value). - **`V7ResourceDocsAggregationTest`**: intentionally failing — encodes the fix for the resource-docs aggregation bug (v7 endpoint returns only ~10 own docs instead of 500+ aggregated). Fix the bug to make this suite pass. -- **Flaky `MakerCheckerTransactionRequestTest` — TTL/proxy connection race in v4 createTransactionRequest** (pre-existing, predates the auth-stack migration). Scenario *"Multiple challenges with maker-checker: different users answer their own challenges"* (`MakerCheckerTransactionRequestTest.scala:246`) fails ~40% of local runs and was observed once in CI shard1. Diagnosed root cause: inside one HTTP request, `LocalMappedConnector.createTransactionRequestv210` writes N rows to `MappedExpectedChallengeAnswer` via the request-scoped proxy connection (auto-commit=false, request-end commit) and then reads them back via `getChallengesByTransactionRequestId`. When `RequestScopeConnection.currentProxy` (a `TransmittableThreadLocal`) fails to propagate to the read `Future`'s worker thread, `RequestAwareConnectionManager.newConnection` returns `null` → falls back to a fresh pool connection (autocommit=true) that cannot see the proxy connection's uncommitted writes → read returns 0 rows. Diagnostic confirmed: in failing runs, `createChallengesC2` is called with the correct 2 userIds, but `MappedExpectedChallengeAnswer.findAll()` (no WHERE clause) returns 0 rows — i.e. the entire table is empty from the read connection's view. Only the multi-user path (`REQUIRED_CHALLENGE_ANSWERS > 1`) hits this because it adds an extra synchronous `Views.views.vend.permissions(...)` inside `getAccountAttributesByAccount.map` that shifts the Future-scheduling timing. The other 3 scenarios in the file always pass because they take the default `REQUIRED_CHALLENGE_ANSWERS=1` shortcut. **Fix direction:** every DB-touching `Future { ... }` inside the connector chain needs to go through `RequestScopeConnection.fromFuture` (which atomically sets+submits+clears the TTL inside `IO.defer`) instead of being raw Scala `Future { ... }` chained via `flatMap`. Alternatively: stop relying on TTL and pass the proxy connection explicitly down the connector call-chain (bigger change, but eliminates the race class entirely). +- **~~Flaky `MakerCheckerTransactionRequestTest` — TTL/proxy connection race in v4 createTransactionRequest~~** — **RESOLVED.** The race (read worker getting a null/stale proxy → fresh pool connection → 0 uncommitted rows visible) is no longer reproducible on current `develop`: instrumented + clean reproduction sweeps (8 + 12 light runs across separate JVMs, plus a 20-iteration in-JVM stress) showed 40 consecutive multi-challenge create→read cycles pass with `currentProxy.get() != null` on every read; at the historical ~40% rate that's P ≈ 1.3×10⁻⁹. Almost certainly fixed by the `RequestScopeConnection` hardening already on `develop` — the `childValue → null` override on `currentProxy` (whose comment describes exactly this *"workers stuck with a stale proxy then read 0 rows for the current request's freshly-written data"* symptom) and the stale-proxy `isClosed()` guard in `RequestAwareConnectionManager.newConnection`. Locked in by `MakerCheckerTransactionRequestTest`'s **"Stress: repeated multi-challenge creates must always read back both challenges (RequestScopeConnection regression guard)"** scenario, which hammers the exact write→read surface in a warm JVM. Historical diagnosis (kept for context): inside one HTTP request, `LocalMappedConnector.createTransactionRequestv210` writes N rows to `MappedExpectedChallengeAnswer` via the request-scoped proxy and reads them back via `getChallengesByTransactionRequestId`; only the multi-user path (`REQUIRED_CHALLENGE_ANSWERS > 1`) exposed it because the extra synchronous `Views.views.vend.permissions(...)` inside `getAccountAttributesByAccount.map` shifted Future scheduling. If a regression resurfaces (the stress test starts flaking), the original fix directions still apply: route DB `Future`s through `RequestScopeConnection.fromFuture`, or thread the proxy connection explicitly down the connector call-chain. diff --git a/obp-api/src/test/scala/code/api/v4_0_0/MakerCheckerTransactionRequestTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/MakerCheckerTransactionRequestTest.scala index 3eba9bf842..ec06e5c455 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/MakerCheckerTransactionRequestTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/MakerCheckerTransactionRequestTest.scala @@ -267,6 +267,62 @@ class MakerCheckerTransactionRequestTest extends V400ServerSetup with DefaultUse addMakerCheckerPermissionToOwnerView() } } + + // Regression guard for the request-scoped-connection TTL race (formerly ~40% flaky; + // resolved by the RequestScopeConnection hardening — childValue→null override + + // stale-proxy isClosed() guard). Each multi-challenge create writes 2 + // MappedExpectedChallengeAnswer rows on the request-scoped proxy connection (autocommit + // off) and then reads them back into the 201 response while building `challenges`. If the + // proxy fails to propagate to the read Future's worker, the read lands on a fresh pool + // connection and sees 0 uncommitted rows → a challenge goes missing. Firing the create + // many times in one warm JVM maximises ForkJoinPool scheduling pressure on that + // write→read surface, so a regression in the connection-propagation logic shows up here. + scenario("Stress: repeated multi-challenge creates must always read back both challenges (RequestScopeConnection regression guard)", ApiEndpoint1) { + val iterations = 20 + val transactionRequestType = COUNTERPARTY.toString + val testBank = createBank("__mc-stress-bank") + val bankId = testBank.bankId + val accountId1 = AccountId("__mc_stress_acc1__") + val accountId2 = AccountId("__mc_stress_acc2__") + val fromCurrency = "AED" + val toCurrency = "INR" + + createAccountRelevantResource(Some(resourceUser1), bankId, accountId1, fromCurrency) + createAccountRelevantResource(Some(resourceUser1), bankId, accountId2, toCurrency) + updateAccountCurrency(bankId, accountId2, toCurrency) + + val fromAccount = BankAccountX(bankId, accountId1).getOrElse(fail("couldn't get from account")) + val counterparty = createCounterparty(bankId.value, accountId1.value, accountId2.value, true, java.util.UUID.randomUUID.toString) + + // REQUIRED_CHALLENGE_ANSWERS = 2 forces the multi-user (quorum > 1) path that exercises the race. + createAccountAttributeViaEndpoint(bankId.value, accountId1.value, "REQUIRED_CHALLENGE_ANSWERS", "2", "INTEGER", Some("LKJL98769G")) + grantUserAccessToViewViaEndpoint(bankId.value, accountId1.value, resourceUser2.userId, user1, + PostViewJsonV400(view_id = SYSTEM_OWNER_VIEW_ID, is_system = true)) + removeMakerCheckerPermissionFromOwnerView() + + try { + val bodyValue = AmountOfMoneyJsonV121(fromCurrency, "30000.00") + val transactionRequestBodyCounterparty = TransactionRequestBodyCounterpartyJSON( + CounterpartyIdJson(counterparty.counterpartyId), bodyValue, "Multi-challenge MC stress", "SHARED") + val createTransReqRequest = (v4_0_0_Request / "banks" / bankId.value / "accounts" / fromAccount.accountId.value / + SYSTEM_OWNER_VIEW_ID / "transaction-request-types" / transactionRequestType / "transaction-requests").POST <@(user1) + + // INITIATED only — no money moves, so we can repeat freely. + (1 to iterations).foreach { iter => + withClue(s"iteration $iter of $iterations: ") { + val createResponse = makePostRequest(createTransReqRequest, write(transactionRequestBodyCounterparty)) + createResponse.code should equal(201) + val json = createResponse.body.extract[TransactionRequestWithChargeJSON400] + json.status should equal(TransactionRequestStatus.INITIATED.toString) + // The race manifests as a missing challenge (read saw 0 uncommitted rows). + json.challenges.find(_.user_id == resourceUser1.userId) should not be (None) + json.challenges.find(_.user_id == resourceUser2.userId) should not be (None) + } + } + } finally { + addMakerCheckerPermissionToOwnerView() + } + } } } From 97c74e09e8f870747a14ed99438ac876a61e240f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Thu, 28 May 2026 08:10:46 +0200 Subject: [PATCH 07/13] =?UTF-8?q?feat(dynamic-endpoint):=20Phase=203a=20?= =?UTF-8?q?=E2=80=94=20scoped=20http4s=20adapter=20off=20the=20generic=20b?= =?UTF-8?q?ridge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dynamic-endpoint data plane (operator-created endpoints under `/obp/dynamic-endpoint/...`) is now served by a native http4s service — `code.api.dynamic.endpoint.Http4sDynamicEndpoint` — that dispatches `OBPAPIDynamicEndpoint` directly inside a scoped `S.init`, rather than letting the request fall through to the generic `Http4sLiftWebBridge`. The runtime Scala codegen (`DynamicEndpoints.compileScalaCode[OBPEndpoint]`) is unchanged: compiled user-supplied bodies still read `request.body`, `request.json`, `request.path.partPath` from a Lift `Req`. Phase 3a keeps that shape and stands a thin shim in front of it; Phase 3b (rewriting the codegen to emit `HttpRoutes[IO]` and drop the synthetic-`Req` coupling) is deferred. How it works: 1. `Http4sDynamicEndpoint.routes` matches `/obp/dynamic-endpoint/*` and reads the request body once. 2. Builds a synthetic Lift `Req` (`Http4sLiftWebBridge.buildLiftReq` — same `Http4sLiftRequest` shim the bridge already uses; full `HTTPRequest` surface satisfied). 3. Inside `S.init(Full(liftReq), session)` for a Lift stateless session, dispatches via `OBPAPIDynamicEndpoint` itself — `RestHelper` IS a `PartialFunction[Req, () => Box[LiftResponse]]`, the already-fully-wrapped shape (apiPrefix + oauthServe + wrappedWithAuthCheck), so URL matching, auth, role and `ResourceDoc`/`operationId` plumbing all run through the same code paths production has always used. No need to invoke the raw `OBPEndpoint` PFs or reimplement `apiPrefix`'s prefix-strip — which is essential, because `wrappedWithAuthCheck`'s `isUrlMatchesResourceDocUrl` (APIUtil.scala:1897) expects the prefix already stripped. 4. The resulting `Box[LiftResponse]` is mapped to a `LiftResponse` with the same `Full` / `Failure` / `ParamFailure` / `Empty` handling as `Http4sLiftWebBridge.runLiftDispatch`. `JsonResponseException` and `net.liftweb.http.rest.ContinuationException` (thrown by Future-based for-comprehension bodies via the OBPReturnType implicit) are caught at both depths and routed through the bridge's `resolveContinuation` reflective shim. 5. `Http4sLiftWebBridge.liftResponseToHttp4s` converts the final `LiftResponse` to `Response[IO]` — handles `InMemoryResponse`, `StreamingResponse`, `OutputStreamResponse`, `BasicResponse`. 6. Unmatched `/obp/dynamic-endpoint/...` paths return 404 with the same `InvalidUri` framing the bridge produces — **authoritative** for the prefix; never falls through to the generic bridge. To enable the reuse, three private helpers in `Http4sLiftWebBridge` had their `private` visibility relaxed to package-visible: `buildLiftReq`, `liftResponseToHttp4s`, `resolveContinuation`. Their bodies are unchanged. `OBPAPIDynamicEndpoint` stays registered on `LiftRules.statelessDispatch` (`APIUtil.scala:2878`) as a dormant fallback — the new http4s service wins by ordering in `Http4sApp.baseServices`. Removed in the bridge-removal PR. Audit impact: after this PR, `/obp/dynamic-endpoint/...` requests stop reaching `Http4sLiftWebBridge`. The bridge's `real_work` audit bucket reduces to the open-banking standards (BG v1.3, UK OB, sandbox); all OBP-native data-plane traffic is now off the generic bridge. Wiring: `gate(ApiVersion.dynamic-endpoint, Http4sDynamicEndpoint.routes)` added to `Http4sApp.baseServices` immediately after the dynamic-entity route and before `Http4sLiftWebBridge.routes`. Verified: 13 suites / 52 tests green — v4 DynamicendPointsTest, DynamicIntegrationTest, DynamicResourceDocTest, DynamicEndpointHelperTest, DynamicMessageDocTest, code.util.DynamicUtilTest, plus v6 DynamicEntityAccessFlagsTest as a regression sanity check (dynamic-entity is unaffected — this commit only adds a separate route for /dynamic-endpoint/). Out of scope (deferred): - Phase 3b — rewrite codegen to emit `HttpRoutes[IO]` directly. - Unregistering `OBPAPIDynamicEndpoint` from `LiftRules` — bridge-removal PR. --- .../endpoint/Http4sDynamicEndpoint.scala | 150 ++++++++++++++++++ .../code/api/util/http4s/Http4sApp.scala | 5 + .../api/util/http4s/Http4sLiftWebBridge.scala | 10 +- 3 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala new file mode 100644 index 0000000000..de5f55fcf6 --- /dev/null +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala @@ -0,0 +1,150 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2019, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.dynamic.endpoint + +import cats.data.{Kleisli, OptionT} +import cats.effect.IO +import code.api.util.APIUtil +import code.api.util.ErrorMessages +import code.api.util.http4s.Http4sLiftWebBridge +import code.api.{APIFailure, JsonResponseException} +import code.util.Helper.MdcLoggable +import net.liftweb.common._ +import net.liftweb.http.{LiftResponse, LiftRules, S} +import org.http4s._ + +/** + * Phase-3a Lift→http4s adapter for the dynamic-endpoint data plane. + * + * The runtime dispatch for `/obp/dynamic-endpoint/...` is built around a runtime Scala + * compiler (`DynamicEndpoints.compileScalaCode[OBPEndpoint]`) that emits Lift-typed + * `OBPEndpoint`s; the compiled user-supplied bodies read directly from `request.body`, + * `request.json`, `request.path.partPath`. Phase 3a keeps that codegen unchanged and + * stands a thin http4s shim in front: build a synthetic Lift `Req` from the http4s + * request, run the standard Lift dispatch ceremony (`S.init` over a stateless session), + * call the already-fully-wrapped `OBPAPIDynamicEndpoint` (apiPrefix + oauthServe + + * wrappedWithAuthCheck — URL matching, auth, role and ResourceDoc bookkeeping all live + * in those wrappings), and convert the resulting `Box[LiftResponse]` back to http4s. + * + * Functionally this is a scoped version of `Http4sLiftWebBridge`. Same Lift-Req + * construction, same response conversion, same async-continuation handling — only the + * dispatcher is narrowed from `LiftRules.statelessDispatch.toList ++ LiftRules.dispatch.toList` + * to one specific `RestHelper` object. The win: + * + * - `/obp/dynamic-endpoint/...` is **authoritative** here: requests never fall through + * to the generic Lift bridge, so the bridge-traffic audit's `real_work` bucket loses + * this prefix entirely (only the open-banking standards remain on the bridge). + * - `OBPAPIDynamicEndpoint` stays registered on `LiftRules` (in `APIUtil.scala:2878`) + * as a dormant fallback — this http4s service wins by ordering. The Lift registration + * is removed in the bridge-removal PR. + * + * Phase 3b (rewriting the codegen to emit `HttpRoutes[IO]` and drop the synthetic-`Req` + * coupling) is a separate, larger workstream; intentionally not done here. + * + * The three bridge helpers reused — `Http4sLiftWebBridge.buildLiftReq`, + * `Http4sLiftWebBridge.liftResponseToHttp4s`, `Http4sLiftWebBridge.resolveContinuation` + * — had their `private` visibility relaxed for this purpose; their bodies are unchanged. + */ +object Http4sDynamicEndpoint extends MdcLoggable { + + private type HttpF[A] = OptionT[IO, A] + + /** + * Run the dispatch under a Lift `S.init` over a stateless session, then map the + * result `Box[LiftResponse]` into a `LiftResponse` (mirroring `runLiftDispatch` in + * `Http4sLiftWebBridge`). `OBPAPIDynamicEndpoint` extends `RestHelper`, which is a + * `PartialFunction[Req, () => Box[LiftResponse]]` — the already-fully-wrapped shape + * Lift's REST machinery itself uses, so URL matching, auth, role checks and the + * `ResourceDoc`/`operationId` plumbing are all already wired up by the same code + * paths that production has always used. + * + * `S.init` is defensive: the compiled `OBPEndpoint` bodies are user-supplied via the + * create-dynamic-resource-doc admin endpoint, so we can't audit them all for `S.*` + * usage. The bridge does the same; we just scope it. + * + * `ContinuationException` can escape both the handler thunk and (rarely) the dispatch + * setup itself — Future-based for-comprehensions inside an `OBPEndpoint` body convert + * to `Box[JsonResponse]` via a continuation-throwing implicit. Mirror the bridge's + * defensive layering and catch at both depths. + */ + private def dispatchUnderLift(liftReq: net.liftweb.http.Req): LiftResponse = { + val session = LiftRules.statelessSession.vend.apply(liftReq) + S.init(Full(liftReq), session) { + try { + if (OBPAPIDynamicEndpoint.isDefinedAt(liftReq)) { + val thunk: () => Box[LiftResponse] = OBPAPIDynamicEndpoint(liftReq) + try { + thunk() match { + case Full(resp) => resp + case ParamFailure(_, _, _, apiFailure: APIFailure) => + APIUtil.errorJsonResponse(apiFailure.msg, apiFailure.responseCode) + case Failure(msg, _, _) => + APIUtil.errorJsonResponse(msg) + case Empty => + APIUtil.errorJsonResponse( + s"${ErrorMessages.InvalidUri}Current Url is (${liftReq.request.uri})", 404) + } + } catch { + case JsonResponseException(jsonResponse) => jsonResponse + case e if e.getClass.getName == "net.liftweb.http.rest.ContinuationException" => + Http4sLiftWebBridge.resolveContinuation(e) + } + } else { + // /obp/dynamic-endpoint/... path but no swagger-defined or compiled endpoint + // matches → 404 with the same `InvalidUri` framing the bridge produces, so + // callers see identical wire behaviour. + APIUtil.errorJsonResponse( + s"${ErrorMessages.InvalidUri}Current Url is (${liftReq.request.uri})", 404) + } + } catch { + case JsonResponseException(jsonResponse) => jsonResponse + case e if e.getClass.getName == "net.liftweb.http.rest.ContinuationException" => + Http4sLiftWebBridge.resolveContinuation(e) + } + } + } + + private def handle(req: Request[IO]): IO[Response[IO]] = + for { + bodyBytes <- req.body.compile.to(Array) + liftReq = Http4sLiftWebBridge.buildLiftReq(req, bodyBytes) + liftResp <- IO { dispatchUnderLift(liftReq) } + http4sResp <- Http4sLiftWebBridge.liftResponseToHttp4s(liftResp) + } yield http4sResp + + val routes: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req: Request[IO] => + // Drop empty segments to mirror Lift's `Req.path.partPath` (e.g. trailing slash). + val segments = req.uri.path.segments.map(_.decoded()).filter(_.nonEmpty).toList + segments match { + // Authoritative for the whole prefix — unmatched paths get a 404 *here* and never + // fall through to `Http4sLiftWebBridge`. That's the whole point: the bridge's + // audit `real_work` bucket loses `/obp/dynamic-endpoint/...` after this PR. + case "obp" :: "dynamic-endpoint" :: _ => OptionT.liftF(handle(req)) + case _ => OptionT.none[IO, Response[IO]] + } + } +} diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala index afa0a4122f..f855614076 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala @@ -78,6 +78,10 @@ object Http4sApp { // Native http4s replacement for the Lift OBPAPIDynamicEntity dispatch; gated by the // same api_disabled_versions / api_enabled_versions machinery as the versioned routes. private val dynamicEntityRoutes: HttpRoutes[IO] = gate(ApiVersion.`dynamic-entity`, code.api.dynamic.entity.Http4sDynamicEntity.routes) + // Dynamic-endpoint data plane (operator-created endpoints under /obp/dynamic-endpoint/*). + // Phase-3a adapter: scoped Lift-Req shim that dispatches OBPAPIDynamicEndpoint directly, + // keeping the runtime Scala codegen (which still emits Lift OBPEndpoint) unchanged. + private val dynamicEndpointRoutes: HttpRoutes[IO] = gate(ApiVersion.`dynamic-endpoint`, code.api.dynamic.endpoint.Http4sDynamicEndpoint.routes) /** * Build the base HTTP4S routes with priority-based routing. @@ -134,6 +138,7 @@ object Http4sApp { .orElse(code.api.DirectLoginRoutes.routes.run(req)) .orElse(code.api.AliveCheckRoutes.routes.run(req)) .orElse(dynamicEntityRoutes.run(req)) + .orElse(dynamicEndpointRoutes.run(req)) .orElse(Http4sLiftWebBridge.routes.run(req)) } } diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sLiftWebBridge.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sLiftWebBridge.scala index 2abdcbaf08..a7ada94b16 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sLiftWebBridge.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sLiftWebBridge.scala @@ -324,7 +324,11 @@ object Http4sLiftWebBridge extends MdcLoggable { } } - private def resolveContinuation(exception: Throwable): LiftResponse = { + // Package-visible so scoped per-prefix dispatchers (e.g. Http4sDynamicEndpoint) can reuse + // exactly the bridge's Lift-Req construction, response conversion, and async-continuation + // handling without duplicating ~100 lines or going through LiftRules.statelessDispatch. + // Same logic; only visibility changed. + def resolveContinuation(exception: Throwable): LiftResponse = { logger.debug(s"Resolving ContinuationException for async Lift handler") val func = ReflectUtils @@ -339,7 +343,7 @@ object Http4sLiftWebBridge extends MdcLoggable { } } - private def buildLiftReq(req: Request[IO], body: Array[Byte]): Req = { + def buildLiftReq(req: Request[IO], body: Array[Byte]): Req = { val headers = http4sHeadersToParams(req.headers.headers) val params = http4sParamsToParams(req.uri.query.multiParams.toList) val httpRequest = new Http4sLiftRequest( @@ -380,7 +384,7 @@ object Http4sLiftWebBridge extends MdcLoggable { } } - private def liftResponseToHttp4s(response: LiftResponse): IO[Response[IO]] = { + def liftResponseToHttp4s(response: LiftResponse): IO[Response[IO]] = { response.toResponse match { case InMemoryResponse(data, headers, _, code) => IO.pure(buildHttp4sResponse(code, data, headers)) From 1e1047da16592d89b6f47501b9592f11adafb631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Thu, 28 May 2026 09:34:08 +0200 Subject: [PATCH 08/13] =?UTF-8?q?fix(error-response):=20handle=20JsonRespo?= =?UTF-8?q?nseException=20=E2=80=94=20fixes=206=20CI=20regressions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shard 1 CI on origin/develop (commits ee05e701a..ca4237e73) reports 6 failing v4 tests, all 500 where 400/403/444 was expected: ForceErrorValidationTest (5 fails) — "dynamic entity endpoints Force-Error" JsonSchemaValidationTest (1 fail) — dynamic entity JSON-schema validation All on the dynamic-entity data plane. Regression introduced by the Phase 2 commit (ee05e701a) `feat(dynamic-entity): native http4s data-plane routes`: - Pre-Phase-2 path: dynamic-entity → Lift bridge. `Http4sLiftWebBridge.runLiftDispatch` has explicit `case JsonResponseException(jsonResponse) => jsonResponse` → the embedded synthesized response (Force-Error 400/403/444, JSON-schema 400, etc.) is sent verbatim. - Post-Phase-2 path: dynamic-entity → `Http4sDynamicEntity` → `EndpointHelpers` → on Future failure, `ErrorResponseConverter.toHttp4sResponse(err, cc)`. That converter only knew `APIFailureNewStyle` and the OBP-prefixed-message form; `JsonResponseException` fell to `unknownErrorToResponse` → 500. `JsonResponseException` (defined `OBPRestHelper.scala:228`) is thrown from `APIUtil.anonymousAccess` (line 3517) and from the after-authenticate interceptor chain in `NewStyle.scala:959` — it is *the* mechanism used by the interceptor framework (Force-Error / JSON schema validation / auth-type validation) to short-circuit with a fully-formed `JsonResponse` from inside a `Future`. Any http4s endpoint whose body crosses one of those interceptors needs this case to translate the carried response to http4s. This is a defensive improvement at the right level: it fixes all http4s endpoints, not just dynamic-entity. Reuses `Http4sLiftWebBridge.liftResponseToHttp4s` (already package-visible from the Phase 3a commit) so the byte-shape of the response matches the bridge's exactly. No production logic moves — only the exception case is added. Verified locally: 19 suites / 111 tests green — ForceErrorValidationTest, JsonSchemaValidationTest, AuthenticationTypeValidationTest (same interceptor mechanism), v4 DynamicendPointsTest, DynamicIntegrationTest, DynamicResourceDocTest, DynamicEndpointHelperTest, DynamicMessageDocTest, code.util.DynamicUtilTest, v6 DynamicEntityAccessFlagsTest, v4 MakerCheckerTransactionRequestTest. Diagnostic that pinned the root cause (kept for the PR description, not in code): `io-compute-*` threads logging `unknownErrorToResponse says: 500 returned` followed by `code.api.JsonResponseException: null` for each of the 5 ForceError scenarios. --- .../api/util/http4s/ErrorResponseConverter.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala index c228d4ce2c..f121e55410 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ErrorResponseConverter.scala @@ -73,10 +73,23 @@ object ErrorResponseConverter { /** * Convert any error to http4s Response[IO]. + * + * `JsonResponseException` carries a fully-formed Lift `JsonResponse` and is the + * mechanism `APIUtil.anonymousAccess` (line 3517) and the after-authenticate + * interceptor chain (Force-Error, JSON-schema validation, etc. — `APIUtil.scala:5064` + * `afterAuthenticateInterceptors`) use to short-circuit a request with a synthesized + * response from inside a `Future`. The Lift bridge handles it with + * `case JsonResponseException(jsonResponse) => jsonResponse` in + * `Http4sLiftWebBridge.runLiftDispatch`; without an equivalent case here, + * EndpointHelpers-based handlers fall through to `unknownErrorToResponse` and + * return 500 where production has always returned the synthesized status (e.g. + * 400/403/444 for Force-Error scenarios). Reuse `liftResponseToHttp4s` so the + * conversion matches the bridge byte-for-byte. */ def toHttp4sResponse(error: Throwable, callContext: CallContext): IO[Response[IO]] = { error match { case e: APIFailureNewStyle => apiFailureToResponse(e, callContext) + case e: code.api.JsonResponseException => Http4sLiftWebBridge.liftResponseToHttp4s(e.jsonResponse) case _ => tryExtractApiFailureFromExceptionMessage(error) match { case Some(apiFailure) => apiFailureToResponse(apiFailure, callContext) From 25a823b14b79ddd335392ac996c342cc306a2277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 29 May 2026 09:08:03 +0200 Subject: [PATCH 09/13] docs(migration): mark UK Open Banking v2.0/v3.1 done (PR #2817) --- LIFT_HTTP4S_MIGRATION.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/LIFT_HTTP4S_MIGRATION.md b/LIFT_HTTP4S_MIGRATION.md index 29717b21c3..df07645ed2 100644 --- a/LIFT_HTTP4S_MIGRATION.md +++ b/LIFT_HTTP4S_MIGRATION.md @@ -448,7 +448,7 @@ Three forks for how the still-active workstream resolves: |---|---|---| | Berlin Group v1.3 | `code/api/berlin/group/v1_3/*` — 7 files (AIS / PIS / PIIS / signing baskets / common) | Lift | | **Berlin Group v2** | `code/api/berlin/group/v2/Http4sBGv2.scala` | ✅ already on http4s | -| UK Open Banking v2.0.0 + v3.1.0 | `code/api/UKOpenBanking/*` — ~20 files | Lift | +| **UK Open Banking v2.0.0 + v3.1.0** | `code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200*.scala`, `code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310*.scala` | ✅ **Done** — migrated to native http4s in PR #2817 (merged to `develop` 2026-05-29). All endpoints (v2.0: 5, v3.1: ~67 across 20 categories) on http4s, wired into `Http4sApp.baseServices` ahead of the Lift bridge. Lift `OBP_UKOpenBanking_{200,310}` reduced to stubs (`routes = Nil`), original Lift code kept as comments. `ResourceDocsAPIMethods.activeResourceDocs` skips the Lift-route filter for both versions. Verified by `UKOpenBankingV200Tests` + `UKOpenBankingV310{Ais,Pis}Tests` (142 scenarios, all passing). | | ~~Bahrain OBF v1.0.0~~ | `code/api/BahrainOBF/*` | ✅ **Retired** — commented out in PR #2814 (`d19af2b92`, 2026-05-22). ScannedApis reflection no longer finds it; URLs 404 with no bridge handling. | | ~~AU OpenBanking v1.0.0~~ | `code/api/AUOpenBanking/*` | ✅ **Retired** — same PR / commit. | | ~~STET v1.4~~ | `code/api/STET/v1_4/*` | ✅ **Retired** — same PR / commit. | @@ -512,7 +512,7 @@ A second decision is *not* required for bridge removal, but is required for the 5. **Auth stack: OAuth2 / GatewayLogin / DAuth** — done. All three turned out to be library-only token validators (no `serve` blocks, no `LiftRules` registration). Vestigial `extends RestHelper` mixins removed. 6. **OpenIdConnect** — the only remaining auth-stack work. Blocked on a portal-session decision (its success path calls `AuthUser.logUserIn` / `S.redirectTo`, which mutate Lift `SessionVar`s — see auth-stack table). OAuth 1.0a was removed entirely in commit `51820c75e`; no migration needed. 7. **Bridge-removal PR** — delete `Http4sLiftWebBridge` + the request-path entries from `Boot.scala` (lines 1–7 above). -8. **Open-banking standards** — decide whether to migrate or keep a thin Lift remnant. Weeks of work if migrating. +8. **Open-banking standards** — Berlin Group v2 and UK Open Banking v2.0/v3.1 now on http4s (the latter in PR #2817); five standards retired (Bahrain, AU, STET, MxOF, Polish). Remaining on Lift: **Berlin Group v1.3** and **Sandbox** — decide whether to migrate or keep a thin Lift remnant. 9. **`lift-mapper`** — separate long-term effort, out of scope here. --- @@ -617,6 +617,9 @@ Binds to `hostname` / `dev.port` from your props file (defaults: `127.0.0.1:8080 | Resource-docs: aggregation bug fix | done | | Resource-docs: `Http4sResourceDocs` service | todo | | Resource-docs: `openapi.yaml` route | todo | +| Std: UK Open Banking v2.0 + v3.1 | done — `Http4sUKOBv200*` / `Http4sUKOBv310*` (PR #2817, merged 2026-05-29). v2.0: 5 endpoints; v3.1: ~67 across 20 categories. Wired into `Http4sApp.baseServices` ahead of the Lift bridge; Lift `OBP_UKOpenBanking_{200,310}` are `routes = Nil` stubs. Verified by `UKOpenBankingV200Tests` + `UKOpenBankingV310{Ais,Pis}Tests` (142 scenarios pass). | +| Std: Berlin Group v2 | done — `code.api.berlin.group.v2.Http4sBGv2` (native http4s). | +| Std: Berlin Group v1.3, Sandbox | todo — still on Lift. | | Dynamic-entity data plane | done — `code.api.dynamic.entity.Http4sDynamicEntity` serves `/obp/dynamic-entity/{entityName \| my/.. \| public/.. \| community/..}` (+ `banks/BANK_ID/..`) natively, wired into `Http4sApp.baseServices` via `gate(ApiVersion.\`dynamic-entity\`, ...)`. Reuses the `EntityName`/`PublicEntityName`/`CommunityEntityName` extractors + `NewStyle.invokeDynamicConnector`; mirrors the Lift `genericEndpoint`/`publicEndpoint`/`communityEndpoint` PFs. `OBPAPIDynamicEntity` stays registered on Lift as a dormant fallback (removed in the bridge-removal PR). | | Dynamic-endpoint data plane | todo — Phase 3. Runtime Scala codegen (`DynamicEndpoints.scala`) compiles strings to Lift `OBPEndpoint`; still served by the bridge. 3a (keep Lift-typed codegen behind an adapter) recommended for the Lift-removal milestone; 3b (emit `HttpRoutes[IO]`) deferred. | From d6c8cb3e7bac765341e63f084c807d42bec159f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 29 May 2026 09:44:55 +0200 Subject: [PATCH 10/13] docs(claude): trim to how-to + gotchas; move status/TODO to MIGRATION.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - drop 45-name migrated-endpoints list and TODO/Phase-Progress section (status → MIGRATION.md) - condense CI Performance Profile to shard map + 1-line perf note - add two gotchas (JVM 64KB limit, isStatisticallyTooPermissive) - drop stale V7ResourceDocsAggregationTest 'intentionally failing' note (bug fixed) --- CLAUDE.md | 90 ++++--------------------------------------------------- 1 file changed, 5 insertions(+), 85 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 08d4c07501..279490b426 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,8 +17,6 @@ The goal is a full http4s migration — replace Lift Web across all version file **Key files**: `Http4s700.scala` (v7.0.0 endpoints), `Http4s200.scala` (v2.0.0 endpoints — 37 own + path-rewriting bridge to Http4s140), `Http4s140.scala` (v1.4.0 endpoints — 11 own + path-rewriting bridge to Http4s130), `Http4s130.scala` (v1.3.0 endpoints — 3 own + path-rewriting bridge to Http4s121), `Http4s121.scala` (v1.2.1 endpoints — all 323 API1_2_1Test scenarios), `Http4sSupport.scala` (EndpointHelpers + recordMetric), `ResourceDocMiddleware.scala` (auth, entity resolution, transaction wrapper), `IdempotencyMiddleware.scala` (Redis-backed idempotency, opt-in via `Idempotency-Key` header, nested inside ResourceDocMiddleware), `RequestScopeConnection.scala` (DB transaction propagation to Futures). -**Migrated endpoints** (45): root, getBanks, getCards, getCardsForBank, getResourceDocsObpV700, getBank, getCurrentUser, getCoreAccountById, getPrivateAccountByIdFull, getExplicitCounterpartyById, deleteEntitlement, addEntitlement, getAccountAccessTrace, getFeatures, getScannedApiVersions, getConnectors, getErrorMessages, getProviders, getUsers, getUserByUserId, getCustomersAtOneBank, getCustomerByCustomerId, getAccountsAtBank, createTradingOffer, getTradingOffer, getTradingOffers, cancelTradingOffer, createMarketOrder, getMarketOrder, cancelMarketOrder, createMarketMatch, getMarketTrade, requestSettlement, requestWithdrawal, getCacheConfig, getCacheInfo, getDatabasePoolInfo, getStoredProcedureConnectorHealth, getMigrations, getCacheNamespaces, createOrganisation, getOrganisations, getOrganisation, updateOrganisation, deleteOrganisation. - **Tests**: `Http4s700RoutesTest` (102 scenarios, port 8087). `makeHttpRequest` returns `(Int, JValue, Map[String, String])`. `makeHttpRequestWithBody(method, path, body, headers)` for POST/PUT. ## Migrating a Lift Endpoint to http4s @@ -263,65 +261,13 @@ How to find overrides for a version: grep `lazy val (\w+)` in the target `APIMet Symptoms in tests: a v4-specific assertion fails (e.g. an entitlement should-be-granted check returns false). The HTTP response is usually a successful 200/201, just from the wrong handler — so it can look like a flaky failure on the surface. -## CI Performance Profile - -Measured from a 3-shard run (2691 tests total, all passing). Numbers are stable across shards. - -### Time budget per shard (~9–11 min total) - -| Phase | Time | % of total | -|---|---|---| -| Main compile (Zinc) | ~130s | ~22% | -| Test compile (Zinc) | ~68s | ~11% | -| Test discovery (ScalaTest) | ~20s | ~3% | -| **Test execution** | **~340–420s** | **~60–64%** | +**JVM 64KB `` limit in per-version files**: around ~140 endpoints, an `Http4sXxx` object's `` exceeds the JVM 64KB-bytecode-per-method limit and won't compile. Adopt from the start (don't wait for the wall): (1) declare endpoints as `lazy val xxx: HttpRoutes[IO] = HttpRoutes.of[IO] { ... }` (not `val`) so lambda materialisation moves out of `` into per-field `lzycompute` methods; (2) group `resourceDocs += ResourceDoc(...)` calls into `private def initXxxResourceDocs(): Unit` blocks of ~10–15 endpoints, each called once from the object body. Each helper def gets its own 64KB budget. (Pattern shipped in `Http4s600.scala`.) -Compile times are consistent across all three shards — Zinc cache restores correctly. Test execution is the dominant cost. +**`isStatisticallyTooPermissive` is sample-pool-dependent**: a fresh local test DB with a single user trips the ABAC-permissiveness check and causes spurious rejections. Seed enough users in any test exercising ABAC rules — it's a test-data issue, not a regression. -### http4s v7 vs Lift — per-test speed +## CI (shard map + run tips) -| Category | Tests | Avg/test | -|---|---|---| -| http4s v7 — unit/pure (no server) | 172 | **0.008s** | -| http4s v7 — integration (real server) | 160 | 0.418s | -| Lift v4 | 515 | 0.448s | -| Lift v3 | 269 | 0.446s | -| Lift v5 | 337 | 0.432s | -| Lift v1 | 431 | 0.425s | -| Lift v2 | 124 | 0.414s | -| Lift v6 | 314 | 0.411s | - -At the integration level both frameworks are similarly server/DB-bound (~0.32–0.45 s/test). The real http4s gain is the **unit/pure tier** — tests that don't need a running server are 54× faster. As more logic moves into pure functions (request parsing, response building, auth checks) these unit tests replace integration tests and the savings compound. - -The 6 integration suites (pre-merge timings; Http4s700RoutesTest is currently 102 scenarios): -- `obp-api/src/test/scala/code/api/http4sbridge/Http4sLiftBridgePropertyTest.scala` — 51 tests, 31.9s -- `obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala` — 102 tests (was 75 pre-merge, 23.8s) -- `obp-api/src/test/scala/code/api/v7_0_0/V7ResourceDocsAggregationTest.scala` — intentionally failing until resource-docs aggregation bug is fixed -- `obp-api/src/test/scala/code/api/http4sbridge/Http4sServerIntegrationTest.scala` — 16 tests, 5.0s -- `obp-api/src/test/scala/code/api/v5_0_0/Http4s500SystemViewsTest.scala` — 13 tests, 4.4s -- `obp-api/src/test/scala/code/api/v7_0_0/Http4s700TransactionTest.scala` — 5 tests, 1.9s - -The 12 pure-unit suites (172 tests, 1.3s total): -- `obp-api/src/test/scala/code/api/util/http4s/Http4sCallContextBuilderTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/Http4sResponseConversionTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/Http4sResponseConversionPropertyTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/Http4sRequestConversionPropertyTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/ResourceDocMatcherTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/Http4sConfigUtilTest.scala` -- `obp-api/src/test/scala/code/api/util/http4s/RequestScopeConnectionTest.scala` -- `obp-api/src/test/scala/code/api/berlin/group/v2/Http4sBGv2AISTest.scala` -- `obp-api/src/test/scala/code/api/berlin/group/v2/Http4sBGv2PISTest.scala` -- `obp-api/src/test/scala/code/api/berlin/group/v2/Http4sBGv2ResourceDocTest.scala` -- `obp-api/src/test/scala/code/api/berlin/group/v2/Http4sBGv2PIISTest.scala` -- `obp-api/src/test/scala/code/api/v5_0_0/Http4s500RoutesTest.scala` - -### Known bottlenecks - -**`API1_2_1Test`** (now http4s-backed via `Http4s121`) — was 143s for 323 tests on the Lift path; expected to improve as Lift bridge overhead is eliminated. The suite is in shard 3 (`code.api.v1_2_1` prefix). - -**`Http4sLiftBridgePropertyTest`** — 31.9s for 51 tests. Property 7 ("Session and Context Adapter Correctness") accounts for 13.4s of that: three ScalaCheck properties exercise concurrent requests through the Lift/http4s bridge, hitting real lock contention between Lift's session manager and the http4s fiber scheduler. Property 7.4 alone is 8.54s. These are the most meaningful slow tests — they exercise a genuine concurrency boundary. - -**`ResourceDocsTest` / `SwaggerDocsTest`** — 34s + 24s = 58s, averaging 0.85s/test — the slowest per-test cost in the suite. Each test serializes the entire API surface (633+ endpoints) into JSON/Swagger. Cost scales linearly with endpoint count. Will worsen as the http4s migration adds endpoints unless ResourceDoc serialization is cached or the heavy tests are isolated. +Perf note: integration tests are DB/HTTP-bound (~0.4 s/test) on both frameworks; the http4s win is the **pure-unit tier** (no running server, ~0.008 s/test). `ResourceDocsTest`/`SwaggerDocsTest` are the slowest per-test cost — they serialize the whole API surface, so cost grows with endpoint count (needs ResourceDoc-serialization caching before the migration completes). ### Shard assignment @@ -337,30 +283,4 @@ Shards are defined by explicit package-prefix allowlists in `.github/workflows/b To explicitly move a package to a different shard, add it to that shard's `test_filter` block — it will be excluded from the catch-all automatically. -### Implication for the migration - -Per-endpoint integration test cost stays roughly constant as endpoints move Lift → http4s (both bound by DB + HTTP). Gains appear from: (1) pure unit tests replacing integration tests, (2) eventual removal of Lift endpoint tests when v6 is retired. ResourceDocs overhead is the one cost that compounds — needs caching before the migration is complete. - -## TODO / Phase Progress - -### Per-version completeness (from `comm -23 lift http4s` on each version's `lazy val ... : OBPEndpoint` declarations) - -| Version | Genuine Lift handlers still on the bridge | -|---|---| -| v1.2.1, v1.3.0, v1.4.0, v2.0.0, v2.1.0, v2.2.0, v3.0.0, v4.0.0, v5.0.0, v5.1.0, v6.0.0 | 0 — fully on http4s | -| v3.1.0 | `getMessageDocsSwagger`, `getObpConnectorLoopback`. `getMessageDocsSwagger`'s URL is in production already served by `Http4sResourceDocs.routes` (the Lift `lazy val` is shadowed dead code), but the Lift definition is intentionally kept — deleting it would reduce v3.1.0's frozen STABLE API surface (caught by `FrozenClassTest`) and require touching a v3.1.0 test. Retires together with the bridge-removal PR. `getObpConnectorLoopback` likewise deferred to the bridge-removal PR. | - -### v6.0.0 migration — done (243 / 243) -Phase 1 (35 overrides) and Phase 2 (208 originals) both complete. All v6 routes live in `Http4s600.scala`, wired into `Http4sApp.baseServices` ahead of the Lift bridge. - -Architectural note from the v6 migration: around the 140-endpoint mark `Implementations6_0_0`'s `` hit the JVM 64KB bytecode-per-method limit. The fix that ships in `Http4s600.scala` — and that future per-version files should adopt — is two-part: - -1. Declare endpoints as `lazy val xxx: HttpRoutes[IO] = HttpRoutes.of[IO] { ... }` instead of `val`. Lambda materialisation moves out of `` into per-field `lzycompute` methods (each with its own 64KB budget). -2. Group `resourceDocs += ResourceDoc(...)` calls into `private def initXxxResourceDocs(): Unit` blocks of ~10–15 endpoints, called once each from the object body. Each helper def gets its own 64KB. - -### Other TODOs -- **OBP-Trading**: trading endpoints (createTradingOffer, getTradingOffer, getTradingOffers, cancelTradingOffer, createMarketOrder, getMarketOrder, cancelMarketOrder, createMarketMatch, getMarketTrade, requestSettlement, requestWithdrawal) are now in `Http4s700.scala`. 5 payment-auth endpoints remain commented out (notifyDeposit, createPaymentAuth, capturePaymentAuth, releasePaymentAuth, getPaymentAuth) — see `ideas/CAPTURE_RELEASE_TRANSACTION_REQUEST_TYPES.md`. -- **CI speed-up** (not done): two-tier fast gate + full suite; surefire parallel forks. -- **Disabled tests to fix**: `Http4s500RoutesTest` (@Ignore, in-process issue), `RootAndBanksTest` (@Ignore), `V500ContractParityTest` (@Ignore), `CardTest` (fully commented out). `v5_0_0`: 13 skipped tests (setup cost paid, no value). -- **`V7ResourceDocsAggregationTest`**: intentionally failing — encodes the fix for the resource-docs aggregation bug (v7 endpoint returns only ~10 own docs instead of 500+ aggregated). Fix the bug to make this suite pass. -- **~~Flaky `MakerCheckerTransactionRequestTest` — TTL/proxy connection race in v4 createTransactionRequest~~** — **RESOLVED.** The race (read worker getting a null/stale proxy → fresh pool connection → 0 uncommitted rows visible) is no longer reproducible on current `develop`: instrumented + clean reproduction sweeps (8 + 12 light runs across separate JVMs, plus a 20-iteration in-JVM stress) showed 40 consecutive multi-challenge create→read cycles pass with `currentProxy.get() != null` on every read; at the historical ~40% rate that's P ≈ 1.3×10⁻⁹. Almost certainly fixed by the `RequestScopeConnection` hardening already on `develop` — the `childValue → null` override on `currentProxy` (whose comment describes exactly this *"workers stuck with a stale proxy then read 0 rows for the current request's freshly-written data"* symptom) and the stale-proxy `isClosed()` guard in `RequestAwareConnectionManager.newConnection`. Locked in by `MakerCheckerTransactionRequestTest`'s **"Stress: repeated multi-challenge creates must always read back both challenges (RequestScopeConnection regression guard)"** scenario, which hammers the exact write→read surface in a warm JVM. Historical diagnosis (kept for context): inside one HTTP request, `LocalMappedConnector.createTransactionRequestv210` writes N rows to `MappedExpectedChallengeAnswer` via the request-scoped proxy and reads them back via `getChallengesByTransactionRequestId`; only the multi-user path (`REQUIRED_CHALLENGE_ANSWERS > 1`) exposed it because the extra synchronous `Views.views.vend.permissions(...)` inside `getAccountAttributesByAccount.map` shifted Future scheduling. If a regression resurfaces (the stress test starts flaking), the original fix directions still apply: route DB `Future`s through `RequestScopeConnection.fromFuture`, or thread the proxy connection explicitly down the connector call-chain. +> **Migration status, per-version progress, drift audit, and open TODOs** live in [`LIFT_HTTP4S_MIGRATION.md`](LIFT_HTTP4S_MIGRATION.md) — the single source of truth for *what's done / what's left*. This file (CLAUDE.md) is **how-to + gotchas only**. From 1545d8ee412c821022231c8f1ccadd38e58668d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 29 May 2026 09:59:18 +0200 Subject: [PATCH 11/13] =?UTF-8?q?docs(migration):=20condense=20to=20status?= =?UTF-8?q?=20+=20TODO=20(630=E2=86=92120=20lines)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - collapse per-version drift tables to a script-pointer + drift-category summary - merge Migration Order + Progress into one per-version table; auth stack to status rows - fold in TODO items moved out of CLAUDE.md (OBP-Trading, CI speed-up, disabled tests, MakerChecker resolved) - drop redundant ASCII server chain, 'Why http4s', duplicated migration-mapping prose - keep all open TODOs, decision gates, risks, bridge-audit playbook, done criteria --- LIFT_HTTP4S_MIGRATION.md | 668 +++++---------------------------------- 1 file changed, 80 insertions(+), 588 deletions(-) diff --git a/LIFT_HTTP4S_MIGRATION.md b/LIFT_HTTP4S_MIGRATION.md index df07645ed2..69ba7b129f 100644 --- a/LIFT_HTTP4S_MIGRATION.md +++ b/LIFT_HTTP4S_MIGRATION.md @@ -1,584 +1,114 @@ # Lift → http4s Migration -## Principle - -API version numbers reflect **API contract changes** (new/changed fields, new behaviour). The underlying framework is invisible to clients. Lift → http4s is a refactoring: it happens **in-place** inside the existing version file at the existing URL. No version bump. - -Use a new version (e.g. v7.0.0) only when the API contract itself changes — new fields, changed request/response shape, new behaviour. - ---- - -## Current Architecture - -OBP-API runs as a **single http4s Ember server** (single process, single port). The application entry point is a Cats Effect `IOApp` (`Http4sServer`). Lift is no longer used as an HTTP server — Jetty and the servlet container have been removed. - -Lift still plays two roles: - -1. **ORM / Database** — Lift Mapper manages schema creation, migrations, and data access. -2. **Legacy endpoint dispatch** — Older API versions are handled through a bridge (`Http4sLiftWebBridge`) that converts http4s requests into Lift requests, runs them through Lift's dispatch tables, and converts the responses back. - -New API versions are implemented as native http4s routes and do not pass through the bridge. - -### Entry point — `Http4sServer.scala` - -`Http4sServer` extends `IOApp`. On startup it: - -1. Calls `bootstrap.liftweb.Boot().boot()` to initialise Lift Mapper, connectors, and OBP configuration. -2. Parses the configured `hostname` and `dev.port` props (defaults: `127.0.0.1`, `8080`). -3. Starts an Ember server with the application defined in `Http4sApp.httpApp`. - -### Priority routing - -Routes are tried in order: `corsHandler` (OPTIONS) → `AppsPage` → `StatusPage` → `Http4s510` → `Http4s600` → `Http4s500` → `Http4s700` → `Http4sBGv2` → `Http4s400` → `Http4s310` → `Http4s300` → `Http4s220` → `Http4s210` → `Http4s200` → `Http4s140` → `Http4s130` → `Http4s121` → `Http4sLiftWebBridge` (Lift fallback). Unhandled `/obp/vX.Y.Z/*` paths fall through silently to Lift — they do not 404. The non-numeric ordering (v510 before v600, v500 after v600 etc.) doesn't affect correctness because each per-version service gates on its own version prefix; the ordering only matters when two services overlap on the same URL pattern. - -``` -HTTP Request - │ - ▼ -Http4sServer (IOApp / Ember) - │ - ▼ -corsHandler → AppsPage → StatusPage → Http4s510 → Http4s600 → Http4s500 → Http4s700 → Http4sBGv2 - │ - Http4s400 → Http4s310 → Http4s300 → Http4s220 → Http4s210 → Http4s200 → Http4s140 → Http4s130 → Http4s121 → Http4sLiftWebBridge - │ │ │ │ │ │ │ │ │ - v4.0.0 v3.1.0 v3.0.0 v2.2.0 v2.1.0 v2.0.0 v1.4.0 v1.3.0 v1.2.1 routes - own routes own routes own routes own routes own routes own routes own routes own routes (all 323 scenarios) - bridge bridge bridge bridge bridge bridge bridge - │ - LiftRules.statelessDispatch - LiftRules.dispatch (REST API) - │ - ▼ -HTTP Response (with standard headers) -``` - -### Version enable/disable semantics - -Two Props govern which API versions are served: `api_disabled_versions` and `api_enabled_versions` (allowlist; empty means "all"). They are enforced **once at startup**, by `Http4sApp.gate`: - -```scala -private def gate(version: ScannedApiVersion, routes: HttpRoutes[IO]): HttpRoutes[IO] = - if (APIUtil.versionIsAllowed(version)) routes else HttpRoutes.empty[IO] -``` - -A disabled version's top-level routes are replaced with `HttpRoutes.empty[IO]`, so a direct `GET /obp/vX.Y.Z/...` falls through to the Lift bridge and 404s. - -**Cascade is intentionally unaffected.** Each `Http4sXxx` has a path-rewriting bridge to the next-lower version that calls `code.api.vN.HttpNxx.wrappedRoutesVNxxServices` *directly*, bypassing `Http4sApp.gate`. `ResourceDocMiddleware` does **not** re-check `implementedInApiVersion` per request either (`ResourceDocMiddleware.isEndpointEnabled` deliberately has no `versionAllowed` parameter — `ResourceDocMiddlewareEnableDisableTest` pins this). So an endpoint originally declared in v2.0.0 stays reachable via `/obp/v4.0.0/...` even when v2.0.0 is disabled, as long as v4.0.0 is enabled. +Single source of truth for **migration status and open TODOs**. The how-to and the gotchas live in `CLAUDE.md` (§ Migrating a Lift Endpoint, § Tricky Parts). -This preserves the documented OBP-API contract: newer versions act as the supported entry point for older endpoints' functionality. Operators can retire a version's *URL prefix* with `api_disabled_versions` without losing the underlying endpoints from newer prefixes. To retire a specific endpoint everywhere, use `api_disabled_endpoints` (operationId list) — that **is** enforced per request by the middleware and so kills the endpoint on every prefix it would otherwise be reachable from. - -A brief regression in early 2026-05 inverted this: a `versionAllowed` check was added inside the middleware, making `api_disabled_versions` kill cascaded reachability too. Restored 2026-05-26. If you're tempted to put the per-request version check back, read the `isEndpointEnabled` docstring first — it spells out the design rationale, and the "version-level gating is delegated to Http4sApp.gate" feature in the unit test will fail loudly. +## Principle -### Lift bridge — `Http4sLiftWebBridge.scala` +API version numbers reflect **API contract changes** (new/changed fields, new behaviour), never the framework. Lift → http4s is an **in-place** refactor at the existing version/URL — no version bump. Use a new version only when the contract itself changes. -Handles any request not matched by a native http4s route: +## Architecture (current) -1. Reads the http4s request body. -2. Constructs a Lift `Req` from the http4s `Request[IO]`. -3. Creates a stateless Lift session. -4. Initialises a Lift `S` context and runs `LiftRules.statelessDispatch` / `LiftRules.dispatch`. -5. Handles Lift's `ContinuationException` pattern for async responses (timeout: `http4s.continuation.timeout.ms`, default 60 s). -6. Converts the Lift response back to http4s. +OBP-API runs as a single **http4s Ember server** (`Http4sServer`, a Cats Effect `IOApp`). Jetty / servlet container are gone. Lift now does only two things: (1) **Mapper ORM** — schema, migrations, all data access; (2) **legacy endpoint dispatch** for not-yet-migrated paths via `Http4sLiftWebBridge` (converts http4s ⇄ Lift `Req`, runs `LiftRules.dispatch`). Native http4s routes never touch the bridge. -### What Lift still does +**Priority routing** (`Http4sApp.baseServices`): `corsHandler` → AppsPage → StatusPage → LiftBridgeTraffic(audit) → Http4sResourceDocs → v510 → v600 → v500 → v700 → BGv2 → **ukV20 → ukV31** → v400 → v310 → v300 → v220 → v210 → v200 → v140 → v130 → v121 → dynamic-entity → dynamic-endpoint → `Http4sLiftWebBridge`. Each per-version service gates on its own prefix, so ordering only matters where URL patterns overlap. Unhandled `/obp/*` paths fall through to Lift (no 404). -| Area | Role | -|------|------| -| **Mapper ORM** | Database schema creation, migrations, and all data access (`MappedBank`, `AuthUser`, etc.) | -| **Boot** | Initialises OBP configuration, connectors, resource docs, and Mapper schemifier | -| **Dispatch tables** | `LiftRules.statelessDispatch` / `LiftRules.dispatch` hold endpoint definitions for versions not yet ported | -| **JSON utilities** | Some serialisation helpers from `net.liftweb.json` are still in use | +**Version enable/disable**: `api_disabled_versions` / `api_enabled_versions` (allowlist; empty = all) are enforced once at startup by `Http4sApp.gate` (disabled → `HttpRoutes.empty`). The path-rewriting cascade between versions bypasses `gate` deliberately, so an endpoint declared in v2.0.0 stays reachable via `/obp/v4.0.0/...` even if v2.0.0's prefix is disabled. To kill an endpoint on every prefix use `api_disabled_endpoints` (enforced per-request by the middleware). `ResourceDocMiddleware.isEndpointEnabled` must **not** re-check version per request — a 2026-05 regression that did was reverted 2026-05-26; `ResourceDocMiddlewareEnableDisableTest` pins this. ---- +## In-place migration, per file -## What "in-place migration" means per file +`APIMethods{version}.scala`: drop `self: RestHelper =>`; `lazy val xyz: OBPEndpoint` → `lazy val xyz: HttpRoutes[IO]`; Lift `case "path" :: Nil JsonGet _` → `case req @ GET -> \`prefixPath\` / "path"`; auth via the right `EndpointHelpers.*`; `ResourceDoc(root, ...)` → `ResourceDoc(null, ..., http4sPartialFunction = Some(root))`. `OBPAPI{version}.scala`: drop `extends OBPRestHelper` / `registerRoutes`, expose routes wired into the `Http4sServer` chain. Full Rule 1–5 reference + gotchas in `CLAUDE.md`. -### `APIMethods{version}.scala` +**One file = one PR; a file is fully Lift or fully http4s.** `APIMethods121` was done as a parallel `Http4s121.scala` (not in-place) because it's a mixin trait inherited by 130/140/etc.; http4s takes priority in the chain and the Lift trait is deleted once all inheritors are migrated. -| Before (Lift) | After (http4s) | -|---|---| -| `self: RestHelper =>` on the trait | removed | -| `lazy val xyz: OBPEndpoint` | `val xyz: HttpRoutes[IO]` | -| `case "path" :: Nil JsonGet _` | `case req @ GET -> \`prefixPath\` / "path"` | -| `authenticatedAccess(cc)` in for-comp | pick the right `EndpointHelpers.*` helper | -| `implicit val ec = EndpointContext(Some(cc))` | removed | -| `yield (json, HttpCode.\`200\`(cc))` | `yield json` | -| `ResourceDoc(root, ...)` | `ResourceDoc(null, ..., http4sPartialFunction = Some(root))` | +## Per-version progress -### `OBPAPI{version}.scala` +All 12 APIMethods files **done** — every functional endpoint on http4s, test suites green: -| Before | After | +| File | http4s | Notes | +|---|---|---| +| 121 | Http4s121 | 70 own; 323 API1_2_1Test scenarios | +| 130 | Http4s130 | 3 own + bridge→121 | +| 140 | Http4s140 | 11 own + bridge→130 | +| 200 | Http4s200 | 37 own + bridge→140 | +| 210 | Http4s210 | 25 own + bridge→200; 79 tests | +| 220 | Http4s220 | 18 own + bridge→210; 27 tests | +| 300 | Http4s300 | 47 own + bridge→220; 86 tests | +| 310 | Http4s310 | 100 own + bridge→300; 181 tests. See v3.1.0 leftovers below | +| 400 | Http4s400 | **258/258** (253 handlers + 8 txn-request-type doc aliases); lazy-val+init-def pattern; all 35 v4 overrides migrated | +| 500 | Http4s500 | 10 own | +| 510 | Http4s510 | 111 own; `createConsent` exposed as `createConsentImplicit` (EMAIL/SMS/IMPLICIT guard) | +| 600 | Http4s600 | **243/243** (35 overrides + 208 originals); introduced the lazy-val+init-def pattern | + +**v3.1.0 bridge leftovers** (both off-bridge in production; Lift definitions deferred to the bridge-removal PR): `getMessageDocsSwagger` — real handler in `Http4sResourceDocs`; `Http4s310` keeps an `HttpRoutes.empty` stub only so `nameOf(...)` compiles for `FrozenClassTest`. `getObpConnectorLoopback` — native http4s route that always returns 400 NotImplemented. Deleting their Lift `lazy val`s shrinks the frozen STABLE surface → needs a snapshot refresh. + +## Workstream status + +| Workstream | Status | |---|---| -| `extends OBPRestHelper` | removed | -| `registerRoutes(routes, allResourceDocs, apiPrefix)` | expose `val allRoutes: HttpRoutes[IO]` | -| registered via Boot / LiftRules | wired into `Http4sServer` chain | - -See `CLAUDE.md § Migrating a Lift Endpoint to http4s` for the full Rule 1–5 reference. - ---- - -## Migration Order - -Bottom-up — each version depends on the one below it being done. - -**Rule: one file = one PR. A file is either fully Lift or fully http4s — no half-converted state.** - -**Note on `APIMethods121`**: v1.2.1 was implemented as a new parallel file `Http4s121.scala` (rather than converting the Lift trait in-place) because `APIMethods121` is a mixin trait inherited by `APIMethods130`, `APIMethods140`, etc. Converting the trait in-place would require all inheriting versions to be migrated simultaneously. The parallel file approach lets v1.2.1 go first — http4s routes take priority in the chain; the Lift trait remains until all inheriting versions are done, at which point the Lift trait can be deleted. - -| # | File | Own endpoints | Notes | -|---|---|---|---| -| 1 | `APIMethods121` | 70 | **Done** — `Http4s121.scala` serves all endpoints; 323 tests pass | -| 2 | `APIMethods130` | 3 | **Done** — `Http4s130.scala`: 3 own endpoints + path-rewriting bridge to `Http4s121`; 2 PhysicalCardsTest scenarios pass | -| 3 | `APIMethods140` | 11 | **Done** — `Http4s140.scala`: 11 own endpoints + path-rewriting bridge to `Http4s130` | -| 4 | `APIMethods200` | 40 | **Done** — `Http4s200.scala`: 37 own endpoints + path-rewriting bridge to `Http4s140` | -| 5 | `APIMethods210` | 28 | **Done** — `Http4s210.scala`: 25 own endpoints + path-rewriting bridge to `Http4s200`; all 79 v2.1.0 tests pass | -| 6 | `APIMethods220` | 19 | **Done** — `Http4s220.scala`: 18 own endpoints + path-rewriting bridge to `Http4s210`; all 27 v2.2.0 tests pass | -| 7 | `APIMethods300` | 47 | **Done** — `Http4s300.scala`: 47 own endpoints + path-rewriting bridge to `Http4s220`; all 86 v3.0.0 tests pass | -| 8 | `APIMethods310` | 102 | **Done** — `Http4s310.scala` has all 100 functional endpoints (42 GET, 10 DELETE, 19 POST, 25 PUT, 1 GET-shaped revoke, 3 SCA aliases) + path-rewriting bridge to `Http4s300`; 181 v3.1.0 tests pass. The Lift `APIMethods310` trait is now a stub (live code commented out). `getObpConnectorLoopback` has a real native http4s route (always returns 400 NotImplemented). `getMessageDocsSwagger` has a `HttpRoutes.empty` stub in `Http4s310` — actual routing is owned by `Http4sResourceDocs`, the stub exists only so `nameOf(getMessageDocsSwagger)` compiles for `FrozenClassTest`. Both stubs deletable in the bridge-removal PR alongside a frozen-snapshot refresh. | -| 9 | `APIMethods400` | 258 | **Done — 258 / 258 (100%)**. `Http4s400.scala` covers all 253 unique handlers (`lazy val NAME: HttpRoutes[IO]`) plus 8 ResourceDoc aliases for the transaction-request-type variants (ACCOUNT, ACCOUNT_OTP, SEPA, COUNTERPARTY, REFUND, FREE_FORM, SIMPLE, AGENT_CASH_WITHDRAWAL — handled by the shared `createTransactionRequest` wildcard handler; the `literalAllCapsSegments` set in `Http4sSupport.scala` dispatches the matcher to the per-type doc for swagger purposes). Adopts the **lazy val + helper-def init pattern** (Batches 1–19) introduced in v6 to dodge the JVM 64KB `` method-size limit. **Bridge-cascade hijack** historically threatened v4's overrides; resolved by migrating all 35/35 v4-over-older URL+verb overrides. | -| 10 | `APIMethods500` | 10 | **Done** — `Http4s500.scala` (all v5.0.0 originals migrated) | -| 11 | `APIMethods510` | 111 | **Done** — `Http4s510.scala`. v5.1.0's `createConsent` Lift handler is exposed in Http4s510 under the alias name `createConsentImplicit` (a single handler with `if scaMethod == "EMAIL" \|\| scaMethod == "SMS" \|\| scaMethod == "IMPLICIT"` guard covers all three SCA-method URLs). | -| 12 | `APIMethods600` | 243 (35 overrides + 208 originals) | **Done — 243 / 243 (100%)**. `Http4s600.scala` covers all v6 originals and overrides. Wired into `Http4sApp.baseServices` ahead of the Lift bridge. Architecturally introduced the **lazy val + helper-def init pattern** to dodge the JVM 64KB `` method-size limit (`val xxx: HttpRoutes[IO]` ⇒ `lazy val xxx`; `resourceDocs += ResourceDoc(...)` calls grouped into `private def initXxxResourceDocs(): Unit` blocks). Future per-version files should adopt the same pattern from the start. | - ---- - -## Resource-docs (separate workstream) - -Resource-docs endpoints are **version-polymorphic**: `GET /obp/v6.0.0/resource-docs/v3.0.0/obp` returns v3.0.0 docs. The URL prefix is cosmetically version-specific but functionally irrelevant — the `API_VERSION` path segment controls the output. This makes resource-docs a natural candidate for a single centralized http4s service rather than per-version handlers. - -### Strategy: centralized `Http4sResourceDocs` - -Add one service to `Http4sApp` (above the Lift bridge, before any per-version service) that handles: - +| Resource-docs serving | **done** — `Http4sResourceDocs` serves `/obp/*/resource-docs/{ver}/{obp,swagger,openapi,openapi.yaml}`, the per-bank variant, and `/message-docs/{conn}/swagger2.0`; 10 Lift dispatch entries + the raw `openapi.yaml serve{}` block retired. ResourceDocsTest (63) + SwaggerDocsTest (10) green. | +| Resource-docs aggregation bug | **done** — `getResourceDocsObpV700` aggregates all versions; `V7ResourceDocsAggregationTest` passes. | +| Auth: DirectLogin | **done** — `DirectLoginRoutes` (bare `/my/logins/direct`, gated on `allow_direct_login`) + per-version paths; Lift dispatch removed. Migration gotcha: `createTokenFuture` ignored its args and re-read `S.request` — use `validatorFutureWithParams` instead. Dead `dlServe` block + `extends RestHelper` cleanup is a small follow-up PR. | +| Auth: GatewayLogin / DAuth / OAuth2 | **done** — all library-only validators (no routes); vestigial `extends RestHelper` removed. | +| Auth: OAuth 1.0a | **done — removed** (`51820c75e`): `oauth1.0.scala` deleted, `OAuthHandshake` unregistered, OAuth header parsing + dead fields removed. | +| Auth: OpenIdConnect | **blocked** — see Decision gates. The only auth handler still on Lift. | +| Dynamic-entity data plane | **done** — `Http4sDynamicEntity` serves `/obp/dynamic-entity/*` natively (`OBPAPIDynamicEntity` stays as a dormant Lift fallback, removed in the bridge-removal PR). | +| Dynamic-endpoint data plane | **todo (Phase 3)** — runtime Scala codegen (`DynamicEndpoints.scala`) emits Lift `OBPEndpoint`; still on the bridge. 3a (keep Lift-typed codegen behind an adapter) recommended for the Lift-removal milestone; 3b (emit `HttpRoutes[IO]`) deferred. The only remaining `real_work` bridge traffic. | +| Std: Berlin Group v2 | **done** — `Http4sBGv2`. | +| Std: UK Open Banking v2.0 + v3.1 | **done** — PR #2817 (merged 2026-05-29). v2.0: 5; v3.1: ~67 / 20 categories; Lift aggregators are `routes = Nil` stubs. 142 test scenarios pass. | +| Std: Berlin Group v1.3 | **todo** — 7 files still on active Lift (`code/api/berlin/group/v1_3/*`). | +| Std: Bahrain / AU / STET / MxOF / Polish | **retired** — commented out in PR #2814 (`d19af2b92`); `RetiredApiStandardsTest` guards against re-registration. | +| Std: Sandbox | **n/a** — `SandboxApiCalls.scala` fully commented out (dead code, registers no routes); deletion candidate. | + +## ResourceDoc parity (content fidelity vs Lift) + +Separate from serving: every migrated http4s `ResourceDoc(...)` should render identically to its Lift original. **`APIMethodsXYZ.scala` (the commented-out Lift) is the source of truth — never edit it to make the audit pass.** When the audit flags a diff, either fix http4s to match, or document a deliberate drift at the http4s site (placeholder rename for `ResourceDocMatcher`, upstream case-class shift, or a genuine improvement). Stub fidelity is verified: 0 field diffs across the v6 (243) and v5.1 (111) stubs vs their pre-stub Lift. + +Run the audit for the **live** drift list — don't transcribe it here (it rots): ``` -GET /obp/*/resource-docs/API_VERSION/obp → version-dispatch via getResourceDocsList -GET /obp/*/resource-docs/API_VERSION/openapi.yaml -GET /obp/*/message-docs/CONNECTOR/swagger2.0 → absorbs APIMethods310.getMessageDocsSwagger +python3 scripts/check_lift_http4s_resource_doc_parity.py [--field=X] [--list-only] ``` +Restoration tools: `rehydrate_resource_docs.py` (descriptions / example bodies from Lift comments) and `restore_resource_doc_bodies.py` (surgical per-field restore). At the last full audit ~60 drifts remained, almost all **middleware-driven placeholder renames** that are required and stay documented at the http4s site: `ACCOUNT_ID`→`NEW_ACCOUNT_ID` (PUT-creates-account), `VIEW_ID`→`*_VIEW_ID` (disambiguation), firehose `*_BANK_ID`/`*_VIEW_ID` (prop-before-bank bypass), `COUNTERPARTY_ID`→`COUNTERPARTY_ID_PARAM`, hyphen→underscore for `DYNAMIC_RESOURCE_DOC_ID`. Genuine fix candidates: verb-casing (`revokeMyConsent` Delete→DELETE), v4 `deleteExplicitCounterparty` POST→DELETE (REST-correct), and v4's two only-lift endpoints never ported (`getAllAuthenticationTypeValidationsPublic`, `getAllJsonSchemaValidationsPublic`). -The wildcard prefix means all resource-doc requests are intercepted regardless of which version prefix the client uses. This workstream is **independent of the per-version migration order** — it can land at any time and immediately removes all resource-docs traffic from the Lift bridge. - -### Prerequisite: fix the aggregation bug - -`V7ResourceDocsAggregationTest` is intentionally failing. The current `getResourceDocsObpV700` has a broken branch for `requestedApiVersion == v7.0.0` that manually iterates `allResourceDocs` (~45 own docs) instead of calling `getResourceDocsList`, which aggregates all 500+. Fix this first — it is the same defect the centralized service must not repeat. - -### `openapi.yaml` - -Currently served via a raw Lift `serve { case Req(..., "openapi.yaml", ...) }` block that bypasses `registerRoutes` entirely. Needs a dedicated http4s route (no ResourceDocMiddleware) added to the centralized service. - -### Caching - -`Caching.getStaticSwaggerDocCache()` / `setStaticSwaggerDocCache()` are framework-agnostic and already used from within the http4s path. No migration work needed. - -### Steps - -1. Fix aggregation bug in `getResourceDocsObpV700` → make `V7ResourceDocsAggregationTest` pass. -2. Extract shared handler logic into `Http4sResourceDocs` service; wire into `Http4sApp`. -3. Add `openapi.yaml` route to the same service. -4. ~~Port `getMessageDocsSwagger` from `APIMethods310` into the same service~~ — **Done.** Now served by `Http4sResourceDocs.handleGetMessageDocsSwagger` via the wildcard `/obp/*/message-docs/{CONNECTOR}/swagger2.0` route matched before any per-version service. The `val getMessageDocsSwagger: HttpRoutes[IO] = HttpRoutes.empty` stub in `Http4s310.scala` exists only to satisfy the `FrozenClassTest` API-surface check. -5. Remove resource-docs from the per-version Lift objects (`ResourceDocs140`–`ResourceDocs600`) once the centralized service covers them. - ---- - -## ResourceDoc parity (per-version drift from Lift) - -Separate from the resource-docs **serving** workstream above, there is a parity workstream covering the **content** of each migrated ResourceDoc declaration. The goal is for every http4s `ResourceDoc(...)` to render identically to its Lift original, so the public API docs aren't silently degraded by migration. - -### Principle - -**`APIMethodsXYZ.scala` (Lift) is the source of truth for migration.** The commented-out Lift ResourceDocs and endpoints inside each `APIMethodsXYZ.scala` are the canonical reference for what the http4s version should render: URL templates, verb casing, summaries, descriptions, example bodies, error lists, tags. **Do NOT edit these files to make the audit pass** — the audit compares http4s against the Lift source-of-truth. When the audit flags a diff, the resolution is either (a) update http4s to match Lift, or (b) document the difference at the http4s site as a known intentional drift (placeholder rename for middleware, upstream-driven case-class shift, etc.). Rewriting the Lift comments runs the comparison backwards and erases the historical record. (Mistakes in commits `d95c1df01` and `6154bf2cc` did this; reverted in `27f48af72`.) - -**Stub fidelity verified.** Commits `810589330` (v6) and `88f46f854` (v5.1) replaced the live Lift code with commented-out stubs. Comparing each stub's uncommented ResourceDoc bodies against the pre-stub live versions: **0 field diffs across 243/243 v6 docs and 111/111 v5.1 docs**. The non-ResourceDoc deltas (imports, etc., ~16KB v6 / ~5KB v5.1) are immaterial. The stubs are an exact preservation of the original Lift ResourceDocs. - -### Tooling (`scripts/`) - -| Script | Role | -|---|---| -| `check_lift_http4s_resource_doc_parity.py` | Read-only audit. Parses both files, matches by `nameOf(...)` (with `.replace("a","b")` evaluation for derived names), reports per-field diffs. `--field=X` to focus, `--list-only` for endpoint-presence summary. | -| `rehydrate_resource_docs.py` | Upstream (simonredfern, `67593ea28`). Lifts positional args 7/8/9 (description, exampleRequestBody, successResponseBody) from commented Lift blocks into http4s. Has a `split-init` subcommand for JVM 64KB method-size workaround. | -| `restore_resource_doc_bodies.py` | Companion to the above. Restores any subset of (summary, description, exampleRequestBody, successResponseBody, errorResponseBodies, tags) from Lift into http4s. Surgical per-field replacement preserves layout. `--fields=X,Y` to scope, `--only=ep` to target one endpoint. | - -### Current drift (audit re-run 2026-05-21 evening) - -| Version | shared | mismatch | only-lift | only-http4s | Status | -|---|---|---|---|---|---| -| v1_2_1 | 70 | 6 | 0 | 0 | semantic fields restored; 6 structural drifts remain | -| v1_3_0 | 3 | 0 | 0 | 0 | clean | -| v1_4_0 | 10 | 1 | 0 | 0 | one minor | -| v2_0_0 | 37 | 1 | 0 | 0 | semantic fields restored; 1 structural drift remains | -| v2_1_0 | 23 | 1 | 5 | 2 | semantic fields restored; 1 structural drift remains | -| v2_2_0 | 18 | 0 | 0 | 18 | Lift trait fully retired upstream (commit `71892f5cb`); audited against pre-stub Lift via git history; 13 fields restored; 3 middleware URL renames remain | -| v3_0_0 | 47 | 4 | 0 | 0 | semantic fields restored; 4 middleware-driven URL renames remain | -| v3_1_0 | 102 | 5 | 0 | 0 | semantic fields restored; 5 structural drifts (placeholder renames) remain | -| v4_0_0 | 254 | 20 | 2 | 5 | semantic fields restored; 20 structural drifts (placeholder renames + 1 verb fix) remain | -| v5_0_0 | 39 | 8 | 0 | 3 | descriptions restored; structural/errors remain | -| v5_1_0 | 111 | 1 | 1 | 2 | one verb-casing drift to fix | -| v6_0_0 | 243 | 12 | 0 | 1 | 11 placeholder renames + 1 routing-shape upstream change | -| **Total** | **956** | **60** | | | | - -### v6.0.0 — 12 specific drifts (each is a fix candidate) - -These are the cases where http4s deviates from Lift. Under the source-of-truth rule, the default is to fix http4s; deliberate exceptions need to be documented at the http4s site. - -| Endpoint | Field | Lift | http4s | Resolution | -|---|---|---|---|---| -| `createCounterpartyAttribute` | requestUrl | `…/counterparties/COUNTERPARTY_ID/attributes` | `…/COUNTERPARTY_ID_PARAM/…` | TBD — verify `ResourceDocMatcher` correctly handles `COUNTERPARTY_ID` as a wildcard (the literal set contains `COUNTERPARTY`, but `COUNTERPARTY_ID` is whole-segment-different). If safe, revert to Lift's name. | -| `deleteCounterpartyAttribute` | requestUrl | same | same | same as above | -| `getAllCounterpartyAttributes` | requestUrl | same | same | same as above | -| `getCounterpartyAttributeById` | requestUrl | same | same | same as above | -| `updateCounterpartyAttribute` | requestUrl | same | same | same as above | -| `createTransactionRequestCardano` | requestUrl | `…/ACCOUNT_ID/owner/transaction-request-types/CARDANO/…` | `…/ACCOUNT_ID/VIEW_ID/…/CARDANO/…` | **Functional broadening** — http4s lets any view, Lift hardcoded `owner`. Keep http4s; document at the http4s ResourceDoc site. | -| `createTransactionRequestHold` | requestUrl | `…/owner/…HOLD/…` | `…/VIEW_ID/…HOLD/…` | same as above | -| `getSystemViewById` | requestUrl | `/management/system-views/VIEW_ID` | `/management/system-views/SYS_VIEW_ID` | TBD — disambiguation rename. If `ResourceDocMatcher` handles both fine, revert. | -| `updateSystemView` | requestUrl | `/system-views/VIEW_ID` | `/system-views/UPD_VIEW_ID` | same as above | -| `removeBankReaction` | requestUrl | `…/reactions/EMOJI` | `…/reactions/EMOJI_REACTION` | `EMOJI` is NOT in `literalAllCapsSegments` (only `EMAIL`/`SMS`/`IMPLICIT` of the SCA cluster are). Rename may have been defensive; safe to revert. | -| `removeSystemReaction` | requestUrl | same | same | same as above | -| `getAccountDirectory` | successResponseBody | `FastFirehoseRoutings(bank_id, account_id)` | `AccountRoutingJsonV121(scheme, address)` | **Upstream functional change** (`9e151c524` / `9dc4c4c46` migrated the case class). Cannot revert; document. Also note: the same change broke `mvn test` (pre-existing upstream compile error in `JSONFactory6.0.0.scala:2934`). | - -Also: 1 only-http4s (`createWebUiProps`) — genuinely http4s-only with no Lift counterpart. Document. - -### v5.1.0 — 1 specific drift - -| Endpoint | Field | Lift | http4s | Resolution | -|---|---|---|---|---| -| `revokeMyConsent` | requestVerb | `"Delete"` | `"DELETE"` | Trivial casing fix on the http4s side. | - -Also: -- 1 only-lift (`createConsentImplicit`) + 1 only-http4s (`createConsent`) — Lift had `lazy val createConsentImplicit = createConsent` aliasing and registered the doc under the alias; http4s registers under the canonical name. Fix: in http4s, either rename the partial function to `createConsentImplicit` to match Lift, or register a second `nameOf(createConsentImplicit)` doc for the same handler. -- 1 only-http4s (`getBanks`) — kept in the v5.1.0 layer for metrics attribution (intentional addition; see comment at `Http4s510.scala:288`). Document. - -### v2.2.0 — 3 specific drifts + 18 only-http4s - -Upstream commit `71892f5cb` retired `APIMethods220.scala`'s Lift trait entirely (1361 lines → 9-line empty stub). For audit purposes, the Lift source-of-truth is preserved in git history at `71892f5cb^`. A one-off Python script using `restore_resource_doc_bodies.py` helpers (in `scripts/`) extracts the pre-stub `APIMethods220.scala` and runs the standard field restoration against `Http4s220.scala`. - -After restoration (13 descriptions + 1 example body + 1 success body), only 3 middleware-driven URL renames remain: - -| Endpoint | Field | Lift | http4s | Resolution | -|---|---|---|---|---| -| `createAccount` | requestUrl | `…/accounts/ACCOUNT_ID` | `…/accounts/NEW_ACCOUNT_ID` | PUT-creates-account bypass. **Document**. | -| `createViewForBankAccount` | requestUrl | `…/accounts/ACCOUNT_ID/views` | `…/accounts/VIEW_ACCOUNT_ID/views` | Account-validation bypass. **Document**. | -| `updateViewForBankAccount` | requestUrl | `…/views/VIEW_ID` | `…/views/UPD_VIEW_ID` | Disambiguation rename. **Document**. | - -The 18 only-http4s entries are the actual v2.2.0 surface — there is no live Lift counterpart in the file anymore. They're audited indirectly against the git-history Lift. - -Two supporting imports were added to `Http4s220.scala` so the restored descriptions / example bodies compile: `code.api.util.Glossary` and `java.util.Date`. - -### v3.0.0 — 4 specific drifts - -After semantic-field restoration, only middleware-driven URL renames remain. - -| Endpoint | Field | Lift | http4s | Resolution | -|---|---|---|---|---| -| `createViewForBankAccount` | requestUrl | `…/accounts/ACCOUNT_ID/views` | `…/accounts/VIEW_ACCOUNT_ID/views` | Middleware account-validation bypass (see CLAUDE.md "Middleware URL template bypass" gotcha). **Document** — required. | -| `updateViewForBankAccount` | requestUrl | `…/views/VIEW_ID` | `…/views/UPD_VIEW_ID` | Disambiguation rename. **Document**. | -| `getFirehoseAccountsAtOneBank` | requestUrl | `/banks/BANK_ID/firehose/accounts/views/VIEW_ID` | `/banks/FIREHOSE_BANK_ID/firehose/accounts/views/FIREHOSE_VIEW_ID` | Firehose middleware bypass. **Document**. | -| `getFirehoseTransactionsForBankAccount` | requestUrl | `/banks/BANK_ID/firehose/accounts/ACCOUNT_ID/views/VIEW_ID/transactions` | `/banks/FIREHOSE_BANK_ID/firehose/accounts/FIREHOSE_ACCOUNT_ID/views/FIREHOSE_VIEW_ID/transactions` | Same firehose pattern. **Document**. | - -No only-lift or only-http4s entries for v3.0.0. - -### v3.1.0 — 5 specific drifts - -After semantic-field restoration (commit `f4b9bd183`), only middleware-driven placeholder renames remain. - -| Endpoint | Field | Lift | http4s | Resolution | -|---|---|---|---|---| -| `createAccount` | requestUrl | `/banks/BANK_ID/accounts/ACCOUNT_ID` | `…/NEW_ACCOUNT_ID` | PUT-creates-account pattern. Middleware would 404 on `ACCOUNT_ID` lookup before the handler. **Document** — required. | -| `deleteSystemView` | requestUrl | `/system-views/VIEW_ID` | `/SYS_VIEW_ID` | Disambiguation from other VIEW_ID usages. **Document**. | -| `getSystemView` | requestUrl | same | same | same | -| `updateSystemView` | requestUrl | same | same | same | -| `getFirehoseCustomers` | requestUrl | `/banks/BANK_ID/firehose/customers` | `…/FIREHOSE_BANK_ID/…` | Firehose middleware bypass — prop check must run before bank lookup (see CLAUDE.md). **Document** — required. | +## Open TODOs — master list for "remove Lift Web" -No only-lift or only-http4s entries for v3.1.0. +**Bridge-traffic audit (data-driven prioritisation).** Every bridge hit is tallied by `Http4sLiftBridgeTraffic`; `GET /admin/lift-bridge-traffic` returns `real_work` (non-404 = migration targets) vs `not_found` (stale/probes, ignore); `POST .../reset` clears it. Playbook: reset on a representative instance → run a normal traffic window (24h + scheduled jobs) → if `real_work[]` is empty the bridge is retirable (modulo documented leftovers). Last CI audit: the only `real_work` left is `/obp/dynamic-endpoint/...` (Phase 3); `/obp/dynamic-entity/...` is now native. -### v4.0.0 — 20 specific drifts + 2 only-lift + 5 only-http4s +1. **Dynamic-endpoint Phase 3** — see workstream table. +2. **OpenIdConnect** — blocked on the OIDC portal-session decision (gate 1). +3. **Bridge-removal PR** — delete `Http4sLiftWebBridge` + the request-path `Boot.scala` hooks: the `LiftRules.statelessDispatch.append(...)` registrations (DirectLogin, ResourceDocs140–600, aliveCheck), `LiftRules.dispatch.append(OpenIdConnect)`, `addToPackages("code")`, the global exception + 404 handlers, the `early`/`supplementalHeaders`/`localeCalculator` request hooks, and `unloadHooks`. The Mapper schemifier stays (that's lift-mapper, not the bridge). Plan a `FrozenClassTest` snapshot refresh in the same PR. +4. **Open-banking standards** — decide BG v1.3's fate (gate 2). +5. **`lift-mapper`** — separate long-term ORM replacement; out of scope here. +6. **Misc**: OBP-Trading payment-auth endpoints (notifyDeposit, create/capture/release/getPaymentAuth) still commented out in `Http4s700` (see `ideas/CAPTURE_RELEASE_TRANSACTION_REQUEST_TYPES.md`); CI speed-up (two-tier fast gate + surefire parallel forks) not done. -After semantic-field restoration (commit `2b24811e5`), the remaining drifts are all structural / functional: +**Small singletons:** `aliveCheck` **done** (`AliveCheckRoutes`, `GET /alive`); `ImporterAPI` **retired** (endpoint + `TransactionInserter` + connector helpers removed). -| Category | Count | Endpoints | Resolution | -|---|---|---|---| -| requestVerb | 1 | `deleteExplicitCounterparty` (Lift `POST` → http4s `DELETE`) | http4s is REST-correct. **Document** as deliberate fix. | -| requestUrl — `VIEW_ID` → `GRANT_VIEW_ID` | 9 | `answerTransactionRequestChallenge` and 8 `createTransactionRequest*` variants (Account/AccountOtp/AgentCashWithDrawal/Counterparty/FreeForm/Refund/Sepa/Simple) | Middleware disambiguation rename. Verify if `VIEW_ID` collides in `ResourceDocMatcher`; if not, revert. If it does, **document**. | -| requestUrl — hyphen→underscore | 6 | `delete`/`get`/`update` × `BankLevelDynamicResourceDoc` / `DynamicResourceDoc` (Lift `DYNAMIC-RESOURCE-DOC-ID` → http4s `DYNAMIC_RESOURCE_DOC_ID`) | The matcher's ALL_CAPS-with-underscores wildcard requires underscores. **Fix Lift**? No — Lift is source-of-truth. **Document** at the http4s site as a required matcher constraint. | -| requestUrl — `COUNTERPARTY_ID` → `COUNTERPARTY_ID_PARAM` | 2 | `deleteExplicitCounterparty`, `getCounterpartyByIdForAnyAccount` | Same as v6's COUNTERPARTY rename family. Verify matcher behavior; revert if safe. | -| requestUrl — `COUNTERPARTY_ID` → `EXPLICIT_COUNTERPARTY_ID` | 1 | `getExplicitCounterpartyById` | Same defensive rename pattern. | -| requestUrl — firehose pattern | 1 | `getFirehoseAccountsAtOneBank` (Lift `BANK_ID/.../VIEW_ID` → http4s `FIREHOSE_BANK_ID/.../FIREHOSE_VIEW_ID`) | Middleware bypass for the prop-check-before-bank-lookup pattern (see CLAUDE.md "Prop check before role check" gotcha). **Document** — required for correctness. | -| requestUrl — Lift URL malformed | 1 | `deleteCustomerAttribute` (Lift `/banks/BANK_ID/CUSTOMER_ID/attributes/.../...` is missing `/customers/`; http4s uses `/banks/BANK_ID/customers/attributes/...`) | Lift URL was buggy. http4s fixed it. **Document** as deliberate URL fix; flag that the Lift comment preserves the original bug as historical record. | +**Disabled / ignored tests to revisit:** `Http4s500RoutesTest`, `RootAndBanksTest`, `V500ContractParityTest` (`@Ignore`); `CardTest` (commented out); v5.0.0 13 skipped; `AbacRuleTests` 6 local fails are environment-dependent (too few users → `isStatisticallyTooPermissive`), not a regression. The `MakerCheckerTransactionRequestTest` proxy/TTL race is **resolved** by the `RequestScopeConnection` hardening on `develop` (regression-guarded by its "Stress: repeated multi-challenge creates" scenario); if it ever flakes again, route DB Futures through `RequestScopeConnection.fromFuture`. -Also: 2 only-lift (`getAllAuthenticationTypeValidationsPublic`, `getAllJsonSchemaValidationsPublic`) — these endpoints exist in Lift v4 but were not migrated to `Http4s400`. **Migration gap** — port them. 5 only-http4s (`createBankLevelDynamicEntity`, `createSystemDynamicEntity`, `updateBankLevelDynamicEntity`, `updateMyDynamicEntity`, `updateSystemDynamicEntity`) — dynamic-entity overrides added in http4s with no Lift equivalent. Document if intentional, or audit whether they should have Lift counterparts. +## Decision gates (stakeholder calls, before the bridge-removal PR) -### v5.0.0 — 8 specific drifts + 3 only-http4s +1. **OIDC portal-session strategy.** The OIDC callback's success path calls `AuthUser.logUserIn` / `S.redirectTo`, which mutate Lift `SessionVar`s the portal reads. Forks: **(a) drop portal-login** — pure http4s callback issues a token but seeds no portal session (behaviour change; needs sign-off from any OIDC portal-UI users); **(b) Lift-session shim** — keep `lift-webkit` for this one callback (cheapest code; "Lift Web removed" never actually ships); **(c) replace portal session** (Redis/JWT-backed; months, but also unblocks lift-mapper later). No tests cover the callback success path. **This is the only thing blocking bridge removal.** +2. **Open-banking standards.** Not required for bridge removal, but for the public claim: if the headline is "Lift Web removed", a feature-flagged Lift remnant (BG v1.3) is acceptable; if "Lift Web removed *from this repo*", BG v1.3 must be migrated or extracted as a plugin project. -| Category | Count | Endpoints | Resolution | -|---|---|---|---| -| requestUrl placeholder rename | 1 | `createAccount` (Lift `ACCOUNT_ID` → http4s `NEW_ACCOUNT_ID` for the PUT-creates pattern) | Verify matcher behavior; may be required for `ACCOUNT_ID` literal handling. | -| errorResponseBodies — SCA val-vs-inline | 3 | `createConsentByConsentRequestIdEmail` / `Sms` / `Implicit` | http4s uses `private val createConsentByConsentRequestIdCommonErrors = List(...)` for DRY; Lift inlined the list. Either inline the val in the 3 doc registrations to match Lift verbatim, or extend the audit script to expand simple `val X = List(...)` references. | -| errorResponseBodies — system-view accuracy | 4 | `createSystemView`, `deleteSystemView`, `getSystemView`, `updateSystemView` | http4s has more accurate errors (`SystemViewNotFound`, `SystemViewCannotBePublicError`, `InvalidSystemViewFormat`). Lift had wrong/legacy errors (`BankAccountNotFound`, `$BankNotFound`, `"user does not have owner access"`). **Genuine improvement** — document at http4s site. | - -Also: 3 only-http4s (`getBanks`, `getProduct`, `getProducts`) — kept in this layer for metrics attribution. Document. - -### Strategy summary - -For each remaining drift on a migrated version: -1. **Default**: fix http4s to match Lift verbatim. Use `restore_resource_doc_bodies.py` for field-level restoration. -2. **Documented exceptions**: where the drift is a deliberate http4s improvement or required by middleware semantics, leave the drift and add a `// Lift had X; we use Y because Z` comment at the http4s ResourceDoc site. -3. **Never**: edit `APIMethodsXYZ.scala` to make the audit pass. The Lift comments are the canonical record. - -Untouched versions (v1_2_1 through v4_0_0, plus v2_1_0) need the same treatment: run `rehydrate_resource_docs.py` then `restore_resource_doc_bodies.py`, then audit and address any residual drifts at the http4s site. - ---- - -## Auth Stack (separate workstream) - -Token-generation paths — not version-file endpoints. Each `extends RestHelper` and needs to become an http4s route or middleware independently. Can run in parallel with the APIMethods migration. - -| Component | Path | Notes | -|---|---|---| -| `DirectLogin` | `POST /my/logins/direct` | Done — served by `Http4s600.directLoginEndpoint` (versioned) and `DirectLoginRoutes` (bare path). | -| `GatewayLogin` | gateway JWT exchange | Library-only validator (no routes). | -| `DAuth` | dAuth JWT exchange | Library-only validator (no routes). | -| `OpenIdConnect` | OIDC callback | Blocked — last hard dependency on Lift Web in the request path. See auth-stack leftovers table. | - -OAuth 1.0a token endpoints were removed entirely in commit `51820c75e` (2026-02-20); the workstream collapsed. - ---- - -## Per-version Lift leftovers - -An `APIMethods{version}` file is marked **done** in the progress table when every *functional* endpoint is on http4s and the version's test suite is green. A small number of endpoints are deliberately *not* migrated inline because they belong to a different workstream or have no behaviour worth porting. They continue to be served by the Lift bridge until the workstream that owns them lands; they do **not** create new follow-up work on the per-version file. - -| Endpoint | Origin | Why on Lift | Retired by | -|---|---|---|---| -| `getMessageDocsSwagger` (`GET /message-docs/CONNECTOR/swagger2.0`) | `Http4s310` (stub) + `Http4sResourceDocs` (real handler) | **Effectively done.** The real handler lives in `Http4sResourceDocs.handleGetMessageDocsSwagger`, matched by the wildcard `/obp/*/message-docs/{CONNECTOR}/swagger2.0` before any per-version service. `Http4s310.scala` keeps a stub `val getMessageDocsSwagger: HttpRoutes[IO] = HttpRoutes.empty` plus a `ResourceDoc` entry so the ResourceDoc surface stays consistent and the `FrozenClassTest` surface check keeps passing (`nameOf(getMessageDocsSwagger)` compiles). No bridge dispatch is involved. | The stub can be deleted as part of the bridge-removal PR alongside the frozen-snapshot refresh; until then the wiring above is correct in production. | -| `getObpConnectorLoopback` (`GET /connector/loopback`) | `Http4s310` | **Done.** Native http4s route in `Http4s310.scala` (~line 4875) — `booleanToFuture(NotImplemented, failCode = 400) { false }`, i.e. the route always returns 400 NotImplemented, mirroring Lift's original deprecated-stub behaviour. No bridge dispatch. | Deletable in the bridge-removal PR (or kept indefinitely as a documented deprecation stub). | -| ~~`testResourceDoc`~~ | ~~`APIMethods140`~~ | Dev-mode-only `/dummy` stub deleted — returned a dummy `APIInfoJSON`, no production behaviour. Removed from `OBPAPI1_4_0.routes` and `Implementations1_4_0`. `FrozenClassTest` did not flag it because v1.4.0's `testResourceDoc` ResourceDoc was registered behind `if (Props.devMode)` — the frozen snapshot (captured in test mode) never contained it. | **Done.** | - -Track new leftovers here when later version files are migrated — the bridge-removal milestone in "Done Criteria" only requires the per-version files to be **done** in this table's sense (functional endpoints migrated, tests green). Leftovers folded into the Resource-docs or Auth-stack workstreams retire via those workstreams. - ---- - -## Migration leftovers (full landscape, beyond per-version files) - -Things still on Lift that block the `Http4sLiftWebBridge` from being removed. Use this section as the master TODO for the "remove Lift Web" milestone. - -### Bridge-traffic audit (data-driven prioritisation) - -Every request that reaches `Http4sLiftWebBridge.dispatch` is tallied in-memory by `Http4sLiftBridgeTraffic` so we can see exactly what still needs migrating before the bridge can be retired. - -- **First hit of any (method, path-bucket, status)** triple is logged at INFO: `[BRIDGE-AUDIT] first hit: METHOD /path/bucket STATUS (original path: /actual/path)`. Subsequent hits only increment an `AtomicLong`. -- **Snapshot endpoint** — `GET /admin/lift-bridge-traffic` returns the tally grouped into `real_work` (non-404) and `not_found` (404). 404 entries are typically test-probe traffic / stale URLs / dead links and are **not** migration work. Each group is sorted by hit count desc: - ```json - { - "unique_buckets": 5, - "total_hits": 248, - "summary": { - "real_work": {"unique_buckets": 3, "total_hits": 230}, - "not_found": {"unique_buckets": 2, "total_hits": 18} - }, - "real_work": [ - {"method": "GET", "bucket": "/auth/openid-connect/callback", "status": 200, "count": 99}, - {"method": "POST", "bucket": "/obp/dynamic-entity/FooBar", "status": 201, "count": 88}, - {"method": "GET", "bucket": "/obp/dynamic-entity/FooBar", "status": 200, "count": 43} - ], - "not_found": [ - {"method": "DELETE", "bucket": "/obp/v4.0.0/banks/{id}/accounts", "status": 404, "count": 16}, - {"method": "GET", "bucket": "/favicon.ico", "status": 404, "count": 2} - ] - } - ``` -- **Reset** — `POST /admin/lift-bridge-traffic/reset` clears the tally (handy for taking a baseline before a load test). -- **Path normalisation** collapses opaque IDs so the map doesn't fill up: UUIDs → `{uuid}`, all-digits → `{n}`, anything with a dot or 12+-char alnum mixed → `{id}`. API-version strings (`v6.0.0`, `v1_2_1`) are kept verbatim. Unit-tested in `Http4sLiftBridgeTrafficTest` (9 cases). - -Operator playbook for "is the bridge ready to retire?": -1. Reset the tally on a representative instance. -2. Let it run through a normal traffic window (e.g. 24h + the daily/weekly jobs). -3. Query `/admin/lift-bridge-traffic`: - - **`real_work[]` empty** → bridge can be retired (modulo any documented leftovers). - - **`real_work[]` non-empty** → those buckets are concrete migration targets. Each entry is a (method, URL pattern) that some live caller still needs Lift to serve. - - **`not_found[]`** is informational — useful for spotting stale callers or unused URL patterns, but not blocking bridge removal. - -First real audit data (shard 1 CI run on 2026-05-25, 515 tests): -- 20 `real_work` entries — all the `/obp/dynamic-entity/...` and `/obp/dynamic-endpoint/...` URLs. These are runtime-generated by Lift's dynamic dispatch when an admin creates a dynamic entity / endpoint at runtime; porting them is a workstream of its own (not endpoint-by-endpoint). -- 2 `not_found` entries — `DELETE /obp/v4.0.0/banks/{id}/accounts`, asserted as 404 by `DeleteBankCascadeTest` to verify the cascade actually wiped the bank. - -**Update (dynamic-entity ported):** the `/obp/dynamic-entity/...` half of that `real_work` bucket is now served natively by `code.api.dynamic.entity.Http4sDynamicEntity` (see Progress table) and no longer reaches the bridge — in the dynamic-entity test runs the only `/obp/dynamic-entity/...` bridge hits left are legitimate 404 fall-throughs (e.g. a `my/` request against a system-level entity that has no personal endpoint). The remaining `real_work` is `/obp/dynamic-endpoint/...` only (runtime Scala codegen → Lift `OBPEndpoint`; Phase 3, still on the bridge). - -### Auth stack — every handler is its own `RestHelper` - -| Handler | File | Routes | Status | -|---|---|---|---| -| `DirectLogin` | `code/api/directlogin.scala` | `POST /my/logins/direct` | **Done.** Versioned path (`/obp/v6.0.0/my/logins/direct`) served by `Http4s600.directLoginEndpoint`; bare path (`/my/logins/direct`) served by `code.api.DirectLoginRoutes` wired into `Http4sApp.baseServices` just before the Lift bridge. `LiftRules.statelessDispatch.append(DirectLogin)` removed from `Boot.scala`. The `allow_direct_login` prop gate moved into `DirectLoginRoutes`. The `dlServe { case Req("my" :: "logins" :: "direct" :: Nil, …) }` block inside `directlogin.scala` is now dead code (no longer registered with `LiftRules`); the surrounding `DirectLogin` object stays — its `getUserFromDirectLoginHeaderFuture` etc. are still called from auth flows. Cleanup of the dead `dlServe` block + `extends RestHelper` is a separate small PR. Key migration gotcha (kept for the auth-stack workstream): `createTokenFuture(allParameters)` ignores its argument and re-reads from Lift's `S.request` via `getAllParameters` — use `validatorFutureWithParams(...)` + `createTokenCommonPart(...)` instead. | -| `GatewayLogin` | `code/api/GatewayLogin.scala` | Gateway JWT exchange | **No routes.** Library only — same shape as `OAuth2Login`. Consumed via `GatewayLogin.getUserFromGatewayLoginHeaderFuture` etc. from auth flows. `extends RestHelper` was vestigial and was removed (Formats implicit re-declared locally). | -| `DAuth` | `code/api/dauth.scala` | dAuth JWT exchange | **No routes.** Library only — same shape as `OAuth2Login`/`GatewayLogin`. `extends RestHelper` was vestigial and was removed (object-level `implicit val formats` added for the `.extract[...]` call sites). | -| `OAuth 1.0a` | — | OAuth 1.0a token endpoints | **Done — removed.** Commit `51820c75e` (2026-02-20, "refactor/(auth): Remove OAuth 1.0a support and consolidate authentication") deleted `oauth1.0.scala`, unregistered `OAuthHandshake` from `Boot.scala`'s `LiftRules.statelessDispatch`, removed OAuth header detection from `OBPRestHelper.scala`, and added `getConsumerFromDirectLoginToken` / `getUserFromDirectLoginToken` to take over the consumer/user-lookup responsibilities previously held by `OAuthHandshake`. **Dead-code follow-up cleanup also done**: `AuthHeaderParser.parseOAuthHeader`, the `oAuthParams` field on `ParsedAuthHeader` / `CallContext`, the `oAuthToken` field on `CallContextLight`, `extractOAuthParams` in `Http4sSupport`, `APIUtil.hasAnOAuthHeader`, and stale `OAuthHandshake` comments in `directlogin.scala` and `AuthUser.scala` all removed. `OpenAPI31JSONFactory`'s phantom OAuth2 `authorizationCode` flow (which pointed at the deleted `/oauth/authorize` and `/oauth/token` URLs) replaced with `type: http, scheme: bearer, bearerFormat: JWT` — accurate for OBP's actual OAuth2 model (Bearer-token validation against external IdPs; OBP does not issue its own OAuth2 tokens). Kept on purpose: `code/model/OAuth.scala` (backs the general `Consumer` entity used by all auth methods, not OAuth 1.0a-specific) and `APIUtil.OAuth` (misnamed but live test infrastructure — the `<@` signer adds `Authorization: DirectLogin token=...` headers and is imported by ~hundreds of test files; renaming is a separate cleanup). | -| `OAuth2` | `code/api/OAuth2.scala` (`OAuth2Login`) | **No routes.** Library only — Bearer-token validator (Google / Yahoo / Azure / Keycloak / OBPOIDC / Hydra) consumed by `APIUtil.getUserFuture` and `OBPRestHelper.OAuth2.getUser`. Both Lift and http4s endpoints already call it. The `extends RestHelper` mixin was vestigial and was removed (the only thing it provided was an implicit `Formats`, now declared locally at the one `extract[List[String]]` site). No remaining auth-stack work in this file. | -| `OpenIdConnect` | `code/api/openidconnect.scala` | OIDC callback — registered via `LiftRules.dispatch.append` | **Lift only — blocked on a portal-session decision.** 3 callback routes (`/auth/openid-connect/callback`, `…/callback-1`, `…/callback-2`) all funnel into `callbackUrlCommonCode`, whose success branch calls `AuthUser.logUserIn(user, () => S.redirectTo(...))`. `logUserIn` is inherited from `MetaMegaProtoUser` and writes the logged-in user into Lift `SessionVar`s that the portal reads; `S.redirectTo` sets Lift's session cookie. No tests cover the callback success path. Three forks: (a) **Drop portal-login** — pure http4s callback that issues a token but doesn't seed a portal session. Behaviour change for anyone using OIDC to sign into the portal UI; cheap if that user is nobody, surprising if it isn't. Needs a stakeholder check. (b) **Lift-session shim** — keep `lift-webkit` forever for this one callback. Cheapest in code, but "Lift Web removed" never actually ships. (c) **Replace portal session entirely** (e.g. Redis/JWT-backed). Months of work; also decouples session storage from Lift, which makes the lift-mapper conversation easier later. | - -DirectLogin's request-path is now off Lift. `OAuth2Login`, `GatewayLogin`, and `DAuth` turned out to be library-only (no routes); their vestigial `extends RestHelper` mixins were dropped. OAuth 1.0a was removed entirely in commit `51820c75e`. OpenIdConnect remains on Lift pending a portal-session decision — it is now the **only** auth handler still blocking bridge removal. - -### Resource-docs workstream - -Already partly described in the next major section, but counted here for completeness: - -- `ResourceDocs140` … `ResourceDocs600` — six separate Lift files, each registered via `LiftRules.statelessDispatch.append` in `Boot.scala`. -- `getResourceDocsObpV700` aggregation bug fix — landed (`V7ResourceDocsAggregationTest` passes). -- `openapi.yaml` route — raw `Lift serve { ... }` block, no native http4s handler. -- ~~`getMessageDocsSwagger` (v3.1.0) — folds into the centralised `Http4sResourceDocs` service when it ships.~~ **Done** — served by `Http4sResourceDocs.handleGetMessageDocsSwagger` via the wildcard `/obp/*/message-docs/{CONNECTOR}/swagger2.0` route. -- One-PR opportunity: build `Http4sResourceDocs` above the Lift bridge in `Http4sApp`, intercept all `/obp/*/resource-docs/*` traffic, retire six Lift dispatch entries in a single change. - -### Small singleton Lift endpoints - -| Endpoint | File | Notes | -|---|---|---| -| `aliveCheck` | `code/api/aliveCheck.scala` → `code/api/AliveCheckRoutes.scala` | **Done.** Native http4s route serves `GET /alive`; `LiftRules.statelessDispatch.append(aliveCheck)` removed from `Boot.scala`. | -| `ImporterAPI` | (deleted) | **Retired.** The legacy `POST /obp_transactions_saver/api/transactions` shared-secret bulk-insert endpoint, its `TransactionInserter` LiftActor, and the connector helpers it relied on (`createImportedTransaction`, `getMatchingTransactionCount`, `updateAccountBalance`, `setBankAccountLastUpdated`) have been removed entirely. Modern callers use connector-driven flows or the `/obp/vX.X.X/transaction-requests/...` endpoints. | -| `OpenIdConnect` | (auth-stack table above) | OIDC callback, registered separately from OAuth2. | - -### Open-banking standards - -Lift implementations of 3rd-party regulatory standards. Each is *not* OBP API per se but an optional regulatory shim. Reflection via `ClassScanUtils.getSubTypeObjects` is what registers them with Lift's dispatch — so commenting out the source removes them from the registry without touching `Boot.scala`. - -Three forks for how the still-active workstream resolves: - -- **(a) Migrate each to http4s.** Weeks per standard. Highest cost; cleanest end state. -- **(b) "Regulatory mode" feature-flagged Lift.** Keep `Http4sLiftWebBridge` wired in only when an `obf-*` / standards prop is set; otherwise the bridge is unregistered at boot. Lets "Lift Web removed from the OBP API path" ship, but Lift Web stays in the codebase as an opt-in shim. Defeats the milestone technically; ships the headline. -- **(c) Extract as plugin projects.** Move each standard out of this repo into its own project that depends on OBP API. Probably right long-term — these are optional, externally-governed standards on different release cadences — but socially expensive and reshapes the build. - -| Standard | Files / location | Status | -|---|---|---| -| Berlin Group v1.3 | `code/api/berlin/group/v1_3/*` — 7 files (AIS / PIS / PIIS / signing baskets / common) | Lift | -| **Berlin Group v2** | `code/api/berlin/group/v2/Http4sBGv2.scala` | ✅ already on http4s | -| **UK Open Banking v2.0.0 + v3.1.0** | `code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200*.scala`, `code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310*.scala` | ✅ **Done** — migrated to native http4s in PR #2817 (merged to `develop` 2026-05-29). All endpoints (v2.0: 5, v3.1: ~67 across 20 categories) on http4s, wired into `Http4sApp.baseServices` ahead of the Lift bridge. Lift `OBP_UKOpenBanking_{200,310}` reduced to stubs (`routes = Nil`), original Lift code kept as comments. `ResourceDocsAPIMethods.activeResourceDocs` skips the Lift-route filter for both versions. Verified by `UKOpenBankingV200Tests` + `UKOpenBankingV310{Ais,Pis}Tests` (142 scenarios, all passing). | -| ~~Bahrain OBF v1.0.0~~ | `code/api/BahrainOBF/*` | ✅ **Retired** — commented out in PR #2814 (`d19af2b92`, 2026-05-22). ScannedApis reflection no longer finds it; URLs 404 with no bridge handling. | -| ~~AU OpenBanking v1.0.0~~ | `code/api/AUOpenBanking/*` | ✅ **Retired** — same PR / commit. | -| ~~STET v1.4~~ | `code/api/STET/v1_4/*` | ✅ **Retired** — same PR / commit. | -| ~~MxOF / CNBV9 v1.0.0~~ | `code/api/MxOF/*` | ✅ **Retired** — same PR / commit. | -| ~~Polish v2.1.1.1~~ | `code/api/Polish/v2_1_1_1/*` | ✅ **Retired** — same PR / commit. | -| Sandbox / `SandboxApiCalls.scala` | `code/api/sandbox/*` | Lift | - -The retired five total ~22,000 lines of Scala kept in-tree as line-comments. The same `// ` stub pattern used for the per-version OBPAPI files; uncomment to bring them back. After ~1 month with no operator complaints and the bridge audit showing zero hits for their URLs, candidates for outright deletion. - -A regression-guard test (`code.api.util.http4s.RetiredApiStandardsTest`) asserts that `ClassScanUtils.getSubTypeObjects` does **not** return any object whose package matches the retired standards — so a partial uncomment that re-registers them with Lift trips CI. - -### `Boot.scala` scaffolding - -Currently runs on startup and goes away once the Lift bridge is removable: - -1. `LiftRules.statelessDispatch.append(...)` registrations: `DirectLogin`, `ResourceDocs140`–`ResourceDocs600`, `aliveCheck`. -2. `LiftRules.dispatch.append(OpenIdConnect)`. -3. `LiftRules.addToPackages("code")` — Lift package scanner. -4. `LiftRules.exceptionHandler.prepend { ... }` — global exception handler. -5. `LiftRules.uriNotFound.prepend { ... }` — 404 handler. -6. `LiftRules.early`, `LiftRules.supplementalHeaders`, `LiftRules.localeCalculator`, etc. — request-path hooks. -7. `LiftRules.unloadHooks.append(...)` — shutdown hooks (DB pool, Redis). -8. **Mapper schemifier** — DB schema init. Belongs to the long-term `lift-mapper` removal effort, not the bridge milestone. - -Everything in lines 1–7 is request-path-related and will go in the bridge-removal PR. Line 8 stays until lift-mapper is replaced. - -### Tests +## Risks -| Item | Status | +| Risk | Mitigation | |---|---| -| `Http4s500RoutesTest`, `RootAndBanksTest`, `V500ContractParityTest` | `@Ignore`. | -| `CardTest` | Commented out. | -| v5.0.0: 13 skipped tests | Setup cost paid, no value. | -| `V7ResourceDocsAggregationTest` | Was intentionally failing; aggregation bug fix landed → now passes. | -| `AbacRuleTests` (6 local fails) | Environment-dependent — too few users in local DB triggers `isStatisticallyTooPermissive`. Not a regression. | - -### Reusable lessons from v6.0.0 - -1. **JVM 64KB `` limit** — see CLAUDE.md. Adopt `lazy val xxx: HttpRoutes[IO] = ...` plus `private def initXxxResourceDocs(): Unit` blocks in every per-version file from the start; don't wait until you hit the wall. -2. **DirectLogin pattern** — `S.request`-bound Lift handlers need an http4s-friendly entry point that accepts pre-parsed parameters. `validatorFutureWithParams` is the model; replicate this for `GatewayLogin` / `OAuth` when their migration starts. -3. **`Future.failed(new Exception)` produces 500** — use `unboxFullOrFail(Empty, ..., 400)` or `NewStyle.function.tryons(msg, 400, ...)` to return the intended 4xx. Pattern showed up in WebUiProps and RetailCustomer fixes. -4. **`isStatisticallyTooPermissive` is sample-pool-dependent** — locally, a fresh test DB with a single user causes spurious rejections. Tests built against this check must seed enough users. -5. **Reserved ALL_CAPS placeholders** in middleware (`BANK_ID`, `ACCOUNT_ID`, `VIEW_ID`, `COUNTERPARTY_ID`) — when an endpoint needs a same-shape var without middleware lookup, rename to a non-reserved variant (e.g. `COUNTERPARTY_ID_PARAM`) in both the http4s and Lift ResourceDocs. +| `FrozenClassTest` ratchet — deleting any Lift `lazy val ... : OBPEndpoint` shrinks the STABLE surface and trips the test | Refresh the frozen snapshot inside the bridge-removal PR; list each removed `lazy val` in the PR body. | +| OIDC callback success path has no tests | Write a Keycloak-container integration test before picking a fork. | +| `S.request` translation — handlers re-read `S.request`/`S.param` invisibly (bit DirectLogin's `createTokenFuture`) | Audit for `S.request`/`S.param`/`S.queryString` before designing the http4s entry; replicate the `validatorFutureWithParams` pattern. | +| Bridge-cascade hijack on partial migrations (see CLAUDE.md) | When wiring a new `Http4sXxx` into `baseServices`, migrate its URL+verb overrides vs older versions first. | +| `isStatisticallyTooPermissive` flakiness — too few local users | Seed enough users in any ABAC test. | -### Decision gates - -Two non-engineering decisions must land before the bridge-removal PR can be cut. They are stakeholder calls, not author calls — making either of them in code reviews tends to surface objections after the fact. - -1. **OIDC portal-session strategy** (auth-stack OpenIdConnect row, options a/b/c). Until one of the three forks is picked, the OIDC callback can't be migrated and the bridge can't be removed. The cheapest option (drop portal-login) is a behaviour change and needs explicit sign-off from anyone using OIDC as a portal-UI sign-in. **This is now the only auth-handler decision blocking bridge removal.** - -A second decision is *not* required for bridge removal, but is required for the public claim that follows it: - -2. **Open-banking standards strategy** (forks a/b/c above). If "Lift Web removed" is the headline, fork (b) is acceptable. If "Lift Web removed *from this repo*" is the headline, only (a) or (c) qualify. Cf. the "Lift Web removed vs. Lift removed" note under Done Criteria. - -### Suggested ordering for the remaining work - -1. ~~**v4.0.0 bulk port**~~ — done (258/258, 100%). -2. ~~**DirectLogin**~~ — done. `code.api.DirectLoginRoutes` serves the bare `/my/logins/direct`; per-version paths served by their own `Http4sXxx`. `LiftRules.statelessDispatch.append(DirectLogin)` retired. -3. ~~**`aliveCheck`**~~ — done. `code.api.AliveCheckRoutes` serves `GET /alive`; Lift dispatch retired. **`ImporterAPI`** — retired entirely (no http4s replacement); the legacy shared-secret bulk-transaction-importer endpoint has been removed along with `TransactionInserter` and the connector helpers it relied on. -4. ~~**`Http4sResourceDocs` centralised service**~~ — done. `code.api.util.http4s.Http4sResourceDocs` serves `/obp/*/resource-docs/{API_VERSION}/{obp,swagger,openapi,openapi.yaml}`, `/obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/obp`, and `/obp/*/message-docs/{CONNECTOR}/swagger2.0`. 10 `LiftRules.statelessDispatch.append(ResourceDocs140..600)` retired + `openapi.yaml` raw `serve { ... }` block removed. ResourceDocsTest (63) + SwaggerDocsTest (10) green. -5. **Auth stack: OAuth2 / GatewayLogin / DAuth** — done. All three turned out to be library-only token validators (no `serve` blocks, no `LiftRules` registration). Vestigial `extends RestHelper` mixins removed. -6. **OpenIdConnect** — the only remaining auth-stack work. Blocked on a portal-session decision (its success path calls `AuthUser.logUserIn` / `S.redirectTo`, which mutate Lift `SessionVar`s — see auth-stack table). OAuth 1.0a was removed entirely in commit `51820c75e`; no migration needed. -7. **Bridge-removal PR** — delete `Http4sLiftWebBridge` + the request-path entries from `Boot.scala` (lines 1–7 above). -8. **Open-banking standards** — Berlin Group v2 and UK Open Banking v2.0/v3.1 now on http4s (the latter in PR #2817); five standards retired (Bahrain, AU, STET, MxOF, Polish). Remaining on Lift: **Berlin Group v1.3** and **Sandbox** — decide whether to migrate or keep a thin Lift remnant. -9. **`lift-mapper`** — separate long-term effort, out of scope here. - ---- - -## Server Chain After Full Migration - -``` -corsHandler - → Http4sResourceDocs (/obp/*/resource-docs/*) ← centralized, all version prefixes - → Http4s700 (/obp/v7.0.0/*) - → Http4s600 (/obp/v6.0.0/*) - → Http4s510 (/obp/v5.1.0/*) - → Http4s500 (/obp/v5.0.0/*) - → Http4s400 (/obp/v4.0.0/*) - → Http4s310 (/obp/v3.1.0/*) - → Http4s300 (/obp/v3.0.0/*) - → Http4s220 (/obp/v2.2.0/*) - → Http4s210 (/obp/v2.1.0/*) - → Http4s200 (/obp/v2.0.0/*) - → Http4s140 (/obp/v1.4.0/*) ← done - → Http4s130 (/obp/v1.3.0/*) ← done - → Http4s121 (/obp/v1.2.1/*) ← done - → Http4sBGv2 - ← Lift bridge removed -``` - ---- - -## Done Criteria +## Done criteria | Milestone | Condition | |---|---| -| Version file done | All *functional* endpoints are `HttpRoutes[IO]`; the version's test suite is green. Endpoints folded into the Resource-docs / Auth-stack workstreams or marked as non-functional stubs are listed in "Per-version Lift leftovers" rather than blocking the file's done status. | -| Lift bridge removable | All 12 APIMethods files done (per the row above) + auth stack done + Resource-docs workstream done. Any remaining stubs from "Per-version Lift leftovers" are ported or deleted in the bridge-removal PR. | -| Lift Web removed | `lift-webkit` removed from `pom.xml`; `Boot.scala` reduced to DB init + scheduler startup. | -| `lift-mapper` | Separate long-term effort — not in scope here. | - -**"Lift Web removed" ≠ "Lift removed."** The two are distinct milestones and the difference matters for public claims: - -- *Lift Web removed* means the HTTP request path no longer touches Lift — `lift-webkit` is out of `pom.xml`, `Http4sLiftWebBridge` is deleted, `Boot.scala` request-path hooks are gone. `lift-mapper` is still present and still the ORM. -- *Lift removed* means `net.liftweb:*` is fully out of the dependency graph — requires the multi-month `lift-mapper` replacement (Doobie/Slick or similar). +| Version file done | All functional endpoints are `HttpRoutes[IO]`; suite green. Workstream-owned stubs (resource-docs / auth / leftovers) don't block. | +| Lift bridge removable | All 12 APIMethods done + auth stack done + resource-docs done + dynamic-endpoint ported; remaining stubs deleted in the bridge-removal PR. | +| Lift Web removed | `lift-webkit` out of `pom.xml`; `Http4sLiftWebBridge` deleted; `Boot.scala` reduced to DB init + scheduler. | +| Lift removed | `net.liftweb:*` fully out — requires the multi-month `lift-mapper` replacement (Doobie/Slick or similar). | -Decide which bar a release is hitting before announcing it; conflating them invites either an overstatement or an avoidable months-long delay before the announcement. - ---- - -## Risks - -Things that can derail the remaining workstreams. The facts behind each are documented in the relevant section above; collected here so the bridge-removal PR author doesn't have to rediscover them. - -| Risk | Detail | Mitigation | -|---|---|---| -| `FrozenClassTest` ratchet | Every deletion of a Lift `lazy val ... : OBPEndpoint` reduces the STABLE-API surface and trips `FrozenClassTest`. The v3.1.0 leftovers (`getMessageDocsSwagger`, `getObpConnectorLoopback`) are deferred to the bridge-removal PR specifically because of this; subsequent ports may surface more. | Plan the frozen-snapshot refresh as part of the bridge-removal PR, not as a follow-up. Document each removed `lazy val` in the PR description. | -| OIDC callback success path has no tests | Whichever of the three OIDC forks ships, there is no automated safety net. Manual integration test against a real OIDC provider is the only verification. | Before picking a fork, write at least one integration test against a test OIDC provider (Keycloak in a container is the established pattern in this repo). | -| `S.request` translation gotchas | DirectLogin's `createTokenFuture` ignored its parameters and re-read from `S.request` via `getAllParameters`; the http4s migration needed `validatorFutureWithParams` to thread parsed params through. If a future auth/handshake handler is migrated (e.g. OIDC's callback), expect the same shape — its handlers will reference `S.request` in ways the existing function signatures hide. | Audit the handler for `S.request`/`S.param`/`S.queryString` reads before designing the http4s entry point. Replicate the DirectLogin pattern. | -| Bridge-cascade hijack on partial migrations | Documented in CLAUDE.md. Surfaced once during v4 migration; can resurface anywhere a new version is wired into the chain before its overrides are migrated. | When adding a new `Http4sXxx` to `baseServices`, audit URL+verb overrides against older versions first. | -| `isStatisticallyTooPermissive` flakiness | Local test DB with too few users trips the ABAC permissiveness check. Not a regression, but easy to misdiagnose during the bridge-removal PR's full test run. | Seed enough test users in any test that exercises ABAC rules. Document in the suite, not as a runtime mitigation. | - -## Why http4s? - -- **Non-blocking I/O** — Uses a small fixed thread pool (CPU cores) and suspends fibres on I/O. Thousands of concurrent requests without thread-pool tuning. -- **Lower memory** — No thread-per-request overhead. -- **Modern Scala ecosystem** — First-class Cats Effect, fs2 streaming, and functional patterns. -- **No servlet container** — Removes Jetty and WAR packaging entirely. - ---- +**"Lift Web removed" ≠ "Lift removed".** The first means the HTTP path no longer touches Lift (lift-mapper still the ORM); the second means no `net.liftweb:*` at all. Decide which bar a release hits before announcing — conflating them invites an overstatement or an avoidable months-long delay. ## Running @@ -587,42 +117,4 @@ MAVEN_OPTS="-Xms3G -Xmx6G -XX:MaxMetaspaceSize=2G" \ mvn -pl obp-api -am clean package -DskipTests=true -Dmaven.test.skip=true && \ java -jar obp-api/target/obp-api.jar ``` - -Binds to `hostname` / `dev.port` from your props file (defaults: `127.0.0.1:8080`). - ---- - -## Progress - -| File | Status | -|---|---| -| `APIMethods121` | done — `Http4s121.scala` (all 323 API1_2_1Test scenarios pass) | -| `APIMethods130` | done — `Http4s130.scala` (2 PhysicalCardsTest scenarios pass) | -| `APIMethods140` | done — `Http4s140.scala` (all 11 own endpoints; path-rewriting bridge to Http4s130) | -| `APIMethods200` | done — `Http4s200.scala` (37 own endpoints; path-rewriting bridge to Http4s140) | -| `APIMethods210` | done — `Http4s210.scala` (25 own endpoints; path-rewriting bridge to Http4s200) | -| `APIMethods220` | done — `Http4s220.scala` (18 own endpoints; path-rewriting bridge to Http4s210) | -| `APIMethods300` | done — `Http4s300.scala` (47 own endpoints; path-rewriting bridge to Http4s220; all 86 v3.0.0 tests pass) | -| `APIMethods310` | done — `Http4s310.scala` (100 own endpoints + `updateCustomerAddress`; path-rewriting bridge to Http4s300). Two former Lift leftovers now both off-bridge: `getMessageDocsSwagger` served by `Http4sResourceDocs` (in-file stub kept only for `FrozenClassTest`), `getObpConnectorLoopback` served by a native http4s route that returns 400 NotImplemented. | -| `APIMethods400` | **done — 258 / 258 (100%)**. `Http4s400.scala` covers all 253 unique handlers + 8 ResourceDoc aliases for transaction-request-type variants (served by the shared wildcard handler). | -| `APIMethods500` | done — `Http4s500.scala` (all 10 v5.0.0 originals on http4s) | -| `APIMethods510` | done — `Http4s510.scala` (all 111 v5.1.0 originals on http4s; `createConsent` exposed as `createConsentImplicit` with a guard covering EMAIL/SMS/IMPLICIT SCA methods) | -| `APIMethods600` | **done — 243 / 243 (100%)**. `Http4s600.scala` covers all 35 overrides + 208 originals. | -| Auth: DirectLogin | done — `code.api.DirectLoginRoutes` serves the bare `/my/logins/direct` (gated on `allow_direct_login`); per-version paths served by their own `Http4sXxx`; `LiftRules.statelessDispatch.append(DirectLogin)` removed from `Boot.scala` | -| Auth: GatewayLogin | done — library-only (no `serve` block, no `LiftRules` registration). Vestigial `extends RestHelper` removed. | -| Auth: DAuth | done — library-only (no `serve` block, no `LiftRules` registration). Vestigial `extends RestHelper` removed. | -| Auth: OAuth2 | done — library-only Bearer-token validator. Vestigial `extends RestHelper` removed. | -| Auth: OAuth 1.0a | done — removed entirely in commit `51820c75e` (2026-02-20). `oauth1.0.scala` deleted, `OAuthHandshake` unregistered from `Boot.scala`, header detection removed from `OBPRestHelper.scala`. See "Per-version Lift leftovers → Auth stack" for surviving dead-code references that are cleanup candidates. | -| Auth: OpenIdConnect | blocked — callback success path calls `AuthUser.logUserIn` / `S.redirectTo` (Lift `SessionVar`s). Needs a portal-session decision before migration. | -| Resource-docs: aggregation bug fix | done | -| Resource-docs: `Http4sResourceDocs` service | todo | -| Resource-docs: `openapi.yaml` route | todo | -| Std: UK Open Banking v2.0 + v3.1 | done — `Http4sUKOBv200*` / `Http4sUKOBv310*` (PR #2817, merged 2026-05-29). v2.0: 5 endpoints; v3.1: ~67 across 20 categories. Wired into `Http4sApp.baseServices` ahead of the Lift bridge; Lift `OBP_UKOpenBanking_{200,310}` are `routes = Nil` stubs. Verified by `UKOpenBankingV200Tests` + `UKOpenBankingV310{Ais,Pis}Tests` (142 scenarios pass). | -| Std: Berlin Group v2 | done — `code.api.berlin.group.v2.Http4sBGv2` (native http4s). | -| Std: Berlin Group v1.3, Sandbox | todo — still on Lift. | -| Dynamic-entity data plane | done — `code.api.dynamic.entity.Http4sDynamicEntity` serves `/obp/dynamic-entity/{entityName \| my/.. \| public/.. \| community/..}` (+ `banks/BANK_ID/..`) natively, wired into `Http4sApp.baseServices` via `gate(ApiVersion.\`dynamic-entity\`, ...)`. Reuses the `EntityName`/`PublicEntityName`/`CommunityEntityName` extractors + `NewStyle.invokeDynamicConnector`; mirrors the Lift `genericEndpoint`/`publicEndpoint`/`communityEndpoint` PFs. `OBPAPIDynamicEntity` stays registered on Lift as a dormant fallback (removed in the bridge-removal PR). | -| Dynamic-endpoint data plane | todo — Phase 3. Runtime Scala codegen (`DynamicEndpoints.scala`) compiles strings to Lift `OBPEndpoint`; still served by the bridge. 3a (keep Lift-typed codegen behind an adapter) recommended for the Lift-removal milestone; 3b (emit `HttpRoutes[IO]`) deferred. | - -### Cleanup done - -- `getCards` and `getCardsForBank` removed from `Http4s700` — these had the same API signature as the v1.3.0 originals and belonged in `APIMethods130`, not v7.0.0. The Lift implementation in `APIMethods130` serves them at `/obp/v1.3.0/` until that file is migrated. +Binds to `hostname` / `dev.port` from your props file (defaults `127.0.0.1:8080`). From dabbb376c074b89c2894baf51a46975089c4ac14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 29 May 2026 15:54:12 +0200 Subject: [PATCH 12/13] docs(migration): dynamic-endpoint now native (upstream merge); record sandbox classloader test issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dynamic-endpoint data plane: todo Phase 3 → done (fully-native http4s, off-bridge) - bridge-audit: no known real_work remains; OIDC is now the last code blocker - record DynamicResourceDocTest sandbox specifyStreamHandler failures (upstream-inherited, not the merge) --- LIFT_HTTP4S_MIGRATION.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/LIFT_HTTP4S_MIGRATION.md b/LIFT_HTTP4S_MIGRATION.md index 69ba7b129f..8e0b788f3c 100644 --- a/LIFT_HTTP4S_MIGRATION.md +++ b/LIFT_HTTP4S_MIGRATION.md @@ -52,7 +52,7 @@ All 12 APIMethods files **done** — every functional endpoint on http4s, test s | Auth: OAuth 1.0a | **done — removed** (`51820c75e`): `oauth1.0.scala` deleted, `OAuthHandshake` unregistered, OAuth header parsing + dead fields removed. | | Auth: OpenIdConnect | **blocked** — see Decision gates. The only auth handler still on Lift. | | Dynamic-entity data plane | **done** — `Http4sDynamicEntity` serves `/obp/dynamic-entity/*` natively (`OBPAPIDynamicEntity` stays as a dormant Lift fallback, removed in the bridge-removal PR). | -| Dynamic-endpoint data plane | **todo (Phase 3)** — runtime Scala codegen (`DynamicEndpoints.scala`) emits Lift `OBPEndpoint`; still on the bridge. 3a (keep Lift-typed codegen behind an adapter) recommended for the Lift-removal milestone; 3b (emit `HttpRoutes[IO]`) deferred. The only remaining `real_work` bridge traffic. | +| Dynamic-endpoint data plane | **done** — `Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint` serves `/obp/dynamic-endpoint/*` **fully native** (3b): proxy via `DynamicEndpointHelper.DynamicReq.resolveProxyTarget` + `APIMethodsDynamicEndpoint.proxyHandle`, and runtime-compiled docs via `ResourceDoc.dynamicHttp4sFunction` / `authCheckIO` — no Lift dispatch. Wired into `Http4sApp.baseServices` ahead of the bridge. Landed via the upstream merge (`74ead2134`, 2026-05-29). **Known test issue** (inherited from upstream, not the merge): 3 `DynamicResourceDocTest` runtime-compiled end-to-end scenarios fail locally — see test note below. | | Std: Berlin Group v2 | **done** — `Http4sBGv2`. | | Std: UK Open Banking v2.0 + v3.1 | **done** — PR #2817 (merged 2026-05-29). v2.0: 5; v3.1: ~67 / 20 categories; Lift aggregators are `routes = Nil` stubs. 142 test scenarios pass. | | Std: Berlin Group v1.3 | **todo** — 7 files still on active Lift (`code/api/berlin/group/v1_3/*`). | @@ -71,18 +71,17 @@ Restoration tools: `rehydrate_resource_docs.py` (descriptions / example bodies f ## Open TODOs — master list for "remove Lift Web" -**Bridge-traffic audit (data-driven prioritisation).** Every bridge hit is tallied by `Http4sLiftBridgeTraffic`; `GET /admin/lift-bridge-traffic` returns `real_work` (non-404 = migration targets) vs `not_found` (stale/probes, ignore); `POST .../reset` clears it. Playbook: reset on a representative instance → run a normal traffic window (24h + scheduled jobs) → if `real_work[]` is empty the bridge is retirable (modulo documented leftovers). Last CI audit: the only `real_work` left is `/obp/dynamic-endpoint/...` (Phase 3); `/obp/dynamic-entity/...` is now native. +**Bridge-traffic audit (data-driven prioritisation).** Every bridge hit is tallied by `Http4sLiftBridgeTraffic`; `GET /admin/lift-bridge-traffic` returns `real_work` (non-404 = migration targets) vs `not_found` (stale/probes, ignore); `POST .../reset` clears it. Playbook: reset on a representative instance → run a normal traffic window (24h + scheduled jobs) → if `real_work[]` is empty the bridge is retirable (modulo documented leftovers). Both dynamic-entity and dynamic-endpoint are now served natively, so **no known `real_work` remains** — re-run the audit on a representative instance to confirm before cutting the bridge-removal PR. -1. **Dynamic-endpoint Phase 3** — see workstream table. -2. **OpenIdConnect** — blocked on the OIDC portal-session decision (gate 1). -3. **Bridge-removal PR** — delete `Http4sLiftWebBridge` + the request-path `Boot.scala` hooks: the `LiftRules.statelessDispatch.append(...)` registrations (DirectLogin, ResourceDocs140–600, aliveCheck), `LiftRules.dispatch.append(OpenIdConnect)`, `addToPackages("code")`, the global exception + 404 handlers, the `early`/`supplementalHeaders`/`localeCalculator` request hooks, and `unloadHooks`. The Mapper schemifier stays (that's lift-mapper, not the bridge). Plan a `FrozenClassTest` snapshot refresh in the same PR. -4. **Open-banking standards** — decide BG v1.3's fate (gate 2). -5. **`lift-mapper`** — separate long-term ORM replacement; out of scope here. -6. **Misc**: OBP-Trading payment-auth endpoints (notifyDeposit, create/capture/release/getPaymentAuth) still commented out in `Http4s700` (see `ideas/CAPTURE_RELEASE_TRANSACTION_REQUEST_TYPES.md`); CI speed-up (two-tier fast gate + surefire parallel forks) not done. +1. **OpenIdConnect** — blocked on the OIDC portal-session decision (gate 1). With dynamic-endpoint now native, this is the **last code blocker** for bridge removal. +2. **Bridge-removal PR** — delete `Http4sLiftWebBridge` + the request-path `Boot.scala` hooks: the `LiftRules.statelessDispatch.append(...)` registrations (DirectLogin, ResourceDocs140–600, aliveCheck), `LiftRules.dispatch.append(OpenIdConnect)`, `addToPackages("code")`, the global exception + 404 handlers, the `early`/`supplementalHeaders`/`localeCalculator` request hooks, and `unloadHooks`. The Mapper schemifier stays (that's lift-mapper, not the bridge). Plan a `FrozenClassTest` snapshot refresh in the same PR. +3. **Open-banking standards** — decide BG v1.3's fate (gate 2). +4. **`lift-mapper`** — separate long-term ORM replacement; out of scope here. +5. **Misc**: OBP-Trading payment-auth endpoints (notifyDeposit, create/capture/release/getPaymentAuth) still commented out in `Http4s700` (see `ideas/CAPTURE_RELEASE_TRANSACTION_REQUEST_TYPES.md`); CI speed-up (two-tier fast gate + surefire parallel forks) not done. **Small singletons:** `aliveCheck` **done** (`AliveCheckRoutes`, `GET /alive`); `ImporterAPI` **retired** (endpoint + `TransactionInserter` + connector helpers removed). -**Disabled / ignored tests to revisit:** `Http4s500RoutesTest`, `RootAndBanksTest`, `V500ContractParityTest` (`@Ignore`); `CardTest` (commented out); v5.0.0 13 skipped; `AbacRuleTests` 6 local fails are environment-dependent (too few users → `isStatisticallyTooPermissive`), not a regression. The `MakerCheckerTransactionRequestTest` proxy/TTL race is **resolved** by the `RequestScopeConnection` hardening on `develop` (regression-guarded by its "Stress: repeated multi-challenge creates" scenario); if it ever flakes again, route DB Futures through `RequestScopeConnection.fromFuture`. +**Disabled / ignored tests to revisit:** `Http4s500RoutesTest`, `RootAndBanksTest`, `V500ContractParityTest` (`@Ignore`); `CardTest` (commented out); v5.0.0 13 skipped; `AbacRuleTests` 6 local fails are environment-dependent (too few users → `isStatisticallyTooPermissive`), not a regression. The `MakerCheckerTransactionRequestTest` proxy/TTL race is **resolved** by the `RequestScopeConnection` hardening on `develop` (regression-guarded by its "Stress: repeated multi-challenge creates" scenario); if it ever flakes again, route DB Futures through `RequestScopeConnection.fromFuture`. **`DynamicResourceDocTest`** — 3 runtime-compiled end-to-end scenarios (practise + no-role + role-gated `pieceC`) fail with `AccessControlException: specifyStreamHandler` because the compiled dynamic class is **lazily loaded inside `DynamicUtil$Sandbox.runInSandbox`** (which grants no `NetPermission`). Files are byte-identical to `upstream/develop`; not caused by our merge. Fix options: warm the compiled class *before* entering the sandbox, or add `specifyStreamHandler` to the sandbox permission set. Confirm against upstream CI before treating as a real regression vs. a warm-classloader timing artifact. ## Decision gates (stakeholder calls, before the bridge-removal PR) From 596206ac0c18db6c67aabe57413a67ee4b078f76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 2 Jun 2026 07:29:38 +0200 Subject: [PATCH 13/13] fix(resource-docs): restore self-documenting ResourceDoc entries in the docs list The resource-docs / swagger / openapi endpoints are served natively by Http4sResourceDocs, but their self-describing ResourceDoc entries were lost when serving moved Lift -> http4s: the Lift `localResourceDocs += ...` registrations were commented out and Http4sResourceDocs registered none, so `localResourceDocs` was empty. Since `getResourceDocsList` only appends `localResourceDocs` for the obp standard, "Get Resource Docs" and friends disappeared from the listing shown by API Explorer. Repopulate `localResourceDocs` with the four endpoints (getResourceDocsObp, getBankLevelDynamicResourceDocsObp, getResourceDocsSwagger, getResourceDocsOpenAPI31), restored verbatim from the pre-comment source so summaries/descriptions/tags/roles/error-lists match the originals. These are documentation-only entries (partialFunction = null, no http4sPartialFunction) because routing is handled upstream by Http4sResourceDocs. The duplicate getResourceDocsObpV400 registration removed in efb97531e is intentionally not reintroduced. Update the stale Http4s700 comment to reflect the registration. Verified: ResourceDocsTest (63) and V7ResourceDocsAggregationTest (14) pass. --- .../ResourceDocsAPIMethods.scala | 214 ++++++++++++++++++ .../scala/code/api/v7_0_0/Http4s700.scala | 7 +- 2 files changed, 218 insertions(+), 3 deletions(-) diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index a81e267710..102b8cf1e9 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -74,6 +74,220 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth val implementedInApiVersion = ApiVersion.v1_4_0 + // The resource-docs / swagger / openapi endpoints are served natively by + // `code.api.util.http4s.Http4sResourceDocs` (wired into `Http4sApp.baseServices` ahead of the + // per-version routes). Their Lift `lazy val` handlers below are commented-out dead code, but the + // self-documenting ResourceDoc entries must still be published so they appear in the resource-docs + // listing (API Explorer reads this list). `getResourceDocsList` appends `localResourceDocs` to the + // aggregated docs for the `obp` standard, so registering them here is all that is required — + // `partialFunction` is null and no `http4sPartialFunction` is set because the routing is external. + def getResourceDocsDescription(isBankLevelResourceDoc: Boolean) = { + + val endpointBankIdPath = if (isBankLevelResourceDoc) "/banks/BANK_ID" else "" + + s"""Get documentation about the RESTful resources on this server including example bodies for POST and PUT requests. + | + |This is the native data format used to document OBP endpoints. Each endpoint has a Resource Doc (a Scala case class) defined in the source code. + | + | This endpoint is used by OBP API Explorer to display and work with the API documentation. + | + | Most (but not all) fields are also available in swagger format. (The Swagger endpoint is built from Resource Docs.) + | + | API_VERSION is the version you want documentation about e.g. v3.0.0 + | + | You may filter this endpoint with tags parameter e.g. ?tags=Account,Bank + | + | You may filter this endpoint with functions parameter e.g. ?functions=enableDisableConsumers,getConnectorMetrics + | + | For possible function values, see implemented_by.function in the JSON returned by this endpoint or the OBP source code or the footer of the API Explorer which produces a comma separated list of functions that reflect the server or filtering by API Explorer based on tags etc. + | + | You may filter this endpoint using the 'content' url parameter, e.g. ?content=dynamic + | if set content=dynamic, only show dynamic endpoints, if content=static, only show the static endpoints. if omit this parameter, we will show all the endpoints. + | + | You may need some other language resource docs, now we support en_GB and es_ES at the moment. + | + | You can filter with api-collection-id, but api-collection-id can not be used with others together. If api-collection-id is used in URL, it will ignore all other parameters. + | + |See the Resource Doc endpoint for more information. + | + |Note: Dynamic Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds + | Static Resource Docs are cached, TTL is ${GET_STATIC_RESOURCE_DOCS_TTL} seconds + | + | + |Following are more examples: + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?tags=Account,Bank + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?functions=getBanks,bankById + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?locale=es_ES + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?content=static,dynamic,all + |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?api-collection-id=4e866c86-60c3-4268-a221-cb0bbf1ad221 + | + |
    + |
  • operation_id is concatenation of "v", version and function and should be unique (used for DOM element IDs etc. maybe used to link to source code)
    • + |
  • version references the version that the API call is defined in.
    • + |
  • function is the (scala) partial function that implements this endpoint. It is unique per version of the API.
    • + |
  • request_url is empty for the root call, else the path. It contains the standard prefix (e.g. /obp) and the implemented version (the version where this endpoint was defined) e.g. /obp/v1.2.0/resource
    • + |
  • specified_url (recommended to use) is empty for the root call, else the path. It contains the standard prefix (e.g. /obp) and the version specified in the call e.g. /obp/v3.1.0/resource. In OBP, endpoints are first made available at the request_url, but the same resource (function call) is often made available under later versions (specified_url). To access the latest version of all endpoints use the latest version available on your OBP instance e.g. /obp/v3.1.0 - To get the original version use the request_url. We recommend to use the specified_url since non semantic improvements are more likely to be applied to later implementations of the call.
    • + |
  • summary is a short description inline with the swagger terminology.
    • + |
  • description may contain html markup (generated from markdown on the server).
    • + |
+ """ + } + + localResourceDocs += ResourceDoc( + null, + implementedInApiVersion, + "getResourceDocsObp", + "GET", + "/resource-docs/API_VERSION/obp", + "Get Resource Docs.", + getResourceDocsDescription(false), + EmptyBody, + EmptyBody, + UnknownError :: Nil, + List(apiTagDocumentation, apiTagApi), + Some(List(canReadResourceDoc)) + ) + + localResourceDocs += ResourceDoc( + null, + implementedInApiVersion, + "getBankLevelDynamicResourceDocsObp", + "GET", + "/banks/BANK_ID/resource-docs/API_VERSION/obp", + "Get Bank Level Dynamic Resource Docs.", + getResourceDocsDescription(true), + EmptyBody, + EmptyBody, + UnknownError :: Nil, + List(apiTagDocumentation, apiTagApi), + Some(List(canReadDynamicResourceDocsAtOneBank)) + ) + + localResourceDocs += ResourceDoc( + null, + implementedInApiVersion, + "getResourceDocsSwagger", + "GET", + "/resource-docs/API_VERSION/swagger", + "Get Swagger documentation", + s"""Returns documentation about the RESTful resources on this server in Swagger format. + | + |API_VERSION is the version you want documentation about e.g. v3.0.0 + | + |You may filter this endpoint using the 'tags' url parameter e.g. ?tags=Account,Bank + | + |(All endpoints are given one or more tags which for used in grouping) + | + |You may filter this endpoint using the 'functions' url parameter e.g. ?functions=getBanks,bankById + | + |(Each endpoint is implemented in the OBP Scala code by a 'function') + | + |See the Resource Doc endpoint for more information. + | + | Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds + | + |Following are more examples: + |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger + |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger?tags=Account,Bank + |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger?functions=getBanks,bankById + |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger?tags=Account,Bank,PSD2&functions=getBanks,bankById + | + """, + EmptyBody, + EmptyBody, + UnknownError :: Nil, + List(apiTagDocumentation, apiTagApi) + ) + + localResourceDocs += ResourceDoc( + null, + implementedInApiVersion, + "getResourceDocsOpenAPI31", + "GET", + "/resource-docs/API_VERSION/openapi", + "Get OpenAPI 3.1 documentation", + s"""Returns documentation about the RESTful resources on this server in OpenAPI 3.1 format. + | + |API_VERSION is the version you want documentation about e.g. v6.0.0 + | + |## Query Parameters + | + |You may filter this endpoint using the following optional query parameters: + | + |**tags** - Filter by endpoint tags (comma-separated list) + | • Example: ?tags=Account,Bank or ?tags=Account-Firehose + | • All endpoints are given one or more tags which are used for grouping + | • Empty values will return error OBP-10053 + | + |**functions** - Filter by function names (comma-separated list) + | • Example: ?functions=getBanks,bankById + | • Each endpoint is implemented in the OBP Scala code by a 'function' + | • Empty values will return error OBP-10054 + | + |**content** - Filter by endpoint type + | • Values: static, dynamic, all (case-insensitive) + | • static: Only show static/core API endpoints + | • dynamic: Only show dynamic/custom endpoints + | • all: Show both static and dynamic endpoints (default) + | • Invalid values will return error OBP-10052 + | + |**locale** - Language for localized documentation + | • Example: ?locale=en_GB or ?locale=es_ES + | • Supported locales: en_GB, es_ES, ro_RO + | • Invalid locales will return error OBP-10041 + | + |**api-collection-id** - Filter by API collection UUID + | • Example: ?api-collection-id=4e866c86-60c3-4268-a221-cb0bbf1ad221 + | • Returns only endpoints belonging to the specified collection + | • Empty values will return error OBP-10055 + | + |This endpoint generates OpenAPI 3.1 compliant documentation with modern JSON Schema support. + | + |For YAML format, use the corresponding endpoint: /resource-docs/API_VERSION/openapi.yaml + | + |See the Resource Doc endpoint for more information. + | + |Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds + | + |## Examples + | + |Basic usage: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi + | + |Filter by tags: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?tags=Account,Bank + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?tags=Account-Firehose + | + |Filter by content type: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?content=static + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?content=dynamic + | + |Filter by functions: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?functions=getBanks,bankById + | + |Combine multiple parameters: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?content=static&tags=Account-Firehose + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?tags=Account,Bank,PSD2&functions=getBanks,bankById + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?content=static&locale=en_GB&tags=Account + | + |Filter by API collection: + |${getObpApiRoot}/v6.0.0/resource-docs/v6.0.0/openapi?api-collection-id=4e866c86-60c3-4268-a221-cb0bbf1ad221 + | + """, + EmptyBody, + EmptyBody, + InvalidApiVersionString :: + ApiVersionNotSupported :: + InvalidLocale :: + InvalidContentParameter :: + InvalidTagsParameter :: + InvalidFunctionsParameter :: + InvalidApiCollectionIdParameter :: + UnknownError :: Nil, + List(apiTagDocumentation, apiTagApi) + ) + implicit val formats = CustomJsonFormats.rolesMappedToClassesFormats // avoid repeat execute method getSpecialInstructions, here save the calculate results. diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index dfc9528fce..0b3ae27ef0 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -237,9 +237,10 @@ object Http4s700 { // Note: resource-docs requests (`GET /obp/v7.0.0/resource-docs/...`) are intercepted by // `Http4sResourceDocs.routes`, which is registered earlier in `Http4sApp.baseServices` - // (line 109, ahead of `v700Routes` at line 113). The ResourceDoc metadata for that URL - // is contributed by `ResourceDocs1_4_0.ResourceDocsAPIMethods.localResourceDocs` and - // surfaces through `getResourceDocsList`'s localResourceDocs append for the obp standard. + // (line 109, ahead of `v700Routes` at line 113). The self-documenting ResourceDoc metadata + // for those URLs is registered in `ResourceDocs1_4_0.ResourceDocsAPIMethods.localResourceDocs` + // (getResourceDocsObp / Swagger / OpenAPI31 / bank-level) and surfaces through + // `getResourceDocsList`'s localResourceDocs append for the obp standard. // There is intentionally no v7-specific handler here. // ── POC endpoints — one per EndpointHelper category ────────────────────