diff --git a/tilemaker/client/simple.py b/tilemaker/client/simple.py index 8078d5b..8d90306 100644 --- a/tilemaker/client/simple.py +++ b/tilemaker/client/simple.py @@ -15,6 +15,7 @@ def create_sample_metadata(filename: str): return [ MapGroup( + map_group_id="example-map-group", name="Map Group", description="Example", maps=[ diff --git a/tilemaker/metadata/core.py b/tilemaker/metadata/core.py index 7ed2eae..1fa1022 100644 --- a/tilemaker/metadata/core.py +++ b/tilemaker/metadata/core.py @@ -6,7 +6,7 @@ from pydantic import BaseModel from .boxes import Box -from .definitions import Layer, MapGroup +from .definitions import Band, Layer, MapGroup from .sources import SourceGroup @@ -22,6 +22,59 @@ def merge(self, other: "DataConfiguration") -> "DataConfiguration": source_groups=self.source_groups + other.source_groups, ) + def _match(self, name: str, query: str) -> bool: + return query.lower() in name.lower() + + def filter_map_groups(self, authorized_map_groups: list, query: str) -> dict: + matched_ids: set[str] = set() + filtered_groups = [] + + for group in authorized_map_groups: + # Group name matches — keep entire subtree intact + if self._match(group["name"], query): + matched_ids.add(group["map_group_id"]) + filtered_groups.append(group) + continue + + filtered_maps = [] + for map in group.get("maps", []): + # Map name matches — keep entire subtree intact + if self._match(map["name"], query): + matched_ids.add(map["map_id"]) + filtered_maps.append(map) + continue + + filtered_bands = [] + for band in map.get("bands", []): + # Band name matches — keep entire subtree intact + if self._match(band["name"], query): + matched_ids.add(band["band_id"]) + filtered_bands.append(band) + continue + + filtered_layers = [ + layer + for layer in band.get("layers", []) + if self._match(layer["name"], query) + and matched_ids.add(layer["layer_id"]) is None + ] + if filtered_layers: + filtered_bands.append({**band, "layers": filtered_layers}) + + if filtered_bands: + filtered_maps.append({**map, "bands": filtered_bands}) + + if filtered_maps: + filtered_groups.append({**group, "maps": filtered_maps}) + + return {"filtered_map_groups": filtered_groups, "matched_ids": matched_ids} + + @property + def bands(self) -> Iterable[Band]: + return itertools.chain.from_iterable( + map.bands for group in self.map_groups for map in group.maps + ) + @property def layers(self) -> Iterable[Layer]: return itertools.chain.from_iterable( diff --git a/tilemaker/metadata/database.py b/tilemaker/metadata/database.py index a5fa531..10ac2c2 100644 --- a/tilemaker/metadata/database.py +++ b/tilemaker/metadata/database.py @@ -61,6 +61,13 @@ def create_tables(self): """Create all tables in the database.""" Base.metadata.create_all(self.engine) + """ + TODO: Use database queries to filter map groups based on query string + """ + + def filter_map_groups(self, authorized_map_groups: list, query: str) -> dict: + return {"filtered_map_groups": [], "matched_ids": []} + @property def map_groups(self) -> list[MapGroup]: """Retrieve all map groups from the database.""" @@ -87,6 +94,13 @@ def source_groups(self) -> list[SourceGroup]: for orm_group in orm_groups ] + @property + def bands(self) -> Iterable[Band]: + """Retrieve all bands from the database.""" + return itertools.chain.from_iterable( + map.bands for group in self.map_groups for map in group.maps + ) + @property def layers(self) -> Iterable[Layer]: """Retrieve all layers from the database.""" @@ -262,6 +276,7 @@ def _orm_to_map_group(self, session: Session, orm_group: MapGroupORM) -> MapGrou description=orm_group.description, maps=maps, grant=orm_group.grant, + map_group_id=orm_group.map_group_id, ) def populate_from_config(self, config: "DataConfiguration") -> None: @@ -286,6 +301,7 @@ def populate_from_config(self, config: "DataConfiguration") -> None: name=map_group.name, description=map_group.description, grant=map_group.grant, + map_group_id=map_group.map_group_id, ) session.add(orm_group) session.flush() diff --git a/tilemaker/metadata/definitions.py b/tilemaker/metadata/definitions.py index 6da2f31..aabf6f4 100644 --- a/tilemaker/metadata/definitions.py +++ b/tilemaker/metadata/definitions.py @@ -81,11 +81,13 @@ def auth(self, grants: set[str]): return self.grant is None or self.grant in grants -class Layer(AuthenticatedModel): +class LayerSummary(AuthenticatedModel): layer_id: str name: str description: str | None = None + +class Layer(LayerSummary): provider: FITSLayerProvider | FITSCombinationLayerProvider bounding_left: float | None = None @@ -115,26 +117,47 @@ def model_post_init(self, _): self.tile_size, self.number_of_levels = self.provider.calculate_tile_size() -class Band(AuthenticatedModel): +class LayerWithMenuState(Layer): + map_group_id: str + map_id: str + band_id: str + + +class BandBase(AuthenticatedModel): band_id: str name: str description: str + +class Band(BandBase): layers: list[Layer] -class Map(AuthenticatedModel): +class BandMenuState(BandBase): + layers: list[LayerSummary] + + +class MapBase(AuthenticatedModel): map_id: str name: str description: str + +class Map(MapBase): bands: list[Band] -class MapGroup(AuthenticatedModel): +class MapMenuState(MapBase): + bands: list[BandMenuState] + + +class MapGroupBase(AuthenticatedModel): + map_group_id: str name: str description: str + +class MapGroup(MapGroupBase): maps: list[Map] def get_layer(self, layer_id: str) -> Layer | None: @@ -145,3 +168,20 @@ def get_layer(self, layer_id: str) -> Layer | None: return layer return None + + +class MapGroupMenuState(MapGroupBase): + maps: list[MapMenuState] + + +class LayerDefault(AuthenticatedModel): + layer: Layer | None + default_layer_menu: list[MapGroupMenuState] + default_map_group_id: str | None + default_map_id: str | None + default_band_id: str | None + + +class SearchResponse(AuthenticatedModel): + filtered_layer_menu: list[MapGroupMenuState] + matched_ids: list[str] diff --git a/tilemaker/metadata/generation.py b/tilemaker/metadata/generation.py index 5567253..fe4e59c 100644 --- a/tilemaker/metadata/generation.py +++ b/tilemaker/metadata/generation.py @@ -3,6 +3,7 @@ """ import os +import uuid from hashlib import md5 from pathlib import Path from typing import Any, Literal @@ -76,7 +77,10 @@ def map_group_from_fits( ) return MapGroup( - name="Auto-Populated", description="No description provided", maps=maps + map_group_id=f"map-group-{uuid.uuid4()}", + name="Auto-Populated", + description="No description provided", + maps=maps, ) diff --git a/tilemaker/metadata/orm.py b/tilemaker/metadata/orm.py index f84b6f1..0f81a40 100644 --- a/tilemaker/metadata/orm.py +++ b/tilemaker/metadata/orm.py @@ -21,6 +21,7 @@ class MapGroupORM(Base): __tablename__ = "map_groups" id = Column(Integer, primary_key=True) + map_group_id = Column(String, unique=True, nullable=False) name = Column(String, nullable=False) description = Column(String) grant = Column(String) diff --git a/tilemaker/server/app.py b/tilemaker/server/app.py index 4165d17..b10312f 100644 --- a/tilemaker/server/app.py +++ b/tilemaker/server/app.py @@ -12,9 +12,13 @@ from ..settings import settings from .analysis import analysis_router from .auth import setup_auth +from .bands import bands_router from .highlights import highlights_router from .histogram import histogram_router +from .layers import layers_router +from .map_groups import map_groups_router from .maps import maps_router +from .search import search_router from .sources import sources_router @@ -61,7 +65,11 @@ async def lifespan(app: FastAPI): app.include_router(highlights_router) app.include_router(histogram_router) app.include_router(sources_router) +app.include_router(map_groups_router) app.include_router(maps_router) +app.include_router(bands_router) +app.include_router(layers_router) +app.include_router(search_router) app.include_router(analysis_router) if settings.serve_frontend: diff --git a/tilemaker/server/bands.py b/tilemaker/server/bands.py new file mode 100644 index 0000000..bcb7b16 --- /dev/null +++ b/tilemaker/server/bands.py @@ -0,0 +1,32 @@ +""" +Endpoint for summary data of a band's layers +""" + +from fastapi import ( + APIRouter, + Request, +) + +from tilemaker.metadata.definitions import LayerSummary + +bands_router = APIRouter(prefix="/bands", tags=["List of Bands"]) + + +@bands_router.get( + "/{band_id}/layers", + response_model=list[LayerSummary], + summary="Get the list of layer summaries associated with a Band.", + description="Retrieve a list of LayerSummary objects that belong to a particular Band.", +) +def get_layer_summaries_of_band(band_id: str, request: Request): + layer_summaries = [] + for band in request.app.config.bands: + if band.band_id == band_id and band.auth(request.auth.scopes): + for layer in band.layers: + layer_summary = LayerSummary( + layer_id=layer.layer_id, + name=layer.name, + description=layer.description, + ) + layer_summaries.append(layer_summary) + return layer_summaries diff --git a/tilemaker/server/layers.py b/tilemaker/server/layers.py new file mode 100644 index 0000000..0e91d7e --- /dev/null +++ b/tilemaker/server/layers.py @@ -0,0 +1,283 @@ +""" +Endpoint for layer and tile data +""" + +import io +from typing import Literal + +from astropy.io import fits +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Request, + Response, +) + +from tilemaker.metadata.definitions import ( + BandMenuState, + LayerDefault, + LayerSummary, + LayerWithMenuState, + MapGroupMenuState, + MapMenuState, +) +from tilemaker.processing.extractor import extract +from tilemaker.providers.fits import PullableTile + +from ..processing.renderer import Renderer, RenderOptions + +renderer = Renderer(format="webp") + +layers_router = APIRouter(prefix="/layers", tags=["Layers and Tiles"]) + + +""" + Sets default layer to the first layer found after filtering for auth status, i.e. + auth_map_groups[0].maps[0].bands[0].layers[0]. Also creates a menu state for the + client to render an initial layer menu. +""" + + +@layers_router.get( + "/default", + response_model=LayerDefault, + summary="Get the hierarchy and layer data for a default layer.", + description="Gets layer data needed for the menu and map when first loaded.", +) +def get_default_layer(request: Request): + authorized_map_groups = list( + filter(lambda x: x.auth(request.auth.scopes), request.app.config.map_groups) + ) + + # Return early because we have no default layer nor menu state + if not authorized_map_groups: + return LayerDefault( + layer=None, + default_layer_menu=[], + default_map_group_id=None, + default_map_id=None, + default_band_id=None, + ) + + # Otherwise, set defaults to first index at each level + default_map_group = authorized_map_groups[0] + default_map = default_map_group.maps[0] + default_band = default_map.bands[0] + default_layer = default_band.layers[0] + + # Make layer summaries for all the default band's layers + default_layer_summaries = [ + LayerSummary( + layer_id=layer.layer_id, name=layer.name, description=layer.description + ) + for layer in default_band.layers + ] + + # Add the default_layer_summaries only to the default band; otherwise, assign an empty list + default_band_summaries = [] + for band in default_map.bands: + default_band_summaries.append( + BandMenuState( + band_id=band.band_id, + name=band.name, + description=band.description, + layers=default_layer_summaries + if band.band_id == default_band.band_id + else [], + ) + ) + + # Add the default_band_summaries only to the default map; otherwise, assign an empty list + default_map_summaries = [] + for map in default_map_group.maps: + default_map_summaries.append( + MapMenuState( + map_id=map.map_id, + name=map.name, + description=map.description, + bands=default_band_summaries + if map.map_id == default_map.map_id + else [], + ) + ) + + # Add the default_map_summaries only to the default map group; otherwise, assign an empty list + default_map_groups = [] + for map_group in authorized_map_groups: + default_map_groups.append( + MapGroupMenuState( + map_group_id=map_group.map_group_id, + name=map_group.name, + description=map_group.description, + maps=default_map_summaries + if map_group.map_group_id == default_map_group.map_group_id + else [], + ) + ) + + return LayerDefault( + layer=default_layer, + default_layer_menu=default_map_groups, + default_map_group_id=default_map_group.map_group_id, + default_map_id=default_map.map_id, + default_band_id=default_band.band_id, + ) + + +@layers_router.get( + "/{layer_id}", + response_model=LayerWithMenuState, + summary="Get the Layer data.", + description="Retrieve the Layer data to be rendered in the mapping client.", +) +def get_layer_with_menu_state(layer_id: str, request: Request): + for map_group in request.app.config.map_groups: + if map_group.auth(request.auth.scopes): + for map in map_group.maps: + for band in map.bands: + for layer in band.layers: + if layer.layer_id == layer_id and map_group.auth( + request.auth.scopes + ): + return LayerWithMenuState( + **layer.model_dump(), + map_group_id=map_group.map_group_id, + map_id=map.map_id, + band_id=band.band_id, + ) + + +@layers_router.get( + "/{layer_id}/submap/{left}/{right}/{top}/{bottom}/image.{ext}", + summary="Generate a cut-out of the map.", + description="Download and extract (from the base map) a rendered cut-out. Downloads at the full resolution of the underlying map with no additional filter function.", +) +def get_submap( + layer_id: str, + left: float, + right: float, + top: float, + bottom: float, + ext: Literal["jpg", "webp", "png", "fits"], + request: Request, + bt: BackgroundTasks, + render_options: RenderOptions = Depends(RenderOptions), + show_grid: bool = False, +): + """ + Get a submap of the specified band. + """ + + submap, pushables = extract( + layer_id=layer_id, + left=left, + right=right, + top=top, + bottom=bottom, + tiles=request.app.tiles, + grants=request.auth.scopes, + metadata=request.app.config, + show_grid=show_grid, + ) + + bt.add_task(request.app.tiles.push, pushables) + + if ext == "jpg": + with io.BytesIO() as output: + renderer.render(output, submap, render_options=render_options) + return Response(content=output.getvalue(), media_type="image/jpg") + elif ext == "webp": + with io.BytesIO() as output: + renderer.render(output, submap, render_options=render_options) + return Response(content=output.getvalue(), media_type="image/webp") + elif ext == "png": + with io.BytesIO() as output: + renderer.render(output, submap, render_options=render_options) + return Response(content=output.getvalue(), media_type="image/png") + elif ext == "fits": + with io.BytesIO() as output: + hdu = fits.PrimaryHDU(submap) + hdu.writeto(output) + return Response(content=output.getvalue(), media_type="image/fits") + + +def core_tile_retrieval( + layer_id: str, + level: int, + y: int, + x: int, + bt: BackgroundTasks, + request: Request, +): + tile, pushables = request.app.tiles.pull( + PullableTile( + layer_id=layer_id, x=x, y=y, level=level, grants=request.auth.scopes + ) + ) + + bt.add_task(request.app.tiles.push, pushables) + + return tile.data + + +@layers_router.get( + "/{layer_id}/{level}/{y}/{x}/tile.{ext}", + summary="Retrieve an individual tile.", + description="Individual tiles are hosted at a layer level, with them having three axes: `level`, `y`, and `x`. We support extensions of 'png', 'webp', and 'jpg'.", +) +def get_tile( + layer_id: str, + level: int, + y: int, + x: int, + ext: str, + request: Request, + bt: BackgroundTasks, + render_options: RenderOptions = Depends(RenderOptions), +): + """ + Grab an individual tile. This should be very fast, because we use + a composite primary key for band, level, x, and y. + + Supported extensions: + - jpg + - webp + - png + + Note: This does not support FITS tiles, as they are not + typically used for rendering. If you need FITS images, please + use the `/maps/{map}/{band}/submap/{left}/{right}/{top}/{bottom}/image.fits` + endpoint instead. + """ + + if render_options.flip: + # Flipping is really a reconfiguration of -180 < RA < 180 to 360 < RA < 0; + # it's a card-folding operation. + if level != 0: + # Level of zero requires no flipping apart from at the tile level. + midpoint = 2 ** (level) + if x < midpoint: + x = (2 ** (level) - 1) - x + else: + x = (2 ** (level) - 1) - (x - midpoint) + midpoint + + if ext not in ["jpg", "webp", "png"]: + raise HTTPException(status_code=400, detail="Not an acceptable extension") + + numpy_buf = core_tile_retrieval( + layer_id=layer_id, + level=level, + y=y, + x=x, + request=request, + bt=bt, + ) + + if numpy_buf is None: + raise HTTPException(status_code=404, detail="Tile not found") + + with io.BytesIO() as output: + renderer.render(output, numpy_buf, render_options=render_options) + return Response(content=output.getvalue(), media_type=f"image/{ext}") diff --git a/tilemaker/server/map_groups.py b/tilemaker/server/map_groups.py new file mode 100644 index 0000000..2a28b57 --- /dev/null +++ b/tilemaker/server/map_groups.py @@ -0,0 +1,54 @@ +""" +Endpoints for getting list of map group summaries and a list of a map group's map summaries. +""" + +from fastapi import ( + APIRouter, + Request, +) + +from tilemaker.metadata.definitions import MapBase, MapGroupBase + +map_groups_router = APIRouter(prefix="/map-groups", tags=["List of Map Groups"]) + + +@map_groups_router.get( + "", + response_model=list[MapGroupBase], + summary="Get the list of map group summaries.", + description="Retrieve a list of MapGroupSummary objects.", +) +def get_map_group_summaries(request: Request): + map_group_summaries = [] + for x in request.app.config.map_groups: + if x.auth(request.auth.scopes): + map_group_summary = MapGroupBase( + map_group_id=x.map_group_id, + name=x.name, + description=x.description, + ) + map_group_summaries.append(map_group_summary) + + return map_group_summaries + + +@map_groups_router.get( + "/{map_group_id}/maps", + response_model=list[MapBase], + summary="Get the list of map summaries associated with a Map Group.", + description="Retrieve a list of MapSummary objects that belong to a particular Map Group.", +) +def get_map_summaries_of_map_group(map_group_id: str, request: Request): + map_summaries = [] + for map_group in request.app.config.map_groups: + if map_group.map_group_id == map_group_id and map_group.auth( + request.auth.scopes + ): + for map in map_group.maps: + map_summary = MapBase( + map_id=map.map_id, + name=map.name, + description=map.description, + ) + map_summaries.append(map_summary) + return map_summaries diff --git a/tilemaker/server/maps.py b/tilemaker/server/maps.py index e56df78..93a6208 100644 --- a/tilemaker/server/maps.py +++ b/tilemaker/server/maps.py @@ -2,167 +2,30 @@ Endpoints for maps. """ -import io -from typing import Literal +from fastapi import APIRouter, Request -from astropy.io import fits -from fastapi import ( - APIRouter, - BackgroundTasks, - Depends, - HTTPException, - Request, - Response, -) - -from tilemaker.metadata.definitions import MapGroup -from tilemaker.processing.extractor import extract -from tilemaker.providers.fits import PullableTile - -from ..processing.renderer import Renderer, RenderOptions - -renderer = Renderer(format="webp") +from tilemaker.metadata.definitions import BandBase maps_router = APIRouter(prefix="/maps", tags=["Maps and Tiles"]) @maps_router.get( - "", - response_model=list[MapGroup], - summary="Get the list of map groups.", - description="Retrieve a list of MapGroup shaped objects, each containing a list of Maps, with a list of Bands, and finally a list of Layers.", -) -def get_maps(request: Request): - return [x for x in request.app.config.map_groups if x.auth(request.auth.scopes)] - - -@maps_router.get( - "/{layer_id}/submap/{left}/{right}/{top}/{bottom}/image.{ext}", - summary="Generate a cut-out of the map.", - description="Download and extract (from the base map) a rendered cut-out. Downloads at the full resolution of the underlying map with no additional filter function.", -) -def get_submap( - layer_id: str, - left: float, - right: float, - top: float, - bottom: float, - ext: Literal["jpg", "webp", "png", "fits"], - request: Request, - bt: BackgroundTasks, - render_options: RenderOptions = Depends(RenderOptions), - show_grid: bool = False, -): - """ - Get a submap of the specified band. - """ - - submap, pushables = extract( - layer_id=layer_id, - left=left, - right=right, - top=top, - bottom=bottom, - tiles=request.app.tiles, - grants=request.auth.scopes, - metadata=request.app.config, - show_grid=show_grid, - ) - - bt.add_task(request.app.tiles.push, pushables) - - if ext == "jpg": - with io.BytesIO() as output: - renderer.render(output, submap, render_options=render_options) - return Response(content=output.getvalue(), media_type="image/jpg") - elif ext == "webp": - with io.BytesIO() as output: - renderer.render(output, submap, render_options=render_options) - return Response(content=output.getvalue(), media_type="image/webp") - elif ext == "png": - with io.BytesIO() as output: - renderer.render(output, submap, render_options=render_options) - return Response(content=output.getvalue(), media_type="image/png") - elif ext == "fits": - with io.BytesIO() as output: - hdu = fits.PrimaryHDU(submap) - hdu.writeto(output) - return Response(content=output.getvalue(), media_type="image/fits") - - -def core_tile_retrieval( - layer: str, - level: int, - y: int, - x: int, - bt: BackgroundTasks, - request: Request, -): - tile, pushables = request.app.tiles.pull( - PullableTile(layer_id=layer, x=x, y=y, level=level, grants=request.auth.scopes) - ) - - bt.add_task(request.app.tiles.push, pushables) - - return tile.data - - -@maps_router.get( - "/{layer}/{level}/{y}/{x}/tile.{ext}", - summary="Retrieve an individual tile.", - description="Individual tiles are hosted at a layer level, with them having three axes: `level`, `y`, and `x`. We support extensions of 'png', 'webp', and 'jpg'.", + "/{map_id}/bands", + response_model=list[BandBase], + summary="Get the list of band summaries associated with a Map.", + description="Retrieve a list of BandSummary objects that belong to a particular Map.", ) -def get_tile( - layer: str, - level: int, - y: int, - x: int, - ext: str, - request: Request, - bt: BackgroundTasks, - render_options: RenderOptions = Depends(RenderOptions), -): - """ - Grab an individual tile. This should be very fast, because we use - a composite primary key for band, level, x, and y. - - Supported extensions: - - jpg - - webp - - png - - Note: This does not support FITS tiles, as they are not - typically used for rendering. If you need FITS images, please - use the `/maps/{map}/{band}/submap/{left}/{right}/{top}/{bottom}/image.fits` - endpoint instead. - """ - - if render_options.flip: - # Flipping is really a reconfiguration of -180 < RA < 180 to 360 < RA < 0; - # it's a card-folding operation. - if level != 0: - # Level of zero requires no flipping apart from at the tile level. - midpoint = 2 ** (level) - if x < midpoint: - x = (2 ** (level) - 1) - x - else: - x = (2 ** (level) - 1) - (x - midpoint) + midpoint - - if ext not in ["jpg", "webp", "png"]: - raise HTTPException(status_code=400, detail="Not an acceptable extension") - - numpy_buf = core_tile_retrieval( - layer=layer, - level=level, - y=y, - x=x, - request=request, - bt=bt, - ) - - if numpy_buf is None: - raise HTTPException(status_code=404, detail="Tile not found") - - with io.BytesIO() as output: - renderer.render(output, numpy_buf, render_options=render_options) - return Response(content=output.getvalue(), media_type=f"image/{ext}") +def get_layer_summaries_of_band(map_id: str, request: Request): + for map_group in request.app.config.map_groups: + for map in map_group.maps: + if map.map_id == map_id and map_group.auth(request.auth.scopes): + for band in map.bands: + band_summaries = [] + band_summary = BandBase( + band_id=band.band_id, + name=band.name, + description=band.description, + ) + band_summaries.append(band_summary) + return band_summaries + return [] diff --git a/tilemaker/server/search.py b/tilemaker/server/search.py new file mode 100644 index 0000000..7b9f039 --- /dev/null +++ b/tilemaker/server/search.py @@ -0,0 +1,26 @@ +""" +Endpoint to search map groups and its children +""" + +from fastapi import ( + APIRouter, + Query, + Request, +) + +from tilemaker.metadata.definitions import SearchResponse + +search_router = APIRouter(prefix="/search", tags=["Search map groups"]) + + +@search_router.get("", response_model=SearchResponse) +def search_layers(request: Request, q: str = Query(..., min_length=1)): + authorized_groups = [ + g.dict() for g in request.app.config.map_groups if g.auth(request.auth.scopes) + ] + result = request.app.config.filter_map_groups(authorized_groups, q) + + return SearchResponse( + filtered_layer_menu=result["filtered_map_groups"], + matched_ids=list(result["matched_ids"]), + )