From 4a36c827064999b05758859ec8cea011796f83d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Mon, 6 Jul 2026 17:31:27 -0300 Subject: [PATCH] fix(http): replace http by web for better naming and include a preview of the web service on home page --- README.md | 8 +- docs/source/index.rst | 3 + docs/source/installation.rst | 19 +++ poetry.lock | 4 +- pyproject.toml | 5 +- pysus/cli/__init__.py | 6 +- pysus/http/app.py | 94 ------------ pysus/tests/data/test_dbf_reader.py | 20 +-- pysus/{http => web}/__init__.py | 2 +- pysus/web/app.py | 151 ++++++++++++++++++ pysus/{http => web}/assets/logo.svg | 0 pysus/{http => web}/assets/logo_large.svg | 0 pysus/{http => web}/pages/1_client.py | 162 ++++++++++++++------ pysus/{http => web}/pages/2_examples.py | 0 pysus/{http => web}/translations.py | 178 +++++++++++++++++++++- 15 files changed, 475 insertions(+), 177 deletions(-) delete mode 100644 pysus/http/app.py rename pysus/{http => web}/__init__.py (64%) create mode 100644 pysus/web/app.py rename pysus/{http => web}/assets/logo.svg (100%) rename pysus/{http => web}/assets/logo_large.svg (100%) rename pysus/{http => web}/pages/1_client.py (87%) rename pysus/{http => web}/pages/2_examples.py (100%) rename pysus/{http => web}/translations.py (50%) diff --git a/README.md b/README.md index 3ff0517c..5dffa0af 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ pip install pysus For the local Streamlit web interface: ```bash -pip install pysus[http] +pip install pysus[web] ``` ### Docker @@ -129,19 +129,19 @@ async def main(): Launch the local web interface: ```bash -pysus http +pysus web ``` Or with a custom port: ```bash -pysus http -p 8080 +pysus web -p 8080 ``` Or run directly with Streamlit: ```bash -streamlit run pysus/http/app.py +streamlit run pysus/web/app.py ``` The web interface provides three data sources: diff --git a/docs/source/index.rst b/docs/source/index.rst index c6cf09cb..b9e64b02 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,6 +20,9 @@ PySUS is a Python package for accessing and analyzing Brazil's public health dat and work with health datasets including SINAN (disease notifications), SIM (mortality), SINASC (births), SIH (hospitalizations), SIA (ambulatory), CIHA, CNES, PNI, and more. +A local web server (`pysus web`) is included for interactive browsing and downloading +of datasets through a graphical Streamlit interface. + This documentation covers PySUS 2.0+. .. toctree:: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index dc994612..4034c22a 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -60,6 +60,25 @@ Using Poetry: poetry install +Web Interface +------------- + +PySUS includes a local Streamlit-based web server for browsing and downloading +datasets interactively. Start it with: + +.. code-block:: bash + + pysus web + +Or directly: + +.. code-block:: bash + + streamlit run pysus/web/app.py + +This opens a browser at ``http://localhost:8501`` with a graphical interface for +querying PySUS s3, DATASUS FTP, and dados.gov.br sources. + Configuration ------------- diff --git a/poetry.lock b/poetry.lock index 46ff2041..ab5a0044 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5292,9 +5292,9 @@ files = [ dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"] [extras] -http = [] +web = [] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "2fdd6705ff86f8c17e9ab4a869440f57910087935a3c6a434d9b7ff9e2398208" +content-hash = "ea50177e1fc511f88ad0bb568c1af13255eee0fa049a3e13bd0e8d0ceac11855" diff --git a/pyproject.toml b/pyproject.toml index b8461abb..5731cb7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ exclude = [ "pysus/tests/**", "pysus/management", "pysus/management/**", - "pysus/http/**", ] [tool.poetry.dependencies] @@ -47,7 +46,7 @@ typer = "^0.24.1" humanize = "^4.8.0" [tool.poetry.extras] -http = ["streamlit"] +web = ["streamlit"] [tool.poetry.group.dev.dependencies] pytest = ">=6.1.0" @@ -106,7 +105,7 @@ exclude = ["*.git", "docs/"] omit = [ "pysus/management/client.py", "pysus/tui/*", - "pysus/http/*", + "pysus/web/*", "pysus/cli/*", ] diff --git a/pysus/cli/__init__.py b/pysus/cli/__init__.py index b998853d..29135914 100644 --- a/pysus/cli/__init__.py +++ b/pysus/cli/__init__.py @@ -10,7 +10,7 @@ def version(): @app.command() -def http( +def web( port: int = typer.Option( # noqa: B008 8501, "-p", @@ -25,13 +25,13 @@ def http( except ImportError: raise ImportError( "The HTTP UI requires extra dependencies. " - "Install them with: pip install pysus[http]" + "Install them with: pip install pysus[web]" ) import os import sys import webbrowser - app_path = os.path.join(os.path.dirname(__file__), "..", "http", "app.py") + app_path = os.path.join(os.path.dirname(__file__), "..", "web", "app.py") app_path = os.path.abspath(app_path) from streamlit.web import cli as stcli diff --git a/pysus/http/app.py b/pysus/http/app.py deleted file mode 100644 index f85869bf..00000000 --- a/pysus/http/app.py +++ /dev/null @@ -1,94 +0,0 @@ -"""PySUS Streamlit App — localhost visual interface for the PySUS package. - -Run with: - streamlit run pysus/http/app.py - or - pysus http -""" - -import asyncio - -import streamlit as st - -from pysus import __version__ -from pysus.http.translations import t -from pysus.api.client import PySUS - -LANGUAGES = {"English": "en", "Português": "pt"} -LANG_LABELS = {v: k for k, v in LANGUAGES.items()} - -st.set_page_config( - page_title="PySUS", - page_icon=":hospital:", - layout="wide", -) - -st.markdown( - """ - - """, - unsafe_allow_html=True, -) - - -@st.cache_data(show_spinner="loading datasets...", ttl=600) -def _load_catalog() -> None: - async def _fetch(): - try: - async with PySUS(): - return - except Exception: - _load_catalog.clear() - return - - return asyncio.run(_fetch()) - - -def _init_lang() -> None: - if "lang" not in st.session_state: - st.session_state.lang = "pt" - - -def _on_lang_change() -> None: - label = st.session_state.get("_lang_select", "English") - st.session_state.lang = LANGUAGES[label] - - -def _lang_selector() -> None: - _init_lang() - current_label = LANG_LABELS.get(st.session_state.lang, "Português") - st.sidebar.selectbox( - t("lang_label", st.session_state.lang), - list(LANGUAGES.keys()), - index=list(LANGUAGES.keys()).index(current_label), - key="_lang_select", - on_change=_on_lang_change, - ) - - -def home() -> None: - _lang_selector() - lang: str = st.session_state.lang - - st.title("Datasets") - - -if __name__ == "__main__": - _init_lang() - _load_catalog() - lang = st.session_state.lang - - home_page = st.Page(home, title=f"🏠️ {t('home_page', lang)}", default=True) - client_page = st.Page("pages/1_client.py", title="📥️ Downloads") - - examples_page = st.Page("pages/2_examples.py", title="Examples") - - st.logo( - "https://raw.githubusercontent.com/AlertaDengue/PySUS/db96a5ae94e899851490328ce784b3a5afd68a30/pysus/http/assets/logo_large.svg", - icon_image="https://raw.githubusercontent.com/AlertaDengue/PySUS/db96a5ae94e899851490328ce784b3a5afd68a30/pysus/http/assets/logo.svg", - ) - - pg = st.navigation({"": [home_page, client_page], "docs": [examples_page]}) - pg.run() diff --git a/pysus/tests/data/test_dbf_reader.py b/pysus/tests/data/test_dbf_reader.py index 4df5fc78..fe91203a 100644 --- a/pysus/tests/data/test_dbf_reader.py +++ b/pysus/tests/data/test_dbf_reader.py @@ -2,10 +2,8 @@ from datetime import date from pathlib import Path -import numpy as np import pandas as pd import pytest - from pysus.data.dbf_reader import ( _parse_header, read_dbf_fast, @@ -14,7 +12,6 @@ stream_dbf_fast, ) - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -85,9 +82,7 @@ def simple_dbf(tmp_dir): def wide_dbf(tmp_dir): dbf_path = tmp_dir / "wide.dbf" fields = [(f"COL{i}", "C", 8, 0) for i in range(20)] - records = [ - tuple(f"val{i}_{j}" for j in range(20)) for i in range(100) - ] + records = [tuple(f"val{i}_{j}" for j in range(20)) for i in range(100)] _create_dbf(dbf_path, fields, records) return dbf_path @@ -200,7 +195,6 @@ def test_read_fast_wide(wide_dbf): def test_read_fast_strips_nul_bytes(tmp_dir): dbf_path = tmp_dir / "nuls.dbf" today = date.today() - fields = [("VAL", "C", 8, 0)] buf = bytearray() buf.append(0x03) buf.append(today.year - 1900) @@ -289,15 +283,11 @@ def test_read_filtered_column_subset(disease_dbf): def test_read_filtered_missing_column(disease_dbf): with pytest.raises(KeyError, match="not found"): - read_dbf_filtered( - disease_dbf, column="NONEXISTENT", values=["X"] - ) + read_dbf_filtered(disease_dbf, column="NONEXISTENT", values=["X"]) def test_read_filtered_empty_file(empty_dbf): - df = read_dbf_filtered( - empty_dbf, column="NAME", values=["X"] - ) + df = read_dbf_filtered(empty_dbf, column="NAME", values=["X"]) assert len(df) == 0 @@ -378,9 +368,7 @@ def test_large_record_count(tmp_dir): def test_field_name_lowercase_match(): - from pysus.data.dbf_reader import _find_field - - from pysus.data.dbf_reader import DBFSchema, DBFField + from pysus.data.dbf_reader import DBFField, DBFSchema, _find_field schema = DBFSchema( num_records=1, diff --git a/pysus/http/__init__.py b/pysus/web/__init__.py similarity index 64% rename from pysus/http/__init__.py rename to pysus/web/__init__.py index c16bcf3f..00cbe2bd 100644 --- a/pysus/http/__init__.py +++ b/pysus/web/__init__.py @@ -1,3 +1,3 @@ """PySUS HTTP — Streamlit-based visual interface for localhost use.""" -from pysus.http.app import home # noqa +from pysus.web.app import home # noqa diff --git a/pysus/web/app.py b/pysus/web/app.py new file mode 100644 index 00000000..7e47c572 --- /dev/null +++ b/pysus/web/app.py @@ -0,0 +1,151 @@ +"""PySUS Streamlit App — localhost visual interface for the PySUS package. + +Run with: + streamlit run pysus/web/app.py + or + pysus web +""" + +import asyncio + +import streamlit as st +from pysus.api.client import PySUS +from pysus.web.translations import t + +LANGUAGES = {"English": "en", "Português": "pt"} +LANG_LABELS = {v: k for k, v in LANGUAGES.items()} + +st.set_page_config( + page_title="PySUS", + page_icon=":hospital:", + layout="wide", +) + +st.markdown( + """ + + """, + unsafe_allow_html=True, +) + + +@st.cache_data(show_spinner="loading datasets...", ttl=600) +def _load_catalog() -> None: + async def _fetch(): + try: + async with PySUS(): + return + except Exception: # noqa: B902 + _load_catalog.clear() + return + + return asyncio.run(_fetch()) + + +def _init_lang() -> None: + if "lang" not in st.session_state: + st.session_state.lang = "pt" + + +def _on_lang_change() -> None: + label = st.session_state.get("_lang_select", "English") + st.session_state.lang = LANGUAGES[label] + + +def _lang_selector() -> None: + _init_lang() + current_label = LANG_LABELS.get(st.session_state.lang, "Português") + st.sidebar.selectbox( + t("lang_label", st.session_state.lang), + list(LANGUAGES.keys()), + index=list(LANGUAGES.keys()).index(current_label), + key="_lang_select", + on_change=_on_lang_change, + ) + + +def home() -> None: + _lang_selector() + lang: str = st.session_state.lang + + # -- Hero -- + st.title(f":hospital: {t('intro_welcome', lang)}") + st.markdown(t("intro_desc", lang)) + + # -- Data Sources -- + st.subheader(f"📡 {t('intro_available', lang)}") + sc1, sc2, sc3 = st.columns(3) + for col, key in ((sc1, "ducklake"), (sc2, "ftp"), (sc3, "dadosgov")): + with col: + with st.container(border=True, height="stretch"): + st.markdown(t(f"intro_source_{key}", lang)) + + # -- Docs & GitHub -- + with st.container(): + st.markdown(t("intro_package", lang)) + + with st.container(): + st.markdown(t("intro_github", lang)) + + st.divider() + + # -- Databases -- + st.header(f"📊 {t('databases_title', lang)}") + st.markdown(t("databases_desc", lang)) + + databases = [ + ("SINAN", "sinan", "https://portalsinan.saude.gov.br/"), + ("SINASC", "sinasc", "http://sinasc.saude.gov.br/"), + ("SIM", "sim", "http://sim.saude.gov.br/default.asp"), + ("SIH", "sih", "http://sihd.datasus.gov.br/principal/index.php"), + ("SIA", "sia", "https://sia.datasus.gov.br/principal/index.php"), + ( + "PNI", + "pni", + "https://sipni.datasus.gov.br/si-pni-web/faces/inicio.jsf", + ), + ("CNES", "cnes", "https://cnes.datasus.gov.br/"), + ("CIHA", "ciha", "http://ciha.datasus.gov.br/CIHA/index.php"), + ("IBGE", "ibge", "https://www.ibge.gov.br/"), + ] + + for i in range(0, len(databases), 3): + cols = st.columns(3) + for j in range(3): + if i + j >= len(databases): + break + name, db, url = databases[i + j] + with cols[j]: + with st.container(border=True, height="stretch"): + st.markdown(f"**{name}**") + st.caption(f"[{t('databases_source', lang)} ↗]({url})") + st.markdown(t(f"db_{db}_desc", lang)) + + st.divider() + + +if __name__ == "__main__": + _init_lang() + _load_catalog() + lang = st.session_state.lang + + home_page = st.Page(home, title=f"🏠️ {t('home_page', lang)}", default=True) + client_page = st.Page("pages/1_client.py", title="📥️ Downloads") + + examples_page = st.Page("pages/2_examples.py", title="Examples") + + st.logo( + "https://raw.githubusercontent.com/AlertaDengue/PySUS/" + "db96a5ae94e899851490328ce784b3a5afd68a30/" + "pysus/http/assets/logo_large.svg", + icon_image=( + "https://raw.githubusercontent.com/AlertaDengue/PySUS/" + "db96a5ae94e899851490328ce784b3a5afd68a30/" + "pysus/http/assets/logo.svg" + ), + ) + + pg = st.navigation({"": [home_page, client_page], "docs": [examples_page]}) + pg.run() diff --git a/pysus/http/assets/logo.svg b/pysus/web/assets/logo.svg similarity index 100% rename from pysus/http/assets/logo.svg rename to pysus/web/assets/logo.svg diff --git a/pysus/http/assets/logo_large.svg b/pysus/web/assets/logo_large.svg similarity index 100% rename from pysus/http/assets/logo_large.svg rename to pysus/web/assets/logo_large.svg diff --git a/pysus/http/pages/1_client.py b/pysus/web/pages/1_client.py similarity index 87% rename from pysus/http/pages/1_client.py rename to pysus/web/pages/1_client.py index 193fbd60..fd5cfdb5 100644 --- a/pysus/http/pages/1_client.py +++ b/pysus/web/pages/1_client.py @@ -5,11 +5,10 @@ import pandas as pd import streamlit as st from humanize import naturalsize - from pysus import CACHEPATH from pysus.api.client import PySUS from pysus.api.models import BaseRemoteFile -from pysus.http.translations import t +from pysus.web.translations import t STATES = [ "AC", @@ -169,13 +168,15 @@ def _get_group_options( ) -> list[str]: if not ds_name: return [] - target = next((d for d in datasets if d.name.upper() == ds_name.upper()), None) + target = next( + (d for d in datasets if d.name.upper() == ds_name.upper()), None + ) if target is None: return [] if client == "ducklake": - from sqlalchemy import select from pysus.api.ducklake.catalog.orm.dataset import Group as OrmGroup + from sqlalchemy import select async def _fetch() -> list[str]: await target.adapter.connect(callback=callback) @@ -186,7 +187,7 @@ async def _fetch() -> list[str]: try: return _run_async(_fetch()) - except Exception as exc: + except Exception as exc: # noqa: B902 st.warning(t("catalog_query_failed", _lang(), error=str(exc))) return [] @@ -209,26 +210,30 @@ def _get_year_options( return [] if client != "ducklake": return [] - target = next((d for d in datasets if d.name.upper() == ds_name.upper()), None) + target = next( + (d for d in datasets if d.name.upper() == ds_name.upper()), None + ) if target is None: return [] - from sqlalchemy import distinct, select from pysus.api.ducklake.catalog.orm.dataset import File as OrmFile + from sqlalchemy import distinct, select async def _fetch() -> list[int]: await target.adapter.connect(callback=callback) with target.adapter.get_session() as session: stmt = ( select(distinct(OrmFile.year)) - .filter(OrmFile.dataset_id == target.id, OrmFile.year.isnot(None)) + .filter( + OrmFile.dataset_id == target.id, OrmFile.year.isnot(None) + ) .order_by(OrmFile.year) ) return sorted(y for y in session.scalars(stmt).all()) try: return _run_async(_fetch()) - except Exception as exc: + except Exception as exc: # noqa: B902 st.warning(t("catalog_query_failed", _lang(), error=str(exc))) return [] @@ -243,31 +248,37 @@ def _get_month_options( return [] if client != "ducklake": return [] - target = next((d for d in datasets if d.name.upper() == ds_name.upper()), None) + target = next( + (d for d in datasets if d.name.upper() == ds_name.upper()), None + ) if target is None: return [] - from sqlalchemy import distinct, select from pysus.api.ducklake.catalog.orm.dataset import File as OrmFile + from sqlalchemy import distinct, select async def _fetch() -> list[int]: await target.adapter.connect(callback=callback) with target.adapter.get_session() as session: stmt = ( select(distinct(OrmFile.month)) - .filter(OrmFile.dataset_id == target.id, OrmFile.month.isnot(None)) + .filter( + OrmFile.dataset_id == target.id, OrmFile.month.isnot(None) + ) .order_by(OrmFile.month) ) return sorted(m for m in session.scalars(stmt).all()) try: return _run_async(_fetch()) - except Exception as exc: + except Exception as exc: # noqa: B902 st.warning(t("catalog_query_failed", _lang(), error=str(exc))) return [] -def _render_year_filter(client: str, year_options: list[int]) -> list[int] | None: +def _render_year_filter( + client: str, year_options: list[int] +) -> list[int] | None: if year_options: return ( st.multiselect( @@ -288,7 +299,9 @@ def _render_year_filter(client: str, year_options: list[int]) -> list[int] | Non return None -def _render_month_filter(client: str, month_options: list[int]) -> list[int] | None: +def _render_month_filter( + client: str, month_options: list[int] +) -> list[int] | None: if month_options: return ( st.multiselect( @@ -333,7 +346,7 @@ def _on_progress(dl: int, total: int) -> None: try: _run_async(pysus.get_ducklake(callback=_on_progress)) progress.empty() - except Exception: + except Exception: # noqa: B902 progress.empty() st.warning(t("catalog_failed", _lang())) return @@ -348,7 +361,7 @@ def _on_progress(dl: int, total: int) -> None: ds_names, index=None, placeholder=t("browser_choose", _lang()), - key=f"_ds_ducklake", + key="_ds_ducklake", ) ds_key = _ds_key(selected_ds) @@ -362,9 +375,15 @@ def _on_progress(dl: int, total: int) -> None: if not selected_ds: return - group_options = _cached_options_with_progress("ducklake", datasets, ds_key, "group") - year_options = _cached_options_with_progress("ducklake", datasets, ds_key, "year") - month_options = _cached_options_with_progress("ducklake", datasets, ds_key, "month") + group_options = _cached_options_with_progress( + "ducklake", datasets, ds_key, "group" + ) + year_options = _cached_options_with_progress( + "ducklake", datasets, ds_key, "year" + ) + month_options = _cached_options_with_progress( + "ducklake", datasets, ds_key, "month" + ) col1, col2, col3 = st.columns(3) with col1: @@ -379,7 +398,9 @@ def _on_progress(dl: int, total: int) -> None: with col2: state = ( st.multiselect( - t("state", _lang()), STATES, placeholder=t("select_states", _lang()) + t("state", _lang()), + STATES, + placeholder=t("select_states", _lang()), ) or None ) @@ -396,7 +417,7 @@ def _on_progress(dl: int, total: int) -> None: def _render_ftp_filters(pysus: PySUS) -> None: try: datasets = _cached_datasets(pysus, "ftp") - except Exception as exc: + except Exception as exc: # noqa: B902 st.warning(f"{t('ftp_failed', _lang())} {exc}") return if not datasets: @@ -407,7 +428,7 @@ def _render_ftp_filters(pysus: PySUS) -> None: ds_names, index=None, placeholder=t("browser_choose", _lang()), - key=f"_ds_ftp", + key="_ds_ftp", ) ds_key = _ds_key(selected_ds) @@ -436,7 +457,9 @@ def _render_ftp_filters(pysus: PySUS) -> None: with col2: state = ( st.multiselect( - t("state", _lang()), STATES, placeholder=t("select_states", _lang()) + t("state", _lang()), + STATES, + placeholder=t("select_states", _lang()), ) or None ) @@ -476,7 +499,7 @@ def _render_dadosgov_filters(pysus: PySUS) -> None: ds_names, index=None, placeholder=t("browser_choose", _lang()), - key=f"_ds_dadosgov", + key="_ds_dadosgov", ) ds_key = _ds_key(selected_ds) @@ -505,7 +528,9 @@ def _render_dadosgov_filters(pysus: PySUS) -> None: with col2: state = ( st.multiselect( - t("state", _lang()), STATES, placeholder=t("select_states", _lang()) + t("state", _lang()), + STATES, + placeholder=t("select_states", _lang()), ) or None ) @@ -549,7 +574,7 @@ def _parse_and_query( month_vals, ) ) - except Exception as exc: + except Exception as exc: # noqa: B902 st.error(t("query_failed", _lang(), error=str(exc))) return @@ -574,13 +599,19 @@ async def _ftp_dadosgov_search( ) -> list[BaseRemoteFile]: all_files = await target.search() if groups: - all_files = [f for f in all_files if getattr(f.group, "name", None) in groups] + all_files = [ + f for f in all_files if getattr(f.group, "name", None) in groups + ] if states: - all_files = [f for f in all_files if getattr(f, "state", None) in states] + all_files = [ + f for f in all_files if getattr(f, "state", None) in states + ] if years: all_files = [f for f in all_files if getattr(f, "year", None) in years] if months: - all_files = [f for f in all_files if getattr(f, "month", None) in months] + all_files = [ + f for f in all_files if getattr(f, "month", None) in months + ] return all_files @@ -605,12 +636,16 @@ async def _query_client( if client == "ftp": ftp_client = await pysus.get_ftp() datasets = await ftp_client.datasets() - target = next((d for d in datasets if d.name.upper() == ds_name.upper()), None) + target = next( + (d for d in datasets if d.name.upper() == ds_name.upper()), None + ) if target is None: return [] try: - return await _ftp_dadosgov_search(target, groups, states, years, months) - except (ConnectionResetError, BrokenPipeError, OSError): + return await _ftp_dadosgov_search( + target, groups, states, years, months + ) + except OSError: await ftp_client.close() pysus._ftp = None ftp_client = await pysus.get_ftp() @@ -620,11 +655,15 @@ async def _query_client( ) if target is None: return [] - return await _ftp_dadosgov_search(target, groups, states, years, months) + return await _ftp_dadosgov_search( + target, groups, states, years, months + ) else: dadosgov_client = await pysus.get_dadosgov(None) datasets = await dadosgov_client.datasets() # type: ignore[assignment] - target = next((d for d in datasets if d.name.upper() == ds_name.upper()), None) + target = next( + (d for d in datasets if d.name.upper() == ds_name.upper()), None + ) if target is None: return [] return await _ftp_dadosgov_search(target, groups, states, years, months) @@ -679,7 +718,9 @@ def _native_dir_picker(title: str, initialdir: str) -> str: ["kdialog", "--getexistingdirectory", initialdir, "--title", title], ): try: - r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + r = subprocess.run( + cmd, capture_output=True, text=True, timeout=30 + ) return r.stdout.strip() except (FileNotFoundError, subprocess.TimeoutExpired): continue @@ -701,13 +742,17 @@ def _native_dir_picker(title: str, initialdir: str) -> str: return r.stdout.strip() elif system == "Darwin": - applescript = f""" -tell application "System Events" - activate - set f to choose folder with prompt "{title}" default location POSIX file "{initialdir}" - POSIX path of f -end tell -""" + prompt_line = ( + 'set f to choose folder with prompt "{}"' + ' default location POSIX file "{}"' + ).format(title, initialdir) + applescript = ( + f'tell application "System Events"\n' + f" activate\n" + f" {prompt_line}\n" + f" POSIX path of f\n" + f"end tell" + ) r = subprocess.run( ["osascript", "-e", applescript], capture_output=True, @@ -767,8 +812,12 @@ def _show_results(pysus: PySUS, client: str) -> None: col1, col2 = st.columns([3, 1]) with col2: - if selected_indices and st.button(t("add_to_queue", _lang()), width="stretch"): - new_indices = [i for i in selected_indices if i not in queued_indices] + if selected_indices and st.button( + t("add_to_queue", _lang()), width="stretch" + ): + new_indices = [ + i for i in selected_indices if i not in queued_indices + ] if new_indices: st.session_state[queue_key] = sorted( set(queued_indices) | set(new_indices) @@ -809,7 +858,9 @@ def _show_results(pysus: PySUS, client: str) -> None: remove_indices = selection.selection.get("rows", []) # type: ignore dataset_name = st.session_state.get(f"_query_dataset_{client}", "").lower() - default_dir = str(CACHEPATH / "downloads" / client / (dataset_name or "data")) + default_dir = str( + CACHEPATH / "downloads" / client / (dataset_name or "data") + ) dir_key = f"_dl_dir_{client}" if dir_key not in st.session_state: @@ -849,7 +900,9 @@ def _show_results(pysus: PySUS, client: str) -> None: st.rerun() with col3: if st.button(t("download", _lang()), width="stretch", type="primary"): - _download_selected(pysus, client, download_queue, st.session_state[dir_key]) + _download_selected( + pysus, client, download_queue, st.session_state[dir_key] + ) st.rerun() @@ -878,9 +931,16 @@ async def _download(): ) try: await pysus.download(file=f) - except Exception as exc: + except Exception as exc: # noqa: B902 failed.add(f.basename) - st.error(t("download_failed", _lang(), name=f.basename, error=str(exc))) + st.error( + t( + "download_failed", + _lang(), + name=f.basename, + error=str(exc), + ) + ) _run_async(_download()) progress.empty() @@ -950,7 +1010,9 @@ def _fmt_args(exclude_dataset: bool = False) -> str: " ds = next(d for d in datasets if d.name == " + repr(params["dataset"]) + ")\n" - " files = await ds.search(\n" + _fmt_args(exclude_dataset=True) + "\n )\n" + " files = await ds.search(\n" + + _fmt_args(exclude_dataset=True) + + "\n )\n" " for f in files:\n" " print(f.basename)\n" " await pysus.download(file=f)\n" diff --git a/pysus/http/pages/2_examples.py b/pysus/web/pages/2_examples.py similarity index 100% rename from pysus/http/pages/2_examples.py rename to pysus/web/pages/2_examples.py diff --git a/pysus/http/translations.py b/pysus/web/translations.py similarity index 50% rename from pysus/http/translations.py rename to pysus/web/translations.py index 23cdb2f6..df08409f 100644 --- a/pysus/http/translations.py +++ b/pysus/web/translations.py @@ -8,12 +8,18 @@ "sidebar_title": "Datasets", "sidebar_select": "Select a dataset", "home_title": "PySUS", - "home_subtitle": "Tools for dealing with Brazil's Public health data (SUS — Sistema Único de Saúde).", + "home_subtitle": ( + "Tools for dealing with Brazil's Public health data" + " (SUS — Sistema Único de Saúde)." + ), "coming_soon": "coming soon", "datasets": "Datasets", "data_sources": "Data sources", "about_title": "About PySUS", - "about_intro": "PySUS v{version} — Tools for dealing with Brazil's Public health data (SUS — Sistema Único de Saúde).", + "about_intro": ( + "PySUS v{version} — Tools for dealing with Brazil's" + " Public health data (SUS — Sistema Único de Saúde)." + ), "sinan_desc": "Notifiable Diseases Information System", "sinasc_desc": "Live Births Information System", "sim_desc": "Mortality Information System", @@ -79,6 +85,84 @@ "download_failed": "Failed: {name} — {error}", "download_success": "Downloaded {count} file(s) to {dir}", "python_snippet": "📋 Python snippet", + "intro_welcome": "PySUS Web", + "intro_desc": ( + "This is a local web interface for browsing, querying," + " and downloading Brazil's public health datasets (SUS)." + " Use the sidebar to navigate between sections and select" + " your preferred data source." + ), + "intro_available": "Available Data Sources", + "intro_source_ducklake": ( + "**PySUS s3** — the default backend. Browse and query" + " datasets from the PySUS cloud data lake. Fast parquet" + " downloads with column selection. Best for programmatic" + " access and large-scale analysis." + ), + "intro_source_ftp": ( + "**FTP DataSUS** — the legacy DATASUS FTP server." + " Download raw DBF files for SINAN, SINASC, SIM, SIH," + " SIA, PNI, CNES, CIHA and more. Covers historical" + " data back to the 1990s." + ), + "intro_source_dadosgov": ( + "**API DataSUS** — the dados.gov.br open-data portal." + " Query datasets via REST API with metadata and" + " filtering. Requires a free API token from the" + " Brazilian government portal." + ), + "intro_package": ( + "PySUS is a Python package. For package documentation," + " API reference, programmatic usage, and this web server," + " visit [pysus.readthedocs.io](https://pysus.readthedocs.io)." + ), + "intro_github": ( + "Found an issue or want to contribute?" + " Visit the [GitHub repository]" + "(https://github.com/AlertaDengue/PySUS)." + ), + "databases_title": "Available Databases", + "databases_desc": ( + "PySUS provides access to the following DATASUS databases:" + ), + "databases_source": "Official source", + "db_sinan_desc": ( + "Notifiable Diseases Information System. Data on" + " dengue, zika, chikungunya, and other mandatory-report" + " diseases." + ), + "db_sinasc_desc": ( + "Live Births Information System. Birth records with" + " maternal, gestational, and neonatal data." + ), + "db_sim_desc": ( + "Mortality Information System. Death records with" + " cause, location, and demographic data." + ), + "db_sih_desc": ( + "Hospital Information System. Hospital admission" + " records funded by SUS." + ), + "db_sia_desc": ( + "Ambulatory Information System. Outpatient care" + " records from SUS providers." + ), + "db_pni_desc": ( + "National Immunization Program. Vaccination coverage" + " and doses administered across Brazil." + ), + "db_cnes_desc": ( + "National Registry of Health Facilities. Data on" + " hospitals, clinics, and health units." + ), + "db_ciha_desc": ( + "Hospital Admission Communication. Complementary" + " hospital admission and outpatient data." + ), + "db_ibge_desc": ( + "Brazilian Institute of Geography and Statistics." + " Population estimates and demographic data." + ), } PT: Final[dict[str, str]] = { @@ -87,12 +171,18 @@ "sidebar_title": "Bases de dados", "sidebar_select": "Selecione uma base", "home_title": "PySUS", - "home_subtitle": "Ferramentas para dados públicos de saúde do Brasil (SUS — Sistema Único de Saúde).", + "home_subtitle": ( + "Ferramentas para dados públicos de saúde do Brasil" + " (SUS — Sistema Único de Saúde)." + ), "coming_soon": "em breve", "datasets": "Bases de dados", "data_sources": "Fontes de dados", "about_title": "Sobre o PySUS", - "about_intro": "PySUS v{version} — Ferramentas para dados públicos de saúde do Brasil (SUS — Sistema Único de Saúde).", + "about_intro": ( + "PySUS v{version} — Ferramentas para dados públicos" + " de saúde do Brasil (SUS — Sistema Único de Saúde)." + ), "sinan_desc": "Sistema de Informação de Agravos de Notificação", "sinasc_desc": "Sistema de Informações sobre Nascidos Vivos", "sim_desc": "Sistema de Informação sobre Mortalidade", @@ -158,6 +248,86 @@ "download_failed": "Falha: {name} — {error}", "download_success": "{count} arquivo(s) baixado(s) para {dir}", "python_snippet": "📋 Código Python", + "intro_welcome": "PySUS Web", + "intro_desc": ( + "Esta é uma interface web local para navegar, consultar" + " e baixar bases de dados públicos de saúde do Brasil" + " (SUS). Use a barra lateral para navegar entre as" + " seções e selecionar sua fonte de dados preferida." + ), + "intro_available": "Fontes de Dados Disponíveis", + "intro_source_ducklake": ( + "**PySUS s3** — o backend padrão. Navegue e consulte" + " bases do data lake do PySUS. Downloads rápidos em" + " parquet com seleção de colunas. Ideal para acesso" + " programático e análises em larga escala." + ), + "intro_source_ftp": ( + "**FTP DataSUS** — o servidor FTP legado do DATASUS." + " Baixe arquivos DBF brutos de SINAN, SINASC, SIM," + " SIH, SIA, PNI, CNES, CIHA e outros. Cobre dados" + " históricos desde a década de 1990." + ), + "intro_source_dadosgov": ( + "**API DataSUS** — o portal de dados abertos" + " dados.gov.br. Consulte bases via API REST com" + " metadados e filtros. Requer um token gratuito de" + " API do portal do governo brasileiro." + ), + "intro_package": ( + "PySUS é um pacote Python. Para documentação do pacote," + " referência da API, uso programático e este servidor" + " web, acesse [pysus.readthedocs.io]" + "(https://pysus.readthedocs.io)." + ), + "intro_github": ( + "Encontrou um problema ou quer contribuir?" + " Acesse o [repositório no GitHub]" + "(https://github.com/AlertaDengue/PySUS)." + ), + "databases_title": "Bases de Dados Disponíveis", + "databases_desc": "O PySUS oferece acesso às seguintes bases do DATASUS:", + "databases_source": "Fonte oficial", + "db_sinan_desc": ( + "Sistema de Informação de Agravos de Notificação." + " Dados de dengue, zika, chikungunya e outras doenças" + " de notificação obrigatória." + ), + "db_sinasc_desc": ( + "Sistema de Informações sobre Nascidos Vivos." + " Registros de nascimento com dados maternos," + " gestacionais e neonatais." + ), + "db_sim_desc": ( + "Sistema de Informação sobre Mortalidade. Registros" + " de óbito com dados de causa, local e perfil" + " demográfico." + ), + "db_sih_desc": ( + "Sistema de Informações Hospitalares. Registros de" + " internações financiadas pelo SUS." + ), + "db_sia_desc": ( + "Sistema de Informações Ambulatoriais. Registros de" + " atendimento ambulatorial do SUS." + ), + "db_pni_desc": ( + "Programa Nacional de Imunizações. Cobertura vacinal" + " e doses aplicadas em todo o Brasil." + ), + "db_cnes_desc": ( + "Cadastro Nacional de Estabelecimentos de Saúde." + " Dados de hospitais, clínicas e unidades de saúde." + ), + "db_ciha_desc": ( + "Comunicação de Informação Hospitalar e Ambulatorial." + " Dados complementares de internação e atendimento" + " ambulatorial." + ), + "db_ibge_desc": ( + "Instituto Brasileiro de Geografia e Estatística." + " Estimativas populacionais e dados demográficos." + ), } TRANSLATIONS: Final[dict[str, dict[str, str]]] = {