From 7be4afe00c58695ebb17a30300b7453e5f482a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 15:41:16 +0200 Subject: [PATCH 1/9] docs: upgrade docstrings as a first-class documentation surface for AI agents Coding agents treat the installed package as primary documentation (help(), inspect, grepping site-packages), so this improves the docstrings they hit most, verified against code and tests: - propagate_attributes: fix broken examples that called it as a client method (it is a top-level import) and used the nonexistent start_generation; document wrap-the-root-span ordering; add See also - Langfuse.api/async_api: new docstrings covering list-vs-get semantics (list endpoints return observation/score IDs as strings), asynchronous ingestion with a bounded retry snippet, and scores read methods (get_many/get_by_id; there is no scores.get) - flush(): note that server-side ingestion lags flush by a few seconds - @observe: document capture_input/capture_output/transform_to_string, as_type="generation" with update_current_generation example - run_experiment/Evaluation: add RAG LLM-as-a-judge example (faithfulness, answer relevance) and CI regression testing via langfuse/experiment-action; docstring for RunnerContext.run_experiment - package docstring/README/pyproject: lead with full capability map, current env var names, llms.txt pointer; fix stale v3 docs link and broken langfuse.otel import in the Langfuse class example - deprecation hygiene: emit DeprecationWarning when host= or LANGFUSE_HOST is the effective base URL source (behavior change); deprecation messages now name the exact replacement import Generated code under langfuse/api/ is untouched; list-vs-get and ingestion notes for it live on the non-generated api property and should also be mirrored into the fern spec in langfuse/langfuse. Co-Authored-By: Claude Fable 5 --- README.md | 21 ++++- langfuse/__init__.py | 42 ++++++++- langfuse/_client/client.py | 149 +++++++++++++++++++++++++++++++- langfuse/_client/observe.py | 39 ++++++++- langfuse/_client/propagation.py | 36 +++++--- langfuse/_client/span.py | 5 +- langfuse/experiment.py | 21 +++++ pyproject.toml | 2 +- 8 files changed, 291 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index de79a88fb..7beb0a778 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ [![Discord](https://img.shields.io/discord/1111061815649124414?style=flat-square&logo=Discord&logoColor=white&label=Discord&color=%23434EE4)](https://discord.gg/7NXusRtqYU) [![YC W23](https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square)](https://www.ycombinator.com/companies/langfuse) +The [Langfuse](https://langfuse.com) Python SDK covers the full platform, not just tracing: **observability/tracing** (OpenTelemetry-based, with OpenAI and LangChain integrations), **datasets & experiments** (offline evaluation and regression testing of prompt/model changes, including [CI via GitHub Actions](https://github.com/langfuse/experiment-action)), **LLM-as-a-judge and custom evaluations/scores**, **prompt management**, and a **full REST API client**. + ## Installation > [!IMPORTANT] @@ -18,6 +20,23 @@ pip install langfuse ``` +## Quickstart + +```python +# env: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL +from langfuse import get_client, observe + +@observe() +def handle(query: str) -> str: + return "answer" + +handle("hello") +get_client().flush() +``` + ## Docs -Please [see our docs](https://langfuse.com/docs/sdk/python/sdk-v3) for detailed information on this SDK. +- SDK guide: https://langfuse.com/docs/observability/sdk/overview +- Full documentation: https://langfuse.com/docs +- Machine-readable docs index (for AI agents): https://langfuse.com/llms.txt +- API reference of this package: https://python.reference.langfuse.com diff --git a/langfuse/__init__.py b/langfuse/__init__.py index d4d24cd04..6d805e232 100644 --- a/langfuse/__init__.py +++ b/langfuse/__init__.py @@ -1,4 +1,44 @@ -""".. include:: ../README.md""" +"""Langfuse Python SDK — observability, evaluation, and prompt management for LLM applications. + +Capabilities (all in this package, not just tracing): + +- **Tracing / observability**: `@observe` decorator, `Langfuse.start_observation` / + `start_as_current_observation` context managers, OpenTelemetry-based; integrations + for OpenAI (`langfuse.openai`) and LangChain (`langfuse.langchain.CallbackHandler`). +- **Trace attributes**: `propagate_attributes` (top-level function) sets user_id, + session_id, tags, and metadata on all spans in a context. +- **Datasets & experiments**: `Langfuse.get_dataset`, `Langfuse.run_experiment` for + offline evaluation and regression testing of prompt/model changes (CI support via + https://github.com/langfuse/experiment-action and `RegressionError`). +- **Evaluation / LLM-as-a-judge**: `Evaluation` results from custom or model-based + evaluators; scores via `Langfuse.create_score` / `span.score`. +- **Prompt management**: `Langfuse.get_prompt`, `Langfuse.create_prompt` with + client-side caching and version/label control. +- **Full REST API**: `Langfuse.api` (sync) / `Langfuse.async_api` (async) clients. + +Quickstart: + +```python +# env: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL +from langfuse import get_client, observe + +@observe() +def handle(query: str) -> str: + return "answer" + +handle("hello") +get_client().flush() +``` + +Configuration is via constructor args or environment variables: `LANGFUSE_PUBLIC_KEY`, +`LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL` (defaults to https://cloud.langfuse.com; +the older `LANGFUSE_HOST` is deprecated). See `langfuse._client.environment_variables` +for the full list. + +Docs: https://langfuse.com/docs — machine-readable index: https://langfuse.com/llms.txt + +.. include:: ../README.md +""" from langfuse.batch_evaluation import ( BatchEvaluationResult, diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 6ce72d27a..b4eac19f3 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -248,13 +248,13 @@ def mask_otel_spans( Example: ```python - from langfuse.otel import Langfuse + from langfuse import Langfuse # Initialize the client (reads from env vars if not provided) langfuse = Langfuse( public_key="your-public-key", secret_key="your-secret-key", - host="https://cloud.langfuse.com", # Optional, default shown + base_url="https://cloud.langfuse.com", # Optional, default shown ) # Create a trace span @@ -312,6 +312,20 @@ def __init__( id_generator: Optional[IdGenerator] = None, span_exporter: Optional[SpanExporter] = None, ): + if base_url is None and os.environ.get(LANGFUSE_BASE_URL) is None: + if host is not None: + warnings.warn( + "The 'host' parameter is deprecated. Use 'base_url' (or the LANGFUSE_BASE_URL environment variable) instead.", + DeprecationWarning, + stacklevel=2, + ) + elif os.environ.get(LANGFUSE_HOST) is not None: + warnings.warn( + "The LANGFUSE_HOST environment variable is deprecated. Use LANGFUSE_BASE_URL instead.", + DeprecationWarning, + stacklevel=2, + ) + self._base_url = ( base_url or os.environ.get(LANGFUSE_BASE_URL) @@ -421,6 +435,55 @@ def __init__( @property def api(self) -> LangfuseAPI: + """Synchronous client for the full Langfuse REST API (traces, observations, scores, datasets, prompts, ...). + + Use this to read or manage data on the Langfuse server; use the tracing methods + (`start_observation`, `@observe`) to create traces. Use `async_api` for the + asyncio variant. + + Semantics that are easy to miss: + + - **Ingestion is asynchronous.** Traces are exported in the background and + processed server-side. Even after `langfuse.flush()`, reads such as + `api.trace.get(trace_id)` may raise `langfuse.api.NotFoundError` for a few + seconds. The same applies to scores and dataset run reads. Retry with a bound: + + ```python + import time + from langfuse.api import NotFoundError + + langfuse.flush() + for attempt in range(10): + try: + trace = langfuse.api.trace.get(trace_id) + break + except NotFoundError: + time.sleep(2 ** attempt * 0.5) # 0.5s, 1s, 2s, ... + ``` + + - **List endpoints return lightweight views.** `api.trace.list(...)` returns + `TraceWithDetails`, where `observations` and `scores` are lists of ID strings. + Fetch the full objects with `api.trace.get(trace_id)` + (`TraceWithFullDetails`, with full observation and score objects). The same + list-view vs. get-detail pattern applies to other resources. + + - **Scores are read with `api.scores.get_many(...)` (paginated, filterable) or + `api.scores.get_by_id(score_id)`.** There is no `api.scores.get`. Scores are + written via `langfuse.create_score(...)` / `span.score(...)`, which are queued + and flushed in the background — not via this API client. + + Example: + ```python + from langfuse import get_client + + langfuse = get_client() + traces = langfuse.api.trace.list(limit=10) # lightweight views + full = langfuse.api.trace.get(traces.data[0].id) # full observations/scores + ``` + + See also: `async_api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk + and https://langfuse.com/docs/api-and-data-platform/features/public-api + """ if self._resources is None: raise AttributeError("Langfuse client is not initialized") @@ -435,6 +498,22 @@ def api(self, value: LangfuseAPI) -> None: @property def async_api(self) -> AsyncLangfuseAPI: + """Asynchronous (asyncio) client for the full Langfuse REST API. + + Same resources and semantics as `api` — including asynchronous ingestion + (reads may raise `langfuse.api.NotFoundError` for a few seconds after + `flush()`) and lightweight list views (`list` endpoints return observation + and score IDs as strings; `get` endpoints return full objects). All methods + must be awaited. + + Example: + ```python + async def recent_traces(): + return await langfuse.async_api.trace.list(limit=10) + ``` + + See also: `api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk + """ if self._resources is None: raise AttributeError("Langfuse client is not initialized") @@ -1502,7 +1581,8 @@ def update_current_span( @deprecated( "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " + "For trace attributes (user_id, session_id, tags, etc.), use the top-level " + "`from langfuse import propagate_attributes` context manager instead. " "This method will be removed in a future major version." ) def set_current_trace_io( @@ -1519,7 +1599,7 @@ def set_current_trace_io( evaluators). It will be removed in a future major version. For setting other trace attributes (user_id, session_id, metadata, tags, version), - use :meth:`propagate_attributes` instead. + use :func:`langfuse.propagate_attributes` (top-level import) instead. Args: input: Input data to associate with the trace. @@ -2239,6 +2319,12 @@ def flush(self) -> None: # Continue with other work ``` + + Note: + `flush()` blocks until pending data is *sent*, but server-side ingestion + is asynchronous: flushed traces may not be queryable via `api.trace.get()` + for a few seconds (a `langfuse.api.NotFoundError` right after a successful + flush is expected). See the `api` property docs for a bounded retry pattern. """ if self._resources is not None: self._resources.flush() @@ -2671,6 +2757,49 @@ def average_accuracy(*, item_results, **kwargs): ) ``` + LLM-as-a-judge evaluation of a RAG pipeline (faithfulness and answer relevance): + ```python + from langfuse import Evaluation + + def rag_task(*, item, **kwargs): + question = item["input"] + context = retrieve(question) # your retriever + answer = generate(question, context) # your LLM call + return {"answer": answer, "context": context} + + def llm_judge(*, prompt): + response = openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": prompt}], + ) + return float(response.choices[0].message.content.strip()) + + def faithfulness_evaluator(*, input, output, **kwargs): + # Is the answer grounded in the retrieved context, without hallucination? + score = llm_judge( + prompt=f"Rate 0-1 how faithful the answer is to the context.\\n" + f"Context: {output['context']}\\nAnswer: {output['answer']}\\n" + f"Respond with only the number." + ) + return Evaluation(name="faithfulness", value=score) + + def answer_relevance_evaluator(*, input, output, **kwargs): + # Does the answer address the question that was asked? + score = llm_judge( + prompt=f"Rate 0-1 how relevant the answer is to the question.\\n" + f"Question: {input}\\nAnswer: {output['answer']}\\n" + f"Respond with only the number." + ) + return Evaluation(name="answer_relevance", value=score) + + result = langfuse.run_experiment( + name="RAG quality", + data=langfuse.get_dataset("rag-eval-set").items, + task=rag_task, + evaluators=[faithfulness_evaluator, answer_relevance_evaluator], + ) + ``` + Using with Langfuse datasets: ```python # Get dataset from Langfuse @@ -2694,6 +2823,18 @@ def average_accuracy(*, item_results, **kwargs): - When using Langfuse datasets, results are automatically linked for easy comparison - This method works in both sync and async contexts (Jupyter notebooks, web apps, etc.) - Async execution is handled automatically with smart event loop detection + - **Regression testing in CI**: run experiments to compare prompt or model + versions before shipping them. The ``langfuse/experiment-action`` GitHub + Action (https://github.com/langfuse/experiment-action) runs experiments in + CI, comments results on PRs, and can fail the workflow via + `langfuse.RegressionError` when a metric drops below a threshold. + + See also: + `Evaluation` (return type for evaluators), `langfuse.RegressionError` (CI gating), + `DatasetClient.run_experiment` (dataset-linked variant), + https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk, + https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd, + https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge """ return cast( ExperimentResult, diff --git a/langfuse/_client/observe.py b/langfuse/_client/observe.py index 64882a20f..3c3006f73 100644 --- a/langfuse/_client/observe.py +++ b/langfuse/_client/observe.py @@ -109,8 +109,18 @@ def observe( as_type (Optional[Literal]): Set the observation type. Supported values: "generation", "span", "agent", "tool", "chain", "retriever", "embedding", "evaluator", "guardrail". Observation types are highlighted in the Langfuse UI for filtering and visualization. - The types "generation" and "embedding" create a span on which additional attributes such as model metrics - can be set. + The types "generation" and "embedding" create a span on which additional attributes such as model, + usage_details, and cost_details can be set — use `as_type="generation"` for LLM calls and update the + observation via `langfuse.update_current_generation(...)` inside the function. + capture_input (Optional[bool]): Whether to capture the function's arguments as the observation's input. + Defaults to the LANGFUSE_OBSERVE_DECORATOR_IO_CAPTURE_ENABLED environment variable (True if unset). + Set to False for sensitive or very large inputs, then set input explicitly via + `langfuse.update_current_span(input=...)` if needed. + capture_output (Optional[bool]): Whether to capture the function's return value as the observation's output. + Same default and override mechanism as capture_input. + transform_to_string (Optional[Callable[[Iterable], str]]): For functions returning generators, joins the + yielded chunks into the string stored as output. Without it, chunks are concatenated if all are + strings, otherwise stored as a list. Returns: Callable: A wrapped version of the original function that automatically creates and manages Langfuse spans. @@ -126,16 +136,32 @@ def process_user_request(user_id, query): For language model generation tracking: ```python + from langfuse import get_client, observe + @observe(name="answer-generation", as_type="generation") async def generate_answer(query): - # Creates a generation-type span with extended LLM metrics + # Creates a generation-type observation with extended LLM metrics response = await openai.chat.completions.create( model="gpt-4", messages=[{"role": "user", "content": query}] ) + get_client().update_current_generation( + model="gpt-4", + usage_details={ + "input": response.usage.prompt_tokens, + "output": response.usage.completion_tokens, + }, + ) return response.choices[0].message.content ``` + Disabling input/output capture (e.g. for sensitive or large payloads): + ```python + @observe(capture_input=False, capture_output=False) + def handle_pii(user_record): + return process(user_record) + ``` + For trace context propagation between functions: ```python @observe() @@ -161,6 +187,13 @@ def sub_process(): - langfuse_public_key: Use a specific Langfuse project (when multiple clients exist) - For async functions, the decorator returns an async function wrapper. - For sync functions, the decorator returns a synchronous wrapper. + - For generator functions, the observation stays open until the generator is + exhausted; yielded chunks are captured as output (see transform_to_string). + + See also: + `langfuse.propagate_attributes` (set user_id/session_id on all spans), + `Langfuse.start_as_current_observation` (imperative alternative), + https://langfuse.com/docs/observability/sdk/instrumentation """ valid_types = set(get_observation_types_list(ObservationTypeLiteralNoEvent)) if as_type is not None and as_type not in valid_types: diff --git a/langfuse/_client/propagation.py b/langfuse/_client/propagation.py index d97f33317..684157b60 100644 --- a/langfuse/_client/propagation.py +++ b/langfuse/_client/propagation.py @@ -131,9 +131,13 @@ def propagate_attributes( environment, and metadata dimensions that should be consistently applied across all observations in a trace. - **IMPORTANT**: Call this as early as possible within your trace/workflow. Only the - currently active span and spans created after entering this context will have these - attributes. Pre-existing spans will NOT be retroactively updated. + This is a module-level function, not a method on the Langfuse client: + import it with `from langfuse import propagate_attributes`. + + **IMPORTANT**: Call this as early as possible within your trace/workflow — + ideally wrapping the creation of your root span, or immediately inside it. Only + the currently active span and spans created after entering this context will have + these attributes. Pre-existing spans will NOT be retroactively updated. **Why this matters**: Langfuse aggregation queries (e.g., total cost by user_id, filtering by session_id) only include observations that have the attribute set. @@ -188,16 +192,17 @@ def propagate_attributes( Context manager that propagates attributes to all child spans. Example: - Basic usage with user and session tracking: + Basic usage with user and session tracking (note: `propagate_attributes` is a + top-level import, not a client method): ```python - from langfuse import Langfuse + from langfuse import Langfuse, propagate_attributes langfuse = Langfuse() - # Set attributes early in the trace + # Set attributes early: wrap everything inside the root span with langfuse.start_as_current_observation(name="user_workflow") as span: - with langfuse.propagate_attributes( + with propagate_attributes( user_id="user_123", session_id="session_abc", environment="production", @@ -208,7 +213,9 @@ def propagate_attributes( # This span inherits user_id, session_id, environment, and experiment metadata ... - with langfuse.start_generation(name="completion") as gen: + with langfuse.start_observation( + name="completion", as_type="generation" + ) as gen: # This span also inherits all attributes ... ``` @@ -216,12 +223,12 @@ def propagate_attributes( Prompt linking with auto-instrumented libraries: ```python - from langfuse import Langfuse + from langfuse import Langfuse, propagate_attributes langfuse = Langfuse() prompt = langfuse.get_prompt("my-prompt") - with langfuse.propagate_attributes(prompt=prompt): + with propagate_attributes(prompt=prompt): # Generations emitted by auto-instrumentation (LiteLLM langfuse_otel, # OpenAI Agents SDK, OpenInference, ...) within this context are # linked to the prompt version. @@ -240,7 +247,7 @@ def propagate_attributes( early_span.end() # Set attributes in the middle - with langfuse.propagate_attributes(user_id="user_123"): + with propagate_attributes(user_id="user_123"): # Only spans created AFTER this point will have user_id late_span = langfuse.start_observation(name="late_work") late_span.end() @@ -253,7 +260,7 @@ def propagate_attributes( ```python # Service A - originating service with langfuse.start_as_current_observation(name="api_request"): - with langfuse.propagate_attributes( + with propagate_attributes( user_id="user_123", session_id="session_abc", environment="staging", @@ -282,6 +289,11 @@ def propagate_attributes( Raises: No exceptions are raised. Invalid values are logged as warnings and dropped. + + See also: + `Langfuse.start_as_current_observation` (create the root span this wraps), + https://langfuse.com/docs/observability/features/sessions, + https://langfuse.com/docs/observability/features/users """ return _propagate_attributes( user_id=user_id, diff --git a/langfuse/_client/span.py b/langfuse/_client/span.py index 828c7590a..8c257e92f 100644 --- a/langfuse/_client/span.py +++ b/langfuse/_client/span.py @@ -228,7 +228,8 @@ def end(self, *, end_time: Optional[int] = None) -> "LangfuseObservationWrapper" @deprecated( "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " + "For trace attributes (user_id, session_id, tags, etc.), use the top-level " + "`from langfuse import propagate_attributes` context manager instead. " "This method will be removed in a future major version." ) def set_trace_io( @@ -245,7 +246,7 @@ def set_trace_io( evaluators). It will be removed in a future major version. For setting other trace attributes (user_id, session_id, metadata, tags, version), - use :meth:`Langfuse.propagate_attributes` instead. + use :func:`langfuse.propagate_attributes` (top-level import) instead. Args: input: Input data to associate with the trace. diff --git a/langfuse/experiment.py b/langfuse/experiment.py index 404c96e1d..148cf928f 100644 --- a/langfuse/experiment.py +++ b/langfuse/experiment.py @@ -183,6 +183,16 @@ def external_api_evaluator(*, input, output, **kwargs): Note: All arguments must be passed as keywords. Positional arguments are not allowed to ensure code clarity and prevent errors from argument reordering. + + Evaluators commonly wrap an LLM-as-a-judge call — e.g. scoring RAG + faithfulness or answer relevance with a grading prompt — and return the + judge's verdict as an ``Evaluation`` (see :meth:`Langfuse.run_experiment` + for a complete RAG example). + + See also: + `Langfuse.run_experiment` and `RegressionError` (regression-gate prompt/model + changes in CI via https://github.com/langfuse/experiment-action), + https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge """ def __init__( @@ -1122,6 +1132,17 @@ def run_experiment( metadata: Optional[Dict[str, str]] = None, _dataset_version: Optional[datetime] = None, ) -> ExperimentResult: + """Run an experiment, filling in CI-injected defaults for omitted arguments. + + Same signature and semantics as :meth:`Langfuse.run_experiment`, except + `data` and `metadata` fall back to the defaults injected by the + ``langfuse/experiment-action`` GitHub Action (explicit arguments win; + metadata dicts are merged with user keys taking precedence). Raise + :class:`RegressionError` from your experiment function to fail the CI + gate when a metric regresses. + + See also: https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd + """ resolved_data = data if data is not None else self.data if resolved_data is None: raise ValueError( diff --git a/pyproject.toml b/pyproject.toml index 741627b1f..1d63c05b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "langfuse" version = "4.14.0" -description = "A client library for accessing langfuse" +description = "Langfuse Python SDK - LLM observability/tracing, datasets, experiments, LLM-as-a-judge evaluation, and prompt management" readme = "README.md" authors = [{ name = "langfuse", email = "developers@langfuse.com" }] license = "MIT" From 0a6479a2982a4633e309d24bf0b0670cdcbec012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 15:48:18 +0200 Subject: [PATCH 2/9] docs: remove runtime DeprecationWarning for host/LANGFUSE_HOST Keep the change docs-only; the deprecation remains documented in the host parameter docstring and the LANGFUSE_HOST env var description. Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index b4eac19f3..b6361bd34 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -312,20 +312,6 @@ def __init__( id_generator: Optional[IdGenerator] = None, span_exporter: Optional[SpanExporter] = None, ): - if base_url is None and os.environ.get(LANGFUSE_BASE_URL) is None: - if host is not None: - warnings.warn( - "The 'host' parameter is deprecated. Use 'base_url' (or the LANGFUSE_BASE_URL environment variable) instead.", - DeprecationWarning, - stacklevel=2, - ) - elif os.environ.get(LANGFUSE_HOST) is not None: - warnings.warn( - "The LANGFUSE_HOST environment variable is deprecated. Use LANGFUSE_BASE_URL instead.", - DeprecationWarning, - stacklevel=2, - ) - self._base_url = ( base_url or os.environ.get(LANGFUSE_BASE_URL) From 04bdfa50f592d5f85ce04b8faf71d298515f68f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 15:54:51 +0200 Subject: [PATCH 3/9] docs: align docstrings with langfuse.com docs wording and links Cross-checked against the langfuse-docs repo: - api/async_api/flush: align ingestion-lag wording with query-via-sdk docs (typically 15-30s, not "a few seconds"), replace retry snippet with the documented deadline-based pattern (capped backoff), note that a successful trace.get does not imply observations/scores are complete, recommend observations.get_many(trace_id=...) for row-level reads and the Metrics API for aggregation; add anchored doc links - run_experiment RAG example: use item.input (DatasetItem attribute) instead of item["input"], matching the documented dataset task shape - RegressionError: correct action input name to should_fail_on_regression (was should_fail_on_error; verified against experiment-action action.yml) - add See-also links: observation-types (observe/as_type) and environments (propagate_attributes) All 10 docstring URLs verified to map to content pages in langfuse-docs. Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 58 +++++++++++++++++++++------------ langfuse/_client/observe.py | 3 +- langfuse/_client/propagation.py | 3 +- langfuse/experiment.py | 4 +-- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index b6361bd34..2b90c6e2b 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -429,35 +429,46 @@ def api(self) -> LangfuseAPI: Semantics that are easy to miss: - - **Ingestion is asynchronous.** Traces are exported in the background and - processed server-side. Even after `langfuse.flush()`, reads such as - `api.trace.get(trace_id)` may raise `langfuse.api.NotFoundError` for a few - seconds. The same applies to scores and dataset run reads. Retry with a bound: + - **Ingestion is asynchronous.** `langfuse.flush()` only guarantees delivery to + the API, not read visibility: reads such as `api.trace.get(trace_id)` may + raise `langfuse.api.NotFoundError` until processing completes (typically + within 15-30 seconds; longer under load). The same applies to scores and + dataset run reads. Instead of a fixed sleep, retry with a deadline: ```python import time from langfuse.api import NotFoundError - langfuse.flush() - for attempt in range(10): + langfuse.flush() # ensure delivery before polling + deadline, delay = time.monotonic() + 60.0, 1.0 + while True: try: trace = langfuse.api.trace.get(trace_id) break except NotFoundError: - time.sleep(2 ** attempt * 0.5) # 0.5s, 1s, 2s, ... + if time.monotonic() >= deadline: + raise + time.sleep(delay) + delay = min(delay * 2, 10.0) # exponential backoff, capped ``` + A successful `trace.get` means the trace record exists — its observations and + scores may still be arriving. + - **List endpoints return lightweight views.** `api.trace.list(...)` returns `TraceWithDetails`, where `observations` and `scores` are lists of ID strings. - Fetch the full objects with `api.trace.get(trace_id)` - (`TraceWithFullDetails`, with full observation and score objects). The same - list-view vs. get-detail pattern applies to other resources. + Fetch the full objects with `api.trace.get(trace_id)` (`TraceWithFullDetails`), + or prefer `api.observations.get_many(trace_id=...)` for row-level observation + queries. The same list-view vs. get-detail pattern applies to other resources. - **Scores are read with `api.scores.get_many(...)` (paginated, filterable) or `api.scores.get_by_id(score_id)`.** There is no `api.scores.get`. Scores are written via `langfuse.create_score(...)` / `span.score(...)`, which are queued and flushed in the background — not via this API client. + For large-scale aggregation (usage/cost by model, user, etc.), prefer the + Metrics API (`api.metrics.metrics(...)`) over paginating row-level data. + Example: ```python from langfuse import get_client @@ -467,8 +478,11 @@ def api(self) -> LangfuseAPI: full = langfuse.api.trace.get(traces.data[0].id) # full observations/scores ``` - See also: `async_api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk - and https://langfuse.com/docs/api-and-data-platform/features/public-api + See also: `async_api`, + https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk + (ingestion lag: #ingestion-lag, list vs. get: #traces-list-vs-get), + https://langfuse.com/docs/api-and-data-platform/features/observations-api, + https://langfuse.com/docs/metrics/features/metrics-api """ if self._resources is None: raise AttributeError("Langfuse client is not initialized") @@ -487,10 +501,10 @@ def async_api(self) -> AsyncLangfuseAPI: """Asynchronous (asyncio) client for the full Langfuse REST API. Same resources and semantics as `api` — including asynchronous ingestion - (reads may raise `langfuse.api.NotFoundError` for a few seconds after - `flush()`) and lightweight list views (`list` endpoints return observation - and score IDs as strings; `get` endpoints return full objects). All methods - must be awaited. + (reads may raise `langfuse.api.NotFoundError` until processing completes, + typically within 15-30 seconds of `flush()`) and lightweight list views + (`list` endpoints return observation and score IDs as strings; `get` + endpoints return full objects). All methods must be awaited. Example: ```python @@ -2307,10 +2321,12 @@ def flush(self) -> None: ``` Note: - `flush()` blocks until pending data is *sent*, but server-side ingestion - is asynchronous: flushed traces may not be queryable via `api.trace.get()` - for a few seconds (a `langfuse.api.NotFoundError` right after a successful - flush is expected). See the `api` property docs for a bounded retry pattern. + `flush()` guarantees data was *delivered* to the API, not that it is + *readable* yet: server-side ingestion is asynchronous, so flushed traces + may not be queryable via `api.trace.get()` for 15-30 seconds (a + `langfuse.api.NotFoundError` right after a successful flush is expected). + See the `api` property docs for a bounded retry pattern, or + https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk#ingestion-lag """ if self._resources is not None: self._resources.flush() @@ -2748,7 +2764,7 @@ def average_accuracy(*, item_results, **kwargs): from langfuse import Evaluation def rag_task(*, item, **kwargs): - question = item["input"] + question = item.input # DatasetItem attribute (dict items: item["input"]) context = retrieve(question) # your retriever answer = generate(question, context) # your LLM call return {"answer": answer, "context": context} diff --git a/langfuse/_client/observe.py b/langfuse/_client/observe.py index 3c3006f73..1377b4e41 100644 --- a/langfuse/_client/observe.py +++ b/langfuse/_client/observe.py @@ -193,7 +193,8 @@ def sub_process(): See also: `langfuse.propagate_attributes` (set user_id/session_id on all spans), `Langfuse.start_as_current_observation` (imperative alternative), - https://langfuse.com/docs/observability/sdk/instrumentation + https://langfuse.com/docs/observability/sdk/instrumentation, + https://langfuse.com/docs/observability/features/observation-types (as_type values) """ valid_types = set(get_observation_types_list(ObservationTypeLiteralNoEvent)) if as_type is not None and as_type not in valid_types: diff --git a/langfuse/_client/propagation.py b/langfuse/_client/propagation.py index 684157b60..6eaa2819d 100644 --- a/langfuse/_client/propagation.py +++ b/langfuse/_client/propagation.py @@ -293,7 +293,8 @@ def propagate_attributes( See also: `Langfuse.start_as_current_observation` (create the root span this wraps), https://langfuse.com/docs/observability/features/sessions, - https://langfuse.com/docs/observability/features/users + https://langfuse.com/docs/observability/features/users, + https://langfuse.com/docs/observability/features/environments """ return _propagate_attributes( user_id=user_id, diff --git a/langfuse/experiment.py b/langfuse/experiment.py index 148cf928f..3dfe4e8dc 100644 --- a/langfuse/experiment.py +++ b/langfuse/experiment.py @@ -1179,8 +1179,8 @@ class RegressionError(Exception): Intended for use with the ``langfuse/experiment-action`` GitHub Action (https://github.com/langfuse/experiment-action). The action catches this - exception and, when ``should_fail_on_error`` is enabled, fails the - workflow run and renders a callout in the PR comment using + exception and, when ``should_fail_on_regression`` is enabled (default), fails + the workflow run and renders a callout in the PR comment using ``metric``/``value``/``threshold`` if supplied, otherwise ``str(exc)``. Callers choose one of three forms: From acc8c837e1cdd76dade5a606e36f51c6008ef32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 15:59:59 +0200 Subject: [PATCH 4/9] docs: highlight v2 data APIs as recommended, correct score read guidance - api property: call out api.observations and api.metrics as the recommended high-performance v2 endpoints (SDK v4 defaults) and note their v1 equivalents under api.legacy.* are less performant, not recommended for new workflows, and will be deprecated - scores: point reads at api.scores_v3.get_many_v3; the v2 api.scores.get_many/get_by_id are deprecated per the API spec and no longer available on Langfuse v4+ servers Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 2b90c6e2b..a622336ba 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -461,13 +461,22 @@ def api(self) -> LangfuseAPI: or prefer `api.observations.get_many(trace_id=...)` for row-level observation queries. The same list-view vs. get-detail pattern applies to other resources. - - **Scores are read with `api.scores.get_many(...)` (paginated, filterable) or - `api.scores.get_by_id(score_id)`.** There is no `api.scores.get`. Scores are - written via `langfuse.create_score(...)` / `span.score(...)`, which are queued - and flushed in the background — not via this API client. + - **Prefer the v2 data APIs — they are the defaults since SDK v4.** + `api.observations` and `api.metrics` map to the high-performance + `/api/public/v2/...` endpoints and are the recommended read path. Their v1 + equivalents remain available under `api.legacy.observations_v1` / + `api.legacy.metrics_v1` but are less performant at scale, not recommended + for new workflows, and will be deprecated. + + - **Score reads: use `api.scores_v3.get_many_v3(...)`** (cursor-paginated, + filterable; pass `id=...` to fetch a single score). The older + `api.scores.get_many(...)` / `api.scores.get_by_id(...)` (v2) are deprecated + and no longer available on Langfuse v4+ servers. There is no `api.scores.get`. + Scores are written via `langfuse.create_score(...)` / `span.score(...)`, which + are queued and flushed in the background — not via this API client. For large-scale aggregation (usage/cost by model, user, etc.), prefer the - Metrics API (`api.metrics.metrics(...)`) over paginating row-level data. + v2 Metrics API (`api.metrics.metrics(...)`) over paginating row-level data. Example: ```python From 771119c54dc8aee03562ff52055d07319b71b60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 18:14:51 +0200 Subject: [PATCH 5/9] =?UTF-8?q?docs:=20address=20PR=20review=20=E2=80=94?= =?UTF-8?q?=20runnable=20propagation=20example,=20observations-first=20exa?= =?UTF-8?q?mples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - propagate_attributes example: start_observation() returns a plain observation (no __enter__), so both `with` usages crashed at runtime; switch to start_as_current_observation (smoke-ran the example) - api/async_api/flush docstrings: use the v2 observations endpoint (api.observations.get_many with fields selection) as the primary example per review, matching the query-via-sdk docs page examples Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 32 ++++++++++++++++++++++++-------- langfuse/_client/propagation.py | 4 ++-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index a622336ba..3f8c1f348 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -483,8 +483,17 @@ def api(self) -> LangfuseAPI: from langfuse import get_client langfuse = get_client() - traces = langfuse.api.trace.list(limit=10) # lightweight views - full = langfuse.api.trace.get(traces.data[0].id) # full observations/scores + + # Row-level reads: prefer the v2 observations endpoint + observations = langfuse.api.observations.get_many( + trace_id="abcdef1234", + type="GENERATION", + limit=100, + fields="core,basic,usage", + ) + + # Full single-trace tree (observations/scores as full objects) + full_trace = langfuse.api.trace.get("abcdef1234") ``` See also: `async_api`, @@ -517,8 +526,13 @@ def async_api(self) -> AsyncLangfuseAPI: Example: ```python - async def recent_traces(): - return await langfuse.async_api.trace.list(limit=10) + async def trace_generations(trace_id: str): + # Row-level reads: prefer the v2 observations endpoint + return await langfuse.async_api.observations.get_many( + trace_id=trace_id, + type="GENERATION", + fields="core,basic,usage", + ) ``` See also: `api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk @@ -2331,10 +2345,12 @@ def flush(self) -> None: Note: `flush()` guarantees data was *delivered* to the API, not that it is - *readable* yet: server-side ingestion is asynchronous, so flushed traces - may not be queryable via `api.trace.get()` for 15-30 seconds (a - `langfuse.api.NotFoundError` right after a successful flush is expected). - See the `api` property docs for a bounded retry pattern, or + *readable* yet: server-side ingestion is asynchronous, so flushed data + may not be queryable for 15-30 seconds — + `api.observations.get_many(trace_id=...)` may return empty results and + `api.trace.get()` may raise `langfuse.api.NotFoundError` right after a + successful flush. See the `api` property docs for a bounded retry + pattern, or https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk#ingestion-lag """ if self._resources is not None: diff --git a/langfuse/_client/propagation.py b/langfuse/_client/propagation.py index 6eaa2819d..d6e26163a 100644 --- a/langfuse/_client/propagation.py +++ b/langfuse/_client/propagation.py @@ -209,11 +209,11 @@ def propagate_attributes( metadata={"experiment": "variant_a"} ): # All spans created here will have user_id, session_id, environment, and metadata - with langfuse.start_observation(name="llm_call") as llm_span: + with langfuse.start_as_current_observation(name="llm_call") as llm_span: # This span inherits user_id, session_id, environment, and experiment metadata ... - with langfuse.start_observation( + with langfuse.start_as_current_observation( name="completion", as_type="generation" ) as gen: # This span also inherits all attributes From c0fa67753dc2c4e9dd53f1dc910c2ae5fecca1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 18:23:46 +0200 Subject: [PATCH 6/9] docs: clarify v2 score reads are removed in future Langfuse v4 servers Current servers are v3.x; the spec's removal note is forward-looking. Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 3f8c1f348..d2fc42d24 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -471,9 +471,10 @@ def api(self) -> LangfuseAPI: - **Score reads: use `api.scores_v3.get_many_v3(...)`** (cursor-paginated, filterable; pass `id=...` to fetch a single score). The older `api.scores.get_many(...)` / `api.scores.get_by_id(...)` (v2) are deprecated - and no longer available on Langfuse v4+ servers. There is no `api.scores.get`. - Scores are written via `langfuse.create_score(...)` / `span.score(...)`, which - are queued and flushed in the background — not via this API client. + and will not be available on Langfuse v4+ servers. There is no + `api.scores.get`. Scores are written via `langfuse.create_score(...)` / + `span.score(...)`, which are queued and flushed in the background — not via + this API client. For large-scale aggregation (usage/cost by model, user, etc.), prefer the v2 Metrics API (`api.metrics.metrics(...)`) over paginating row-level data. From 17dbd56104f5561238a5f41d3ca949e0e9bef301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 18:36:44 +0200 Subject: [PATCH 7/9] edits --- README.md | 2 +- langfuse/__init__.py | 27 +++++--- langfuse/_client/client.py | 123 +----------------------------------- langfuse/_client/observe.py | 15 ----- langfuse/experiment.py | 25 +------- 5 files changed, 22 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index 7beb0a778..68bfeaf69 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![Discord](https://img.shields.io/discord/1111061815649124414?style=flat-square&logo=Discord&logoColor=white&label=Discord&color=%23434EE4)](https://discord.gg/7NXusRtqYU) [![YC W23](https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square)](https://www.ycombinator.com/companies/langfuse) -The [Langfuse](https://langfuse.com) Python SDK covers the full platform, not just tracing: **observability/tracing** (OpenTelemetry-based, with OpenAI and LangChain integrations), **datasets & experiments** (offline evaluation and regression testing of prompt/model changes, including [CI via GitHub Actions](https://github.com/langfuse/experiment-action)), **LLM-as-a-judge and custom evaluations/scores**, **prompt management**, and a **full REST API client**. +The [Langfuse](https://langfuse.com) Python SDK covers the full platform: **observability/tracing** (OpenTelemetry-based, with OpenAI and LangChain integrations), **datasets & experiments** (offline evaluation and regression testing of prompt/model changes, including [CI via GitHub Actions](https://github.com/langfuse/experiment-action)), **LLM-as-a-judge and custom evaluations/scores**, **prompt management**, and a **full REST API client**. ## Installation diff --git a/langfuse/__init__.py b/langfuse/__init__.py index 6d805e232..6f2bfbb08 100644 --- a/langfuse/__init__.py +++ b/langfuse/__init__.py @@ -1,6 +1,6 @@ """Langfuse Python SDK — observability, evaluation, and prompt management for LLM applications. -Capabilities (all in this package, not just tracing): +Capabilities: - **Tracing / observability**: `@observe` decorator, `Langfuse.start_observation` / `start_as_current_observation` context managers, OpenTelemetry-based; integrations @@ -20,19 +20,28 @@ ```python # env: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL -from langfuse import get_client, observe +from langfuse import get_client -@observe() -def handle(query: str) -> str: - return "answer" +langfuse = get_client() -handle("hello") -get_client().flush() +# Create a span using a context manager +with langfuse.start_as_current_observation(as_type="span", name="process-request") as span: + # Your processing logic here + span.update(output="Processing complete") + + # Create a nested generation for an LLM call + with langfuse.start_as_current_observation(as_type="generation", name="llm-response", model="gpt-3.5-turbo") as generation: + # Your LLM call logic here + generation.update(output="Generated response") + +# All spans are automatically closed when exiting their context blocks + +# Flush events in short-lived applications +langfuse.flush() ``` Configuration is via constructor args or environment variables: `LANGFUSE_PUBLIC_KEY`, -`LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL` (defaults to https://cloud.langfuse.com; -the older `LANGFUSE_HOST` is deprecated). See `langfuse._client.environment_variables` +`LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL` (defaults to https://cloud.langfuse.com). See `langfuse._client.environment_variables` for the full list. Docs: https://langfuse.com/docs — machine-readable index: https://langfuse.com/llms.txt diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index d2fc42d24..caa8eea16 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -435,26 +435,6 @@ def api(self) -> LangfuseAPI: within 15-30 seconds; longer under load). The same applies to scores and dataset run reads. Instead of a fixed sleep, retry with a deadline: - ```python - import time - from langfuse.api import NotFoundError - - langfuse.flush() # ensure delivery before polling - deadline, delay = time.monotonic() + 60.0, 1.0 - while True: - try: - trace = langfuse.api.trace.get(trace_id) - break - except NotFoundError: - if time.monotonic() >= deadline: - raise - time.sleep(delay) - delay = min(delay * 2, 10.0) # exponential backoff, capped - ``` - - A successful `trace.get` means the trace record exists — its observations and - scores may still be arriving. - - **List endpoints return lightweight views.** `api.trace.list(...)` returns `TraceWithDetails`, where `observations` and `scores` are lists of ID strings. Fetch the full objects with `api.trace.get(trace_id)` (`TraceWithFullDetails`), @@ -468,34 +448,9 @@ def api(self) -> LangfuseAPI: `api.legacy.metrics_v1` but are less performant at scale, not recommended for new workflows, and will be deprecated. - - **Score reads: use `api.scores_v3.get_many_v3(...)`** (cursor-paginated, - filterable; pass `id=...` to fetch a single score). The older - `api.scores.get_many(...)` / `api.scores.get_by_id(...)` (v2) are deprecated - and will not be available on Langfuse v4+ servers. There is no - `api.scores.get`. Scores are written via `langfuse.create_score(...)` / - `span.score(...)`, which are queued and flushed in the background — not via - this API client. - - For large-scale aggregation (usage/cost by model, user, etc.), prefer the + - For large-scale aggregation (usage/cost by model, user, etc.), prefer the v2 Metrics API (`api.metrics.metrics(...)`) over paginating row-level data. - Example: - ```python - from langfuse import get_client - - langfuse = get_client() - - # Row-level reads: prefer the v2 observations endpoint - observations = langfuse.api.observations.get_many( - trace_id="abcdef1234", - type="GENERATION", - limit=100, - fields="core,basic,usage", - ) - - # Full single-trace tree (observations/scores as full objects) - full_trace = langfuse.api.trace.get("abcdef1234") - ``` See also: `async_api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk @@ -517,27 +472,6 @@ def api(self, value: LangfuseAPI) -> None: @property def async_api(self) -> AsyncLangfuseAPI: - """Asynchronous (asyncio) client for the full Langfuse REST API. - - Same resources and semantics as `api` — including asynchronous ingestion - (reads may raise `langfuse.api.NotFoundError` until processing completes, - typically within 15-30 seconds of `flush()`) and lightweight list views - (`list` endpoints return observation and score IDs as strings; `get` - endpoints return full objects). All methods must be awaited. - - Example: - ```python - async def trace_generations(trace_id: str): - # Row-level reads: prefer the v2 observations endpoint - return await langfuse.async_api.observations.get_many( - trace_id=trace_id, - type="GENERATION", - fields="core,basic,usage", - ) - ``` - - See also: `api`, https://langfuse.com/docs/api-and-data-platform/features/query-via-sdk - """ if self._resources is None: raise AttributeError("Langfuse client is not initialized") @@ -2785,49 +2719,6 @@ def average_accuracy(*, item_results, **kwargs): ) ``` - LLM-as-a-judge evaluation of a RAG pipeline (faithfulness and answer relevance): - ```python - from langfuse import Evaluation - - def rag_task(*, item, **kwargs): - question = item.input # DatasetItem attribute (dict items: item["input"]) - context = retrieve(question) # your retriever - answer = generate(question, context) # your LLM call - return {"answer": answer, "context": context} - - def llm_judge(*, prompt): - response = openai_client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": prompt}], - ) - return float(response.choices[0].message.content.strip()) - - def faithfulness_evaluator(*, input, output, **kwargs): - # Is the answer grounded in the retrieved context, without hallucination? - score = llm_judge( - prompt=f"Rate 0-1 how faithful the answer is to the context.\\n" - f"Context: {output['context']}\\nAnswer: {output['answer']}\\n" - f"Respond with only the number." - ) - return Evaluation(name="faithfulness", value=score) - - def answer_relevance_evaluator(*, input, output, **kwargs): - # Does the answer address the question that was asked? - score = llm_judge( - prompt=f"Rate 0-1 how relevant the answer is to the question.\\n" - f"Question: {input}\\nAnswer: {output['answer']}\\n" - f"Respond with only the number." - ) - return Evaluation(name="answer_relevance", value=score) - - result = langfuse.run_experiment( - name="RAG quality", - data=langfuse.get_dataset("rag-eval-set").items, - task=rag_task, - evaluators=[faithfulness_evaluator, answer_relevance_evaluator], - ) - ``` - Using with Langfuse datasets: ```python # Get dataset from Langfuse @@ -2851,18 +2742,6 @@ def answer_relevance_evaluator(*, input, output, **kwargs): - When using Langfuse datasets, results are automatically linked for easy comparison - This method works in both sync and async contexts (Jupyter notebooks, web apps, etc.) - Async execution is handled automatically with smart event loop detection - - **Regression testing in CI**: run experiments to compare prompt or model - versions before shipping them. The ``langfuse/experiment-action`` GitHub - Action (https://github.com/langfuse/experiment-action) runs experiments in - CI, comments results on PRs, and can fail the workflow via - `langfuse.RegressionError` when a metric drops below a threshold. - - See also: - `Evaluation` (return type for evaluators), `langfuse.RegressionError` (CI gating), - `DatasetClient.run_experiment` (dataset-linked variant), - https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk, - https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd, - https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge """ return cast( ExperimentResult, diff --git a/langfuse/_client/observe.py b/langfuse/_client/observe.py index 1377b4e41..bd0a3edee 100644 --- a/langfuse/_client/observe.py +++ b/langfuse/_client/observe.py @@ -145,13 +145,6 @@ async def generate_answer(query): model="gpt-4", messages=[{"role": "user", "content": query}] ) - get_client().update_current_generation( - model="gpt-4", - usage_details={ - "input": response.usage.prompt_tokens, - "output": response.usage.completion_tokens, - }, - ) return response.choices[0].message.content ``` @@ -187,14 +180,6 @@ def sub_process(): - langfuse_public_key: Use a specific Langfuse project (when multiple clients exist) - For async functions, the decorator returns an async function wrapper. - For sync functions, the decorator returns a synchronous wrapper. - - For generator functions, the observation stays open until the generator is - exhausted; yielded chunks are captured as output (see transform_to_string). - - See also: - `langfuse.propagate_attributes` (set user_id/session_id on all spans), - `Langfuse.start_as_current_observation` (imperative alternative), - https://langfuse.com/docs/observability/sdk/instrumentation, - https://langfuse.com/docs/observability/features/observation-types (as_type values) """ valid_types = set(get_observation_types_list(ObservationTypeLiteralNoEvent)) if as_type is not None and as_type not in valid_types: diff --git a/langfuse/experiment.py b/langfuse/experiment.py index 3dfe4e8dc..404c96e1d 100644 --- a/langfuse/experiment.py +++ b/langfuse/experiment.py @@ -183,16 +183,6 @@ def external_api_evaluator(*, input, output, **kwargs): Note: All arguments must be passed as keywords. Positional arguments are not allowed to ensure code clarity and prevent errors from argument reordering. - - Evaluators commonly wrap an LLM-as-a-judge call — e.g. scoring RAG - faithfulness or answer relevance with a grading prompt — and return the - judge's verdict as an ``Evaluation`` (see :meth:`Langfuse.run_experiment` - for a complete RAG example). - - See also: - `Langfuse.run_experiment` and `RegressionError` (regression-gate prompt/model - changes in CI via https://github.com/langfuse/experiment-action), - https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge """ def __init__( @@ -1132,17 +1122,6 @@ def run_experiment( metadata: Optional[Dict[str, str]] = None, _dataset_version: Optional[datetime] = None, ) -> ExperimentResult: - """Run an experiment, filling in CI-injected defaults for omitted arguments. - - Same signature and semantics as :meth:`Langfuse.run_experiment`, except - `data` and `metadata` fall back to the defaults injected by the - ``langfuse/experiment-action`` GitHub Action (explicit arguments win; - metadata dicts are merged with user keys taking precedence). Raise - :class:`RegressionError` from your experiment function to fail the CI - gate when a metric regresses. - - See also: https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd - """ resolved_data = data if data is not None else self.data if resolved_data is None: raise ValueError( @@ -1179,8 +1158,8 @@ class RegressionError(Exception): Intended for use with the ``langfuse/experiment-action`` GitHub Action (https://github.com/langfuse/experiment-action). The action catches this - exception and, when ``should_fail_on_regression`` is enabled (default), fails - the workflow run and renders a callout in the PR comment using + exception and, when ``should_fail_on_error`` is enabled, fails the + workflow run and renders a callout in the PR comment using ``metric``/``value``/``threshold`` if supplied, otherwise ``str(exc)``. Callers choose one of three forms: From 63503c623216096e0b9fa8d9cd5e00e1f28ba5c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 18:48:24 +0200 Subject: [PATCH 8/9] docs: revert deprecation message changes to keep PR strictly docs-only Restore the original @deprecated decorator strings on set_trace_io and set_current_trace_io so no runtime-visible string changes remain; the corrected propagate_attributes import guidance stays in the docstrings. Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 3 +-- langfuse/_client/span.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index caa8eea16..504df3507 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -1539,8 +1539,7 @@ def update_current_span( @deprecated( "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use the top-level " - "`from langfuse import propagate_attributes` context manager instead. " + "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " "This method will be removed in a future major version." ) def set_current_trace_io( diff --git a/langfuse/_client/span.py b/langfuse/_client/span.py index 8c257e92f..71f8d68c7 100644 --- a/langfuse/_client/span.py +++ b/langfuse/_client/span.py @@ -228,8 +228,7 @@ def end(self, *, end_time: Optional[int] = None) -> "LangfuseObservationWrapper" @deprecated( "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use the top-level " - "`from langfuse import propagate_attributes` context manager instead. " + "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " "This method will be removed in a future major version." ) def set_trace_io( From 5cce572ea816d13d67941e6d15f87b80692614c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jannik=20Maierh=C3=B6fer?= Date: Wed, 15 Jul 2026 18:52:22 +0200 Subject: [PATCH 9/9] push --- README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 68bfeaf69..e21f83618 100644 --- a/README.md +++ b/README.md @@ -24,14 +24,26 @@ pip install langfuse ```python # env: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL -from langfuse import get_client, observe -@observe() -def handle(query: str) -> str: - return "answer" +from langfuse import get_client -handle("hello") -get_client().flush() +langfuse = get_client() + +# Create a span using a context manager +with langfuse.start_as_current_observation(as_type="span", name="process-request") as span: + # Your processing logic here + span.update(output="Processing complete") + + # Create a nested generation for an LLM call + with langfuse.start_as_current_observation(as_type="generation", name="llm-response", model="gpt-5.6") as generation: + # Your LLM call logic here + generation.update(output="Generated response") + +# All spans are automatically closed when exiting their context blocks + + +# Flush events in short-lived applications +langfuse.flush() ``` ## Docs