From a7c5c082d8156faebec71dd392c9473e4859c07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 14 Jul 2026 16:26:28 +0200 Subject: [PATCH 1/5] feat: downgrade uncaught error log when reconciler handles the error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a reconciler's updateErrorStatus returns any ErrorStatusUpdateControl other than defaultErrorProcessing(), the error is now considered handled by the reconciler. Native retry (including @GradualRetry exponential backoff) is kept intact, but the framework logs the error on DEBUG level instead of emitting the "Uncaught error during event processing" WARN. This lets a reconciler keep retrying an expected, recoverable condition without producing a continuous stream of WARN messages, while it remains free to log the error at whatever level it wants inside updateErrorStatus. Returning defaultErrorProcessing() preserves the previous behavior, including the warning. - PostExecutionControl carries an errorHandledByReconciler flag - ReconciliationDispatcher marks the exception control as handled instead of rethrowing when a non-default control still wants retry - EventProcessor downgrades the retry-aware and no-retry-configured error logs to DEBUG when the error was handled by the reconciler Signed-off-by: Attila Mészáros --- .../documentation/error-handling-retries.md | 10 ++++++ .../processing/event/EventProcessor.java | 35 ++++++++++++++----- .../event/PostExecutionControl.java | 17 +++++++++ .../event/ReconciliationDispatcher.java | 7 +++- .../event/ReconciliationDispatcherTest.java | 34 ++++++++++++++++++ 5 files changed, 94 insertions(+), 9 deletions(-) diff --git a/docs/content/en/docs/documentation/error-handling-retries.md b/docs/content/en/docs/documentation/error-handling-retries.md index 7bd4ad2e22..2d3e1488be 100644 --- a/docs/content/en/docs/documentation/error-handling-retries.md +++ b/docs/content/en/docs/documentation/error-handling-retries.md @@ -108,6 +108,16 @@ Retry can be skipped in cases of unrecoverable errors: ErrorStatusUpdateControl.patchStatus(customResource).withNoRetry(); ``` +When `updateErrorStatus` returns any `ErrorStatusUpdateControl` other than +`ErrorStatusUpdateControl.defaultErrorProcessing()`, the framework considers the error handled by +the reconciler. In that case the native retry (including the exponential backoff from +`@GradualRetry`) is still performed, but the framework no longer logs the "Uncaught error during +event processing" warning; the error is logged on `DEBUG` level instead. This lets a reconciler keep +retrying an expected, recoverable condition without producing a continuous stream of `WARN` messages, +while it remains free to log the error at whatever level it deems appropriate inside +`updateErrorStatus`. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` preserves the +default behavior, including the warning. + ### Correctness and Automatic Retries While it is possible to deactivate automatic retries, this is not desirable unless there is a particular reason. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index c8322e47e5..66a4f6a0d3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -297,7 +297,9 @@ synchronized void eventProcessingFinished( && postExecutionControl.exceptionDuringExecution() && (!state.deleteEventPresent() || triggerOnAllEvents())) { handleRetryOnException( - executionScope, postExecutionControl.getRuntimeException().orElseThrow()); + executionScope, + postExecutionControl.getRuntimeException().orElseThrow(), + postExecutionControl.isErrorHandledByReconciler()); return; } cleanupOnSuccessfulExecution(executionScope); @@ -327,10 +329,17 @@ private boolean isTriggerOnAllEventAndDeleteEventPresent(ResourceState state) { private void logErrorIfNoRetryConfigured( ExecutionScope

executionScope, PostExecutionControl

postExecutionControl) { if (!isRetryConfigured() && postExecutionControl.exceptionDuringExecution()) { - log.error( - "Error during event processing {}", - executionScope, - postExecutionControl.getRuntimeException().orElseThrow()); + if (postExecutionControl.isErrorHandledByReconciler()) { + log.debug( + "Error during event processing {}, but was handled by the reconciler", + executionScope, + postExecutionControl.getRuntimeException().orElseThrow()); + } else { + log.error( + "Error during event processing {}", + executionScope, + postExecutionControl.getRuntimeException().orElseThrow()); + } } } @@ -369,14 +378,16 @@ TimerEventSource

retryEventSource() { * events (received meanwhile retry is in place or already in buffer) instantly or always wait * according to the retry timing if there was an exception. */ - private void handleRetryOnException(ExecutionScope

executionScope, Exception exception) { + private void handleRetryOnException( + ExecutionScope

executionScope, Exception exception, boolean errorHandledByReconciler) { final var state = getOrInitRetryExecution(executionScope); var resourceID = state.getId(); boolean eventPresent = state.eventPresent() || (triggerOnAllEvents() && state.isAdditionalEventPresentAfterDeleteEvent()); state.markEventReceived(triggerOnAllEvents()); - retryAwareErrorLogging(state.getRetry(), eventPresent, exception, executionScope); + retryAwareErrorLogging( + state.getRetry(), eventPresent, errorHandledByReconciler, exception, executionScope); metrics.reconciliationFailed( executionScope.getResource(), state.getRetry(), exception, metricsMetadata); if (eventPresent) { @@ -408,9 +419,17 @@ private void handleRetryOnException(ExecutionScope

executionScope, Exception private void retryAwareErrorLogging( RetryExecution retry, boolean eventPresent, + boolean errorHandledByReconciler, Exception exception, ExecutionScope

executionScope) { - if (!retry.isLastAttempt() + if (errorHandledByReconciler) { + // The reconciler already handled the error in updateErrorStatus (and had the chance to log it + // as needed), so the framework only logs it on debug level while still retrying. + log.debug( + "Error during event processing {}, but was handled by the reconciler", + executionScope, + exception); + } else if (!retry.isLastAttempt() && exception instanceof KubernetesClientException ex && ex.getCode() == HttpURLConnection.HTTP_CONFLICT) { log.debug("Full client conflict error during event processing {}", executionScope, exception); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/PostExecutionControl.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/PostExecutionControl.java index 64d7a25166..ef09c74a08 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/PostExecutionControl.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/PostExecutionControl.java @@ -27,6 +27,7 @@ final class PostExecutionControl { private final Exception runtimeException; private Long reScheduleDelay = null; + private boolean errorHandledByReconciler = false; private PostExecutionControl( boolean finalizerRemoved, @@ -68,6 +69,22 @@ public static PostExecutionControl exceptionDuringExe return new PostExecutionControl<>(false, null, false, exception); } + /** + * Marks that the exception was handled by the reconciler's {@code updateErrorStatus} (i.e. the + * reconciler did not return {@link + * io.javaoperatorsdk.operator.api.reconciler.ErrorStatusUpdateControl#defaultErrorProcessing()}), + * but the error is still retried. In this case the framework logs the error on a lower level, + * since the reconciler already had the chance to handle and log it as needed. + */ + public PostExecutionControl withErrorHandledByReconciler() { + this.errorHandledByReconciler = true; + return this; + } + + public boolean isErrorHandledByReconciler() { + return errorHandledByReconciler; + } + public Optional getUpdatedCustomResource() { return Optional.ofNullable(updatedCustomResource); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java index dfdf37d3e1..e65a611c19 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java @@ -266,7 +266,12 @@ public boolean isLastAttempt() { errorStatusUpdateControl.getScheduleDelay().ifPresent(postExecutionControl::withReSchedule); return postExecutionControl; } - throw e; + // The reconciler handled the error via updateErrorStatus (it did not return + // defaultErrorProcessing()) but still wants the error to be retried. The retry (and its + // backoff) is kept intact, but since the reconciler already had the chance to handle and log + // the error, the framework logs it on a lower level instead of emitting an "uncaught error" + // warning. + return PostExecutionControl.

exceptionDuringExecution(e).withErrorHandledByReconciler(); } private PostExecutionControl

createPostExecutionControl( diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcherTest.java index e874731657..ac24375242 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcherTest.java @@ -481,6 +481,40 @@ void callErrorStatusHandlerEvenOnFirstError() { assertThat(postExecControl.exceptionDuringExecution()).isTrue(); } + @Test + void errorHandledByReconcilerMarkedWhenErrorStatusHandledButRetried() { + testCustomResource.addFinalizer(DEFAULT_FINALIZER); + reconciler.reconcile = + (r, c) -> { + throw new IllegalStateException("Error Status Test"); + }; + reconciler.errorHandler = () -> ErrorStatusUpdateControl.patchStatus(testCustomResource); + + var postExecControl = + reconciliationDispatcher.handleExecution( + new ExecutionScope(null, null, false, false).setResource(testCustomResource)); + + assertThat(postExecControl.exceptionDuringExecution()).isTrue(); + assertThat(postExecControl.isErrorHandledByReconciler()).isTrue(); + } + + @Test + void errorNotHandledByReconcilerOnDefaultErrorProcessing() { + testCustomResource.addFinalizer(DEFAULT_FINALIZER); + reconciler.reconcile = + (r, c) -> { + throw new IllegalStateException("Error Status Test"); + }; + reconciler.errorHandler = () -> ErrorStatusUpdateControl.defaultErrorProcessing(); + + var postExecControl = + reconciliationDispatcher.handleExecution( + new ExecutionScope(null, null, false, false).setResource(testCustomResource)); + + assertThat(postExecControl.exceptionDuringExecution()).isTrue(); + assertThat(postExecControl.isErrorHandledByReconciler()).isFalse(); + } + @Test void errorHandlerCanInstructNoRetryWithUpdate() { testCustomResource.addFinalizer(DEFAULT_FINALIZER); From 5a38a66826034e280768ef0e35a35f175079e968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 14 Jul 2026 17:01:16 +0200 Subject: [PATCH 2/5] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/processing/event/EventProcessor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index 66a4f6a0d3..a556e0bdf2 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -331,12 +331,13 @@ private void logErrorIfNoRetryConfigured( if (!isRetryConfigured() && postExecutionControl.exceptionDuringExecution()) { if (postExecutionControl.isErrorHandledByReconciler()) { log.debug( - "Error during event processing {}, but was handled by the reconciler", + "Error during event processing {}, but was handled by the reconciler. (No retry" + + " configured)", executionScope, postExecutionControl.getRuntimeException().orElseThrow()); } else { log.error( - "Error during event processing {}", + "Error during event processing {}. (No retry configured)", executionScope, postExecutionControl.getRuntimeException().orElseThrow()); } From 601328f4d119931955d8284a35718ee804d44827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 15 Jul 2026 14:22:44 +0200 Subject: [PATCH 3/5] fix: only downgrade handled-error log while retries remain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously errorHandledByReconciler downgraded the error log to DEBUG unconditionally, which also hid final, non-recoverable failures on the last retry attempt. Now the downgrade only applies while retry attempts remain (errorHandledByReconciler && !retry.isLastAttempt()). On the last attempt the existing higher-severity WARN/ERROR logging is preserved. Likewise, when no retry is configured every failure is final, so it keeps the ERROR log regardless of whether the reconciler handled the error. Signed-off-by: Attila Mészáros --- .../documentation/error-handling-retries.md | 14 ++++++----- .../processing/event/EventProcessor.java | 25 ++++++++----------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/docs/content/en/docs/documentation/error-handling-retries.md b/docs/content/en/docs/documentation/error-handling-retries.md index 2d3e1488be..f543812035 100644 --- a/docs/content/en/docs/documentation/error-handling-retries.md +++ b/docs/content/en/docs/documentation/error-handling-retries.md @@ -111,12 +111,14 @@ Retry can be skipped in cases of unrecoverable errors: When `updateErrorStatus` returns any `ErrorStatusUpdateControl` other than `ErrorStatusUpdateControl.defaultErrorProcessing()`, the framework considers the error handled by the reconciler. In that case the native retry (including the exponential backoff from -`@GradualRetry`) is still performed, but the framework no longer logs the "Uncaught error during -event processing" warning; the error is logged on `DEBUG` level instead. This lets a reconciler keep -retrying an expected, recoverable condition without producing a continuous stream of `WARN` messages, -while it remains free to log the error at whatever level it deems appropriate inside -`updateErrorStatus`. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` preserves the -default behavior, including the warning. +`@GradualRetry`) is still performed, and while retry attempts remain the framework no longer logs +the "Uncaught error during event processing" warning; the error is logged on `DEBUG` level instead. +This lets a reconciler keep retrying an expected, recoverable condition without producing a +continuous stream of `WARN` messages, while it remains free to log the error at whatever level it +deems appropriate inside `updateErrorStatus`. On the last retry attempt (or when no retry is +configured), the failure is final, so the framework keeps its higher-severity logging to avoid +hiding a non-recoverable error. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` +preserves the default behavior, including the warning. ### Correctness and Automatic Retries diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index a556e0bdf2..21601b2339 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -329,18 +329,13 @@ private boolean isTriggerOnAllEventAndDeleteEventPresent(ResourceState state) { private void logErrorIfNoRetryConfigured( ExecutionScope

executionScope, PostExecutionControl

postExecutionControl) { if (!isRetryConfigured() && postExecutionControl.exceptionDuringExecution()) { - if (postExecutionControl.isErrorHandledByReconciler()) { - log.debug( - "Error during event processing {}, but was handled by the reconciler. (No retry" - + " configured)", - executionScope, - postExecutionControl.getRuntimeException().orElseThrow()); - } else { - log.error( - "Error during event processing {}. (No retry configured)", - executionScope, - postExecutionControl.getRuntimeException().orElseThrow()); - } + // No retry is configured, so this failure is final. Even if the reconciler handled the error + // in updateErrorStatus, we keep the higher-severity log so a non-recoverable failure is not + // hidden from operators. + log.error( + "Error during event processing {}", + executionScope, + postExecutionControl.getRuntimeException().orElseThrow()); } } @@ -423,9 +418,11 @@ private void retryAwareErrorLogging( boolean errorHandledByReconciler, Exception exception, ExecutionScope

executionScope) { - if (errorHandledByReconciler) { + if (errorHandledByReconciler && !retry.isLastAttempt()) { // The reconciler already handled the error in updateErrorStatus (and had the chance to log it - // as needed), so the framework only logs it on debug level while still retrying. + // as needed), so while there are retries remaining the framework only logs it on debug level. + // On the last attempt we still fall through to the higher-severity logging below, so a final, + // non-recoverable failure is not hidden from operators. log.debug( "Error during event processing {}, but was handled by the reconciler", executionScope, From 7020aebf02256ec8619346f0a409e4d3736df8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 15 Jul 2026 14:53:31 +0200 Subject: [PATCH 4/5] docs: clarify retry is not preserved for retry-disabling controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error-handling docs claimed native retry is still performed for any non-default ErrorStatusUpdateControl, which is inaccurate for controls that disable retry (withNoRetry() / rescheduleAfter()). Qualify the paragraph so it only claims retry/backoff is preserved when a retry is configured and not disabled by the returned control. Signed-off-by: Attila Mészáros --- .../documentation/error-handling-retries.md | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/content/en/docs/documentation/error-handling-retries.md b/docs/content/en/docs/documentation/error-handling-retries.md index f543812035..428c35b9ae 100644 --- a/docs/content/en/docs/documentation/error-handling-retries.md +++ b/docs/content/en/docs/documentation/error-handling-retries.md @@ -110,15 +110,19 @@ Retry can be skipped in cases of unrecoverable errors: When `updateErrorStatus` returns any `ErrorStatusUpdateControl` other than `ErrorStatusUpdateControl.defaultErrorProcessing()`, the framework considers the error handled by -the reconciler. In that case the native retry (including the exponential backoff from -`@GradualRetry`) is still performed, and while retry attempts remain the framework no longer logs -the "Uncaught error during event processing" warning; the error is logged on `DEBUG` level instead. -This lets a reconciler keep retrying an expected, recoverable condition without producing a -continuous stream of `WARN` messages, while it remains free to log the error at whatever level it -deems appropriate inside `updateErrorStatus`. On the last retry attempt (or when no retry is -configured), the failure is final, so the framework keeps its higher-severity logging to avoid -hiding a non-recoverable error. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` -preserves the default behavior, including the warning. +the reconciler and, while retry attempts remain, no longer logs the "Uncaught error during event +processing" warning; the error is logged on `DEBUG` level instead. This lets a reconciler keep +retrying an expected, recoverable condition without producing a continuous stream of `WARN` +messages, while it remains free to log the error at whatever level it deems appropriate inside +`updateErrorStatus`. + +This log downgrade applies only when retry is actually taking place, i.e. a retry is configured and +the returned control does not disable it. Controls that explicitly disable retry — `withNoRetry()` +and `rescheduleAfter()` — cancel the native retry and its `@GradualRetry` backoff, so nothing is +retried in those cases. Likewise, on the last retry attempt (or when no retry is configured) the +failure is final, so the framework keeps its higher-severity logging to avoid hiding a +non-recoverable error. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` preserves the +default behavior, including the warning. ### Correctness and Automatic Retries From 36ce6fddfe780c43a752fad0e7e7fc1720ff770e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 15 Jul 2026 15:09:20 +0200 Subject: [PATCH 5/5] fix: keep conflict log precedence over handled-error downgrade retryAwareErrorLogging checked errorHandledByReconciler before the HTTP_CONFLICT special-case, so handled errors that were also KubernetesClientException conflicts no longer emitted the conflict-specific INFO message. The conflict branch is already low-noise (DEBUG + INFO) and provides actionable info, so it now keeps precedence over the handled-error downgrade. --- .../processing/event/EventProcessor.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index 21601b2339..ddc0f73a27 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -418,24 +418,26 @@ private void retryAwareErrorLogging( boolean errorHandledByReconciler, Exception exception, ExecutionScope

executionScope) { - if (errorHandledByReconciler && !retry.isLastAttempt()) { - // The reconciler already handled the error in updateErrorStatus (and had the chance to log it - // as needed), so while there are retries remaining the framework only logs it on debug level. - // On the last attempt we still fall through to the higher-severity logging below, so a final, - // non-recoverable failure is not hidden from operators. - log.debug( - "Error during event processing {}, but was handled by the reconciler", - executionScope, - exception); - } else if (!retry.isLastAttempt() + if (!retry.isLastAttempt() && exception instanceof KubernetesClientException ex && ex.getCode() == HttpURLConnection.HTTP_CONFLICT) { + // The conflict branch is already low-noise (DEBUG + INFO) and provides actionable info, so it + // keeps precedence over the handled-error downgrade below. log.debug("Full client conflict error during event processing {}", executionScope, exception); log.info( "Resource Kubernetes Resource Creator/Update Conflict during reconciliation. Message:" + " {} Resource name: {}", ex.getMessage(), ex.getFullResourceName()); + } else if (errorHandledByReconciler && !retry.isLastAttempt()) { + // The reconciler already handled the error in updateErrorStatus (and had the chance to log it + // as needed), so while there are retries remaining the framework only logs it on debug level. + // On the last attempt we still fall through to the higher-severity logging below, so a final, + // non-recoverable failure is not hidden from operators. + log.debug( + "Error during event processing {}, but was handled by the reconciler", + executionScope, + exception); } else if (eventPresent || !retry.isLastAttempt()) { log.warn( "Uncaught error during event processing {} - but another reconciliation will be attempted"