nuclio · rokatyy · Mar 3, 2025 · Mar 2, 2025
diff --git a/docs/tasks/index.rst b/docs/tasks/index.rst
@@ -10,4 +10,5 @@ User guide
    deploying-pre-built-functions
    deploy-functions-from-dockerfile
    exporting-and-importing
-   benchmarking
+   benchmarking
+   known-issues
diff --git a/docs/tasks/known-issues.md b/docs/tasks/known-issues.md
@@ -0,0 +1,10 @@
+## Known issues
+
+### 503 Error Code While Scaling Down (Kubernetes Only)
+
+This is a rare issue that primarily occurs in low-latency setups.
+When scaling down a Nuclio function pod, Kubernetes may return a 503 error due to the delay between sending a `SIGTERM` signal and stopping traffic to the pod.
+
+To mitigate this, we've introduced a 5-second wait time before halting event processing after receiving `SIGTERM`. However, this does not fully eliminate the issue.
+
+**Possible Solution**: Implement a client-side retry mechanism when a 503 response is received with an empty body.
diff --git a/pkg/platform/kube/functionres/lazy.go b/pkg/platform/kube/functionres/lazy.go
@@ -2376,7 +2376,7 @@ func (lc *lazyClient) populateDeploymentContainer(ctx context.Context,
 		InitialDelaySeconds: 5,
 		TimeoutSeconds:      1,
 		PeriodSeconds:       1,
-		FailureThreshold:    10,
+		FailureThreshold:    3,
 	}
 
 	container.LivenessProbe = &v1.Probe{

diff --git a/pkg/processor/trigger/trigger.go b/pkg/processor/trigger/trigger.go
@@ -386,7 +386,7 @@ func (at *AbstractTrigger) SignalWorkersToTerminate() error {
 	// DO NOT REMOVE
 	// this sleep is needed to give k8s some time to stop sending traffic to the service
 	// before we shut worker down
-	time.Sleep(1 * time.Second)
+	time.Sleep(5 * time.Second)
 
 	if err := at.WorkerAllocator.SignalTermination(); err != nil {
 		return errors.Wrap(err, "Failed to signal all workers to terminate")