[To dev/1.3] [Pipe] Preserve downstream sink error messages (#18553)

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java

@@ -32,6 +32,7 @@
 import org.apache.iotdb.commons.pipe.resource.PipeStopStrategy;
 import org.apache.iotdb.commons.pipe.resource.log.PipeLogger;
 import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink;
+import org.apache.iotdb.commons.utils.ErrorHandlingCommonUtils;
 import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
 import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
 import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
@@ -111,6 +112,10 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
       "Failed to borrow client from client pool when sending to receiver.";
   private static final String THRIFT_ERROR_FORMATTER_WITH_ENDPOINT =
       "Exception occurred while sending to receiver %s:%s.";
+  private static final String RETRY_QUEUE_FAILURE_MESSAGE =
+      "Failed to retry transferring events in the retry queue. Remaining events: %d (tablet events: %d, tsfile events: %d).";
+  private static final String RETRY_QUEUE_FAILURE_WITH_CAUSE_MESSAGE =
+      "Failed to retry transferring events in the retry queue. Remaining events: %d (tablet events: %d, tsfile events: %d). Last failure: %s.";
 
   private static final boolean isSplitTSFileBatchModeEnabled = true;
 
@@ -122,6 +127,8 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
   // Guarded by this. Events need identity semantics because the same payload may compare equal.
   private final Map<Event, PipeResourceFailureType> retryEvent2ResourceFailureType =
       new IdentityHashMap<>();
+  // Keep only the latest text to avoid retaining the complete exception chain for every event.
+  private volatile String lastRetryFailureMessage;
 
   private IoTDBDataNodeAsyncClientManager clientManager;
   private IoTDBDataNodeAsyncClientManager transferTsFileClientManager;
@@ -629,13 +636,11 @@ private void transferQueuedEventsIfNecessary(final boolean forced) {
 
         if (remainingEvents <= retryEventQueue.size() + retryTsFileQueue.size()) {
           final String message =
-              "Failed to retry transferring events in the retry queue. Remaining events: "
-                  + (retryEventQueue.size() + retryTsFileQueue.size())
-                  + " (tablet events: "
-                  + retryEventQueueEventCounter.getTabletInsertionEventCount()
-                  + ", tsfile events: "
-                  + retryEventQueueEventCounter.getTsFileInsertionEventCount()
-                  + ").";
+              formatRetryQueueFailureMessage(
+                  retryEventQueue.size() + retryTsFileQueue.size(),
+                  retryEventQueueEventCounter.getTabletInsertionEventCount(),
+                  retryEventQueueEventCounter.getTsFileInsertionEventCount(),
+                  lastRetryFailureMessage);
           final PipeResourceFailureType retryQueueResourceFailureType =
               getRetryQueueResourceFailureType();
           if (retryQueueResourceFailureType != null) {
@@ -648,6 +653,12 @@ private void transferQueuedEventsIfNecessary(final boolean forced) {
         }
       }
     }
+
+    synchronized (this) {
+      if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+        lastRetryFailureMessage = null;
+      }
+    }
   }
 
   private void retryTransfer(final TabletInsertionEvent tabletInsertionEvent) {
@@ -731,6 +742,14 @@ private synchronized void addFailureEventToRetryQueue(
       return;
     }
 
+    if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+      lastRetryFailureMessage = null;
+    }
+
+    if (e != null) {
+      lastRetryFailureMessage = getRetryFailureMessage(e);
+    }
+
     if (resourceFailureType != null && event instanceof EnrichedEvent) {
       final EnrichedEvent enrichedEvent = (EnrichedEvent) event;
       final Pair<String, Long> pipeKey =
@@ -778,6 +797,41 @@ public void addFailureEventsToRetryQueue(
     events.forEach(event -> addFailureEventToRetryQueue(event, e, failureRecordedPipes));
   }
 
+  static String formatRetryQueueFailureMessage(
+      final int remainingEvents,
+      final int tabletEventCount,
+      final int tsFileEventCount,
+      final String lastFailureMessage) {
+    if (!hasText(lastFailureMessage)) {
+      return String.format(
+          RETRY_QUEUE_FAILURE_MESSAGE, remainingEvents, tabletEventCount, tsFileEventCount);
+    }
+    return String.format(
+        RETRY_QUEUE_FAILURE_WITH_CAUSE_MESSAGE,
+        remainingEvents,
+        tabletEventCount,
+        tsFileEventCount,
+        lastFailureMessage);
+  }
+
+  private static String getRetryFailureMessage(final Exception exception) {
+    final Throwable rootCause = ErrorHandlingCommonUtils.getRootCause(exception);
+    if (hasText(rootCause.getMessage())) {
+      return rootCause.getMessage();
+    }
+    // Throwable#getMessage() is null for exceptions such as a bare NPE. Keep the type in the
+    // reported sink error instead of falling back to a generic transfer wrapper.
+    return rootCause.toString();
+  }
+
+  private static boolean hasText(final String message) {
+    return message != null && !message.trim().isEmpty();
+  }
+
+  synchronized String getLastRetryFailureMessage() {
+    return lastRetryFailureMessage;
+  }
+
   private synchronized PipeResourceFailureType getRetryQueueResourceFailureType() {
     for (final PipeResourceFailureType failureType : PipeResourceFailureType.values()) {
       if (retryEvent2ResourceFailureType.containsValue(failureType)) {
@@ -944,6 +998,10 @@ && isDroppedPipe((EnrichedEvent) event, committerKey)) {
           }
           return false;
         });
+
+    if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+      lastRetryFailureMessage = null;
+    }
   }
 
   @Override
@@ -998,6 +1056,7 @@ public synchronized void clearRetryEventsReferenceCount() {
       }
     }
     retryEvent2ResourceFailureType.clear();
+    lastRetryFailureMessage = null;
   }
 
   //////////////////////// APIs provided for metric framework ////////////////////////

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java

@@ -66,6 +66,24 @@ StatusUtils.OK, new TSStatus(TSStatusCode.OUT_OF_TTL.getStatusCode()))))
             .getCode());
   }
 
+  @Test
+  public void testMultipleErrorPropagatesSelectedReceiverMessage() {
+    final TSStatus status =
+        IoTDBDataNodeReceiver.STATEMENT_STATUS_VISITOR.process(
+            new InsertRowsStatement(),
+            new TSStatus(TSStatusCode.MULTIPLE_ERROR.getStatusCode())
+                .setSubStatus(
+                    Arrays.asList(
+                        StatusUtils.OK,
+                        new TSStatus(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode())
+                            .setMessage("receiver write queue is full"))));
+
+    Assert.assertEquals(
+        TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode(),
+        status.getCode());
+    Assert.assertEquals("receiver write queue is full", status.getMessage());
+  }
+
   @Test
   public void testLoadTemporaryUnavailableClassification() throws Exception {
     final File tsFile = File.createTempFile("temporary-unavailable", ".tsfile");

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.thrift.async;
+
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class IoTDBDataRegionAsyncSinkTest {
+
+  @Test
+  public void testRetryQueueFailureMessageIncludesRootCauseAndIsCleared() {
+    final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
+    final Event event = Mockito.mock(Event.class);
+
+    sink.addFailureEventToRetryQueue(
+        event,
+        new PipeException(
+            "sink transfer wrapper", new IllegalStateException("receiver rejected request")));
+
+    Assert.assertEquals("receiver rejected request", sink.getLastRetryFailureMessage());
+    Assert.assertTrue(
+        IoTDBDataRegionAsyncSink.formatRetryQueueFailureMessage(
+                1, 1, 0, sink.getLastRetryFailureMessage())
+            .contains("receiver rejected request"));
+
+    sink.clearRetryEventsReferenceCount();
+
+    Assert.assertNull(sink.getLastRetryFailureMessage());
+    Assert.assertFalse(
+        IoTDBDataRegionAsyncSink.formatRetryQueueFailureMessage(
+                0, 0, 0, sink.getLastRetryFailureMessage())
+            .contains("receiver rejected request"));
+  }
+
+  @Test
+  public void testRetryQueueFailureMessageKeepsRootCauseTypeWhenMessageIsMissing() {
+    final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
+    final Event event = Mockito.mock(Event.class);
+
+    sink.addFailureEventToRetryQueue(
+        event, new PipeException("sink transfer wrapper", new NullPointerException()));
+
+    Assert.assertEquals("java.lang.NullPointerException", sink.getLastRetryFailureMessage());
+  }
+}

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java

@@ -19,15 +19,21 @@
 
 package org.apache.iotdb.db.pipe.sink.protocol.thrift.async.handler;
 
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
 
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.File;
+import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.util.Collections;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -70,6 +76,56 @@ public void testCloseKeepsSourceTsFile() throws Exception {
     }
   }
 
+  @Test
+  public void testSealFailurePassesNestedReceiverMessageToRetryQueue() throws Exception {
+    final File file = Files.createTempFile("pipe-transfer-seal-failure", ".tsfile").toFile();
+    try {
+      final PipeTsFileInsertionEvent event = Mockito.mock(PipeTsFileInsertionEvent.class);
+      final IoTDBDataRegionAsyncSink sink = Mockito.mock(IoTDBDataRegionAsyncSink.class);
+      Mockito.when(sink.statusHandler())
+          .thenReturn(new PipeReceiverStatusHandler(false, 60, false, 60, false, false));
+
+      final PipeTransferTsFileHandler handler =
+          new PipeTransferTsFileHandler(
+              sink,
+              Collections.emptyMap(),
+              Collections.singletonList(event),
+              new AtomicInteger(1),
+              new AtomicBoolean(false),
+              file,
+              null,
+              false,
+              null);
+      markSealSignalSent(handler);
+
+      final TSStatus status =
+          new TSStatus(TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
+              .setMessage("aggregate load failure")
+              .setSubStatus(
+                  Collections.singletonList(
+                      new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())
+                          .setMessage("receiver disk is full")));
+
+      Assert.assertFalse(handler.onCompleteInternal(new TPipeTransferResp(status)));
+
+      final ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class);
+      Mockito.verify(sink)
+          .addFailureEventsToRetryQueue(
+              Mockito.eq(Collections.singletonList(event)), exceptionCaptor.capture());
+      Assert.assertEquals("receiver disk is full", exceptionCaptor.getValue().getMessage());
+    } finally {
+      if (file.exists()) {
+        Assert.assertTrue(file.delete());
+      }
+    }
+  }
+
+  private static void markSealSignalSent(final PipeTransferTsFileHandler handler) throws Exception {
+    final Field field = PipeTransferTsFileHandler.class.getDeclaredField("isSealSignalSent");
+    field.setAccessible(true);
+    ((AtomicBoolean) field.get(handler)).set(true);
+  }
+
   private PipeTransferTsFileHandler createHandler(final File file, final EnrichedEvent event)
       throws Exception {
     return new PipeTransferTsFileHandler(