Send a linked list of errors from the worker to the coordinator

This will allow us to format them nicer at a higher level.

Author: shepmaster

compiler/base/orchestrator/src/coordinator.rs

@@ -33,7 +33,7 @@ use crate::{
     message::{
         CommandStatistics, CoordinatorMessage, DeleteFileRequest, ExecuteCommandRequest,
         ExecuteCommandResponse, JobId, Multiplexed, OneToOneResponse, ReadFileRequest,
-        ReadFileResponse, SerializedError, WorkerMessage, WriteFileRequest,
+        ReadFileResponse, SerializedError2, WorkerMessage, WriteFileRequest,
     },
     DropErrorDetailsExt,
 };
@@ -1754,15 +1754,25 @@ impl Container {
                                 WorkerMessage::ExecuteCommand(resp) => {
                                     return Ok(resp);
                                 }
+
                                 WorkerMessage::StdoutPacket(packet) => {
                                     stdout_tx.send(packet).await.ok(/* Receiver gone, that's OK */);
                                 }
+
                                 WorkerMessage::StderrPacket(packet) => {
                                     stderr_tx.send(packet).await.ok(/* Receiver gone, that's OK */);
                                 }
+
                                 WorkerMessage::CommandStatistics(stats) => {
                                     status_tx.send(stats).await.ok(/* Receiver gone, that's OK */);
                                 }
+
+                                WorkerMessage::Error(e) =>
+                                    return Err(SerializedError2::adapt(e)).context(WorkerSnafu),
+
+                                WorkerMessage::Error2(e) =>
+                                    return Err(e).context(WorkerSnafu),
+
                                 _ => return UnexpectedMessageSnafu.fail(),
                             }
                         },
@@ -2084,6 +2094,9 @@ pub enum SpawnCargoError {
     #[snafu(display("Could not start Cargo"))]
     CouldNotStartCargo { source: CommanderError },
 
+    #[snafu(display("The worker operation failed"))]
+    Worker { source: SerializedError2 },
+
     #[snafu(display("Received an unexpected message"))]
     UnexpectedMessage,
 
@@ -2328,7 +2341,7 @@ impl Commander {
     where
         M: Into<CoordinatorMessage>,
         M: OneToOneResponse,
-        Result<M::Response, SerializedError>: TryFrom<WorkerMessage>,
+        Result<M::Response, SerializedError2>: TryFrom<WorkerMessage>,
     {
         use commander_error::*;
 
@@ -2346,9 +2359,8 @@ impl Commander {
             .await
             .context(UnableToReceiveFromDemultiplexerSnafu)?;
 
-        match msg.try_into() {
-            Ok(Ok(v)) => Ok(v),
-            Ok(Err(e)) => WorkerOperationFailedSnafu { text: e.0 }.fail(),
+        match <Result<_, _>>::try_from(msg) {
+            Ok(v) => v.context(WorkerOperationFailedSnafu),
             Err(_) => UnexpectedResponseTypeSnafu.fail(),
         }
     }
@@ -2401,8 +2413,8 @@ pub enum CommanderError {
     #[snafu(display("Did not receive the expected response type from the worker"))]
     UnexpectedResponseType,
 
-    #[snafu(display("The worker operation failed: {text}"))]
-    WorkerOperationFailed { text: String },
+    #[snafu(display("The worker operation failed"))]
+    WorkerOperationFailed { source: SerializedError2 },
 }
 
 pub trait Backend {

compiler/base/orchestrator/src/message.rs

@@ -1,5 +1,8 @@
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::{
+    collections::{HashMap, VecDeque},
+    fmt,
+};
 
 pub type JobId = u64;
 pub type Path = String;
@@ -47,19 +50,22 @@ pub enum WorkerMessage {
     StdoutPacket(String),
     StderrPacket(String),
     CommandStatistics(CommandStatistics),
+    /// Vestigial; remove after a while
     Error(SerializedError),
+    Error2(SerializedError2),
 }
 
 macro_rules! impl_broad_to_narrow_with_error {
     ($enum_type:ident, $($variant_name:ident => $variant_type:ty),* $(,)?) => {
         $(
-            impl TryFrom<$enum_type> for Result<$variant_type, SerializedError> {
+            impl TryFrom<$enum_type> for Result<$variant_type, SerializedError2> {
                 type Error = $enum_type;
 
                 fn try_from(other: $enum_type) -> Result<Self, Self::Error> {
                     match other {
                         $enum_type::$variant_name(x) => Ok(Ok(x)),
-                        $enum_type::Error(e) => Ok(Err(e)),
+                        $enum_type::Error(e) => Ok(Err(SerializedError2::adapt(e))),
+                        $enum_type::Error2(e) => Ok(Err(e)),
                         o => Err(o)
                     }
                 }
@@ -153,6 +159,49 @@ impl SerializedError {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SerializedError2 {
+    pub message: String,
+    pub source: Option<Box<Self>>,
+}
+
+impl SerializedError2 {
+    pub fn new(e: impl snafu::Error) -> Self {
+        let mut messages = snafu::CleanedErrorText::new(&e)
+            .map(|(_, t, _)| t)
+            .filter(|t| !t.is_empty())
+            .collect::<VecDeque<_>>();
+
+        let message = messages
+            .pop_front()
+            .unwrap_or_else(|| "No error message".into());
+        let source = messages.into_iter().rev().fold(None, |source, message| {
+            Some(Box::new(Self { source, message }))
+        });
+
+        Self { message, source }
+    }
+
+    pub fn adapt(other: SerializedError) -> Self {
+        Self {
+            message: other.0,
+            source: None,
+        }
+    }
+}
+
+impl snafu::Error for SerializedError2 {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        self.source.as_ref().map(|x| &**x as &dyn snafu::Error)
+    }
+}
+
+impl fmt::Display for SerializedError2 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.message.fmt(f)
+    }
+}
+
 pub trait OneToOneResponse {
     type Response;
 }

compiler/base/orchestrator/src/worker.rs

@@ -53,7 +53,7 @@ use crate::{
     message::{
         CoordinatorMessage, DeleteFileRequest, DeleteFileResponse, ExecuteCommandRequest,
         ExecuteCommandResponse, JobId, Multiplexed, ReadFileRequest, ReadFileResponse,
-        SerializedError, WorkerMessage, WriteFileRequest, WriteFileResponse,
+        SerializedError2, WorkerMessage, WriteFileRequest, WriteFileResponse,
     },
     DropErrorDetailsExt,
 };
@@ -268,7 +268,7 @@ impl MultiplexingSender {
     }
 
     async fn send_err(&self, e: impl std::error::Error) -> Result<(), MultiplexingSenderError> {
-        self.send_raw(WorkerMessage::Error(SerializedError::new(e)))
+        self.send_raw(WorkerMessage::Error2(SerializedError2::new(e)))
             .await
     }