Add support for automatic server restart on crash

doc/LanguageClient.txt

@@ -657,6 +657,20 @@ margin, set this setting to 0. Example:
 <
 Default: 1
 
+2.43 g:LanguageClient_restartOnCrash          *g:LanguageClient_restartOnCrash*
+
+If enabled, the client will attempt to restart the server on the event that it
+unexpectedly crashes.
+
+Default: 1
+
+2.44 g:LanguageClient_maxRestartRetries          *g:LanguageClient_maxRestartRetries*
+
+Max number of times to attempt to recover from a server crash. Each language
+handles its own count independently.
+
+Default: 5
+
 ==============================================================================
 3. Commands                                           *LanguageClientCommands*
 
@@ -1094,6 +1108,11 @@ This event is triggered when diagnostics changed.
 
 Triggered after textDocument/didOpen notification is sent to language server.
 
+6.5 LanguageServerCrashed
+*LanguageServerCrashed*
+
+This event is triggered when a language server unexpectedly quits.
+
 ==============================================================================
 7. License                                             *LanguageClientLicense*
 

src/language_server_protocol.rs

@@ -214,6 +214,15 @@ impl LanguageClient {
             .as_ref(),
         )?;
 
+        #[allow(clippy::type_complexity)]
+        let (restart_on_crash, max_restart_retries): (u8, u8) = self.vim()?.eval(
+            [
+                "get(g:, 'LanguageClient_restartOnCrash', 1)",
+                "get(g:, 'LanguageClient_maxRestartRetries', 5)",
+            ]
+            .as_ref(),
+        )?;
+
         // vimscript use 1 for true, 0 for false.
         let auto_start = auto_start == 1;
         let selection_ui_auto_open = selection_ui_auto_open == 1;
@@ -328,6 +337,9 @@ impl LanguageClient {
             state.preferred_markup_kind = preferred_markup_kind;
             state.enable_extensions = enable_extensions;
             state.code_lens_hl_group = code_lens_hl_group;
+            state.max_restart_retries = max_restart_retries;
+            state.restart_on_crash = restart_on_crash == 1;
+
             Ok(())
         })?;
 
@@ -3832,12 +3844,20 @@ impl LanguageClient {
                 (child_id, reader, writer)
             };
 
+        let lcn = self.clone();
+        let on_server_crash = move |language_id: &LanguageId| -> () {
+            if let Err(err) = lcn.on_server_crash(language_id) {
+                error!("Restart attempt failed: {}", err);
+            }
+        };
+
         let client = RpcClient::new(
             Some(language_id.clone()),
             reader,
             writer,
             child_id,
             self.get(|state| state.tx.clone())?,
+            on_server_crash,
         )?;
         self.update(|state| {
             state
@@ -3876,6 +3896,55 @@ impl LanguageClient {
         Ok(Value::Null)
     }
 
+    fn on_server_crash(&self, language_id: &LanguageId) -> Result<()> {
+        if language_id.is_none() {
+            return Ok(());
+        }
+
+        self.vim()?
+            .rpcclient
+            .notify("s:ExecuteAutocmd", "LanguageServerCrashed")?;
+        let filename = self.vim()?.get_filename(&Value::Null)?;
+        self.vim()?
+            .rpcclient
+            .notify("setbufvar", json!([filename, VIM_IS_SERVER_RUNNING, 0]))?;
+
+        let restart_on_crash = self.get(|state| state.restart_on_crash)?;
+        if !restart_on_crash {
+            return Ok(());
+        }
+
+        let max_restart_retries = self.get(|state| state.max_restart_retries)?;
+        let mut restarts =
+            self.get(|state| state.restarts.get(language_id).cloned().unwrap_or_default())?;
+        restarts += 1;
+
+        self.update(|state| {
+            let mut restarts = restarts;
+            if restarts > max_restart_retries {
+                restarts = 0;
+            };
+
+            state.clients.remove(language_id);
+            state.restarts.insert(language_id.clone(), restarts);
+            Ok(())
+        })?;
+
+        if restarts > max_restart_retries {
+            self.vim()?.echoerr(format!(
+                "Server for {} restarted too many times, not retrying any more.",
+                language_id.clone().unwrap()
+            ))?;
+            return Ok(());
+        }
+
+        self.vim()?.echoerr("Server crashed, restarting client")?;
+        std::thread::sleep(Duration::from_millis(300 * (restarts as u64).pow(2)));
+        self.start_server(&json!({"languageId": language_id.clone().unwrap()}))?;
+
+        Ok(())
+    }
+
     pub fn handle_server_exited(&self, params: &Value) -> Result<()> {
         let filename = self.vim()?.get_filename(params)?;
         let language_id = self.vim()?.get_language_id(&filename, params)?;

src/rpcclient.rs

@@ -43,15 +43,24 @@ impl RpcClient {
         writer: impl Write + Send + 'static,
         process_id: Option<u32>,
         sink: Sender<Call>,
+        on_crash: impl Fn(&LanguageId) -> () + Clone + Send + 'static,
     ) -> Result<Self> {
         let (reader_tx, reader_rx): (Sender<(Id, Sender<jsonrpc_core::Output>)>, _) = unbounded();
 
         let language_id_clone = language_id.clone();
         let reader_thread_name = format!("reader-{:?}", language_id);
+        let on_crash_clone = on_crash.clone();
         thread::Builder::new()
             .name(reader_thread_name.clone())
             .spawn(move || {
                 if let Err(err) = loop_read(reader, reader_rx, &sink, &language_id_clone) {
+                    match err.downcast_ref::<std::io::Error>() {
+                        Some(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
+                            on_crash_clone(&language_id_clone)
+                        }
+                        _ => {}
+                    }
+
                     error!("Thread {} exited with error: {:?}", reader_thread_name, err);
                 }
             })?;
@@ -63,6 +72,13 @@ impl RpcClient {
             .name(writer_thread_name.clone())
             .spawn(move || {
                 if let Err(err) = loop_write(writer, &writer_rx, &language_id_clone) {
+                    match err.downcast_ref::<std::io::Error>() {
+                        Some(err) if err.kind() == std::io::ErrorKind::BrokenPipe => {
+                            on_crash(&language_id_clone)
+                        }
+                        _ => {}
+                    }
+
                     error!("Thread {} exited with error: {:?}", writer_thread_name, err);
                 }
             })?;

src/types.rs

@@ -142,6 +142,7 @@ pub struct State {
 
     #[serde(skip_serializing)]
     pub clients: HashMap<LanguageId, Arc<RpcClient>>,
+    pub restarts: HashMap<LanguageId, u8>,
 
     #[serde(skip_serializing)]
     pub vim: Vim,
@@ -215,6 +216,8 @@ pub struct State {
     pub server_stderr: Option<String>,
     pub logger: Logger,
     pub preferred_markup_kind: Option<Vec<MarkupKind>>,
+    pub restart_on_crash: bool,
+    pub max_restart_retries: u8,
 }
 
 impl State {
@@ -228,6 +231,7 @@ impl State {
             BufWriter::new(std::io::stdout()),
             None,
             tx.clone(),
+            |_: &LanguageId| {},
         )?);
 
         Ok(Self {
@@ -236,6 +240,7 @@ impl State {
             clients: hashmap! {
                 None => client.clone(),
             },
+            restarts: HashMap::new(),
 
             vim: Vim::new(client),
 
@@ -296,6 +301,8 @@ impl State {
             preferred_markup_kind: None,
             enable_extensions: None,
             code_lens_hl_group: "Comment".into(),
+            restart_on_crash: true,
+            max_restart_retries: 5,
 
             logger,
         })