Merge pull request #720 from OpenC3/dead_tlm

Better handle screen tlm request timeouts

Author: jmthomas

openc3-cosmos-init/plugins/packages/openc3-tool-common/src/components/Openc3Screen.vue

@@ -122,7 +122,18 @@
         </v-tooltip>
       </v-system-bar>
       <v-expand-transition v-if="!editDialog">
-        <div class="pa-1" ref="screen" v-show="expand">
+        <div
+          class="pa-1"
+          style="position: relative"
+          ref="screen"
+          v-show="expand"
+        >
+          <v-overlay
+            style="pointer-events: none"
+            :value="errors.length !== 0"
+            opacity="0.8"
+            absolute
+          />
           <vertical-widget
             :key="screenKey"
             :widgets="layoutStack[0].widgets"
@@ -151,13 +162,8 @@
         <span> Screen: {{ target }} {{ screen }} Errors </span>
         <v-spacer />
       </v-system-bar>
-      <v-card class="pa-3">
-        <v-row class="my-3">
-          <v-textarea readonly rows="13" :value="error" />
-        </v-row>
-        <v-row>
-          <v-btn block @click="clearErrors"> Clear </v-btn>
-        </v-row>
+      <v-card>
+        <v-textarea class="errors" readonly rows="13" :value="error" />
       </v-card>
     </v-dialog>
   </div>
@@ -263,6 +269,7 @@ export default {
       editDialog: false,
       expand: true,
       configParser: null,
+      configError: false,
       currentLayout: null,
       layoutStack: [],
       namedWidgets: {},
@@ -341,10 +348,11 @@ export default {
     } else {
       this.errors.push({
         type: 'error',
-        message: `${err}`,
+        message: err,
         time: new Date().getTime(),
       })
     }
+    this.configError = true
     return false
   },
   created() {
@@ -380,6 +388,7 @@ export default {
     },
     clearErrors: function () {
       this.errors = []
+      this.configError = false
     },
     updateRefreshInterval: function () {
       let refreshInterval = this.pollingPeriod * 1000
@@ -392,7 +401,7 @@ export default {
     },
     parseDefinition: function () {
       // Each time we start over and parse the screen definition
-      this.errors = []
+      this.clearErrors()
       this.namedWidgets = {}
       this.layoutStack = []
       this.dynamicWidgets = []
@@ -470,6 +479,7 @@ export default {
           lineNumber: lines.join(','),
           time: new Date().getTime(),
         })
+        this.configError = true
         // Create a simple VerticalWidget to replace the bad widget so
         // the layout stack can successfully unwind
         this.layoutStack[0] = {
@@ -725,18 +735,31 @@ export default {
       })
     },
     update: function () {
-      if (this.screenItems.length !== 0 && this.errors.length === 0) {
+      if (this.screenItems.length !== 0 && this.configError === false) {
         this.api
           .get_tlm_values(this.screenItems, this.staleTime)
           .then((data) => {
+            this.clearErrors()
             this.updateValues(data)
           })
           .catch((error) => {
-            this.errors.push({
-              type: 'usage',
-              message: error.message,
-              time: new Date().getTime(),
-            })
+            let message = JSON.stringify(error, null, 2)
+            // Anything other than 'no response received' which means the API server is down
+            // is an error the user needs to fix so don't request values until they do
+            if (!message.includes('no response received')) {
+              this.configError = true
+            }
+            if (
+              !this.errors.find((existing) => {
+                existing.message === message
+              })
+            ) {
+              this.errors.push({
+                type: 'error',
+                message: message,
+                time: new Date().getTime(),
+              })
+            }
           })
       }
     },
@@ -760,6 +783,10 @@ export default {
 </script>
 
 <style scoped>
+.errors {
+  padding-top: 0px;
+  margin-top: 0px;
+}
 .v-card {
   background-color: var(--v-tertiary-darken2);
 }

openc3-cosmos-init/plugins/packages/openc3-tool-common/src/services/openc3-api.js

@@ -27,7 +27,13 @@ export class OpenC3Api {
 
   constructor() {}
 
-  async exec(method, params, kwparams = {}, headerOptions = {}) {
+  async exec(
+    method,
+    params,
+    kwparams = {},
+    headerOptions = {},
+    timeout = 60000
+  ) {
     try {
       let refreshed = await OpenC3Auth.updateToken(
         OpenC3Auth.defaultMinValidity
@@ -56,6 +62,7 @@ export class OpenC3Api {
             'Content-Type': 'application/json-rpc',
             ...headerOptions,
           },
+          timeout: timeout,
         }
       )
       // var data = response.data
@@ -79,12 +86,13 @@ export class OpenC3Api {
         // is an instance of XMLHttpRequest in the browser and an instance
         // of http.ClientRequest in Node.js
         err.name = 'Request error'
+        // NOTE: Openc3Screen.vue uses this specific message to determine
+        // if the CmdTlmApi server is down. Don't change unless also changing there.
         err.message = 'Request error, no response received'
       } else {
         // Something happened in setting up the request and triggered an Error
         err.name = 'Unknown error'
       }
-      // console.log(error)
       throw err
     }
   }
@@ -305,9 +313,15 @@ export class OpenC3Api {
   }
 
   async get_tlm_values(items, stale_time = 30) {
-    const data = await this.exec('get_tlm_values', [items], {
-      stale_time: stale_time,
-    })
+    const data = await this.exec(
+      'get_tlm_values',
+      [items],
+      {
+        stale_time: stale_time,
+      },
+      {},
+      10000 // 10s timeout ... should never be this long
+    )
     var len = data[0].length
     var converted = null
     for (var i = 0; i < len; i++) {