frontend: Add support to video decoder using CDR

core/frontend/package.json

@@ -17,6 +17,8 @@
     "@eclipse-zenoh/zenoh-ts": "1.3.4",
     "@ffmpeg/ffmpeg": "^0.12.15",
     "@ffmpeg/util": "^0.12.2",
+    "@foxglove/rosmsg": "^5.0.4",
+    "@foxglove/rosmsg2-serialization": "^3.0.2",
     "@google/model-viewer": "^3.0.0",
     "@mdi/font": "^7.1.96",
     "@sentry/vite-plugin": "^3.2.2",

core/frontend/public/msgs/CompressedVideo.msg

@@ -0,0 +1,58 @@
+# foxglove_msgs/msg/CompressedVideo
+# A single frame of a compressed video bitstream
+
+# Generated by https://github.com/foxglove/foxglove-sdk
+
+# Timestamp of video frame
+#builtin_interfaces/Time timestamp
+# TODO: Allow merging messages on the fly
+# This message communicates ROS Time defined here:
+# https://design.ros2.org/articles/clock_and_time.html
+
+# The seconds component, valid over all int32 values.
+int32 sec
+
+# The nanoseconds component, valid in the range [0, 1e9), to be added to the seconds component.
+# e.g.
+# The time -1.7 seconds is represented as {sec: -2, nanosec: 3e8}
+# The time 1.7 seconds is represented as {sec: 1, nanosec: 7e8}
+uint32 nanosec
+
+
+
+# Frame of reference for the video.
+#
+# The origin of the frame is the optical center of the camera. +x points to the right in the video, +y points down, and +z points into the plane of the video.
+string frame_id
+
+# Compressed video frame data.
+#
+# For packet-based video codecs this data must begin and end on packet boundaries (no partial packets), and must contain enough video packets to decode exactly one image (either a keyframe or delta frame). Note: Foxglove does not support video streams that include B frames because they require lookahead.
+#
+# Specifically, the requirements for different `format` values are:
+#
+# - `h264`
+#   - Use Annex B formatted data
+#   - Each CompressedVideo message should contain enough NAL units to decode exactly one video frame
+#   - Each message containing a key frame (IDR) must also include a SPS NAL unit
+#
+# - `h265` (HEVC)
+#   - Use Annex B formatted data
+#   - Each CompressedVideo message should contain enough NAL units to decode exactly one video frame
+#   - Each message containing a key frame (IRAP) must also include relevant VPS/SPS/PPS NAL units
+#
+# - `vp9`
+#   - Each CompressedVideo message should contain exactly one video frame
+#
+# - `av1`
+#   - Use the "Low overhead bitstream format" (section 5.2)
+#   - Each CompressedVideo message should contain enough OBUs to decode exactly one video frame
+#   - Each message containing a key frame must also include a Sequence Header OBU
+uint8[] data
+
+# Video format.
+#
+# Supported values: `h264`, `h265`, `vp9`, `av1`.
+#
+# Note: compressed video support is subject to hardware limitations and patent licensing, so not all encodings may be supported on all platforms. See more about [H.265 support](https://caniuse.com/hevc), [VP9 support](https://caniuse.com/webm), and [AV1 support](https://caniuse.com/av1).
+string format

core/frontend/src/components/zenoh-inspector/ZenohInspector.vue

@@ -122,15 +122,20 @@
 
 <script lang="ts">
 import {
-  Config, Sample, SampleKind, Session, Subscriber,
+  Config, Encoding, Sample, SampleKind, Session, Subscriber, ZBytes,
 } from '@eclipse-zenoh/zenoh-ts'
+import { parse as parseMessageDefinition } from '@foxglove/rosmsg'
+import { MessageReader } from '@foxglove/rosmsg2-serialization'
+import axios from 'axios'
 import Vue from 'vue'
 
 import RawVideoPlayer from './RawVideoPlayer.vue'
 
 interface ZenohMessage {
   topic: string
-  payload: string
+  payload: ZBytes
+  encoding: string
+  schema: string | undefined
   timestamp: Date
 }
 
@@ -151,6 +156,7 @@ export default Vue.extend({
       session: null as Session | null,
       subscriber: null as Subscriber | null,
       liveliness_subscriber: null as Subscriber | null,
+      video_reader: null as MessageReader | null,
     }
   },
   computed: {
@@ -171,19 +177,26 @@ export default Vue.extend({
       return this.selected_topic?.toLowerCase().includes('video') || false
     },
     videoData(): Uint8Array | null {
-      if (!this.current_message?.payload) {
+      if (!this.current_message?.payload || !this.video_reader) {
         return null
       }
-      return new Uint8Array(JSON.parse(this.current_message.payload)?.data)
+      const msg: { data: Uint8Array } = this.video_reader.readMessage(this.current_message.payload.to_bytes())
+      return msg.data
     },
   },
   async mounted() {
+    await this.setupVideoReader()
     await this.setupZenoh()
   },
   beforeDestroy() {
     this.disconnectZenoh()
   },
   methods: {
+    async setupVideoReader() {
+      const CompressedVideo = await axios.get('/msgs/CompressedVideo.msg').then((response) => response.data as string)
+      const definition = parseMessageDefinition(CompressedVideo)
+      this.video_reader = new MessageReader(definition)
+    },
     formatMessage(message: ZenohMessage | null): string {
       if (!message) return 'No messages received yet'
 
@@ -196,17 +209,19 @@ export default Vue.extend({
           : this.topic_liveliness[message.topic] ? 'Alive' : 'Dead',
         topic_type: this.topic_types[message.topic] || 'Unknown',
         message_type: this.topic_message_types[message.topic] || 'Unknown',
-        payload: message.payload,
+        payload: message.payload.toString(),
       }
 
-      if (typeof message.payload === 'string') {
+      if (message.encoding === Encoding.APPLICATION_JSON.toString()) {
+        formattedMessage.payload = JSON.parse(message.payload.to_string())
+      } else if (message.encoding === Encoding.ZENOH_BYTES.toString()) {
         try {
-          formattedMessage.payload = JSON.parse(message.payload)
+          formattedMessage.payload = JSON.parse(message.payload.to_string())
         } catch (exception) {
           // Keep the raw payload if it's not valid JSON
+          formattedMessage.payload = message.payload.toString()
         }
       }
-
       return JSON.stringify(formattedMessage, null, 2)
     },
 
@@ -222,9 +237,13 @@ export default Vue.extend({
           handler: async (sample: Sample) => {
             const topic = sample.keyexpr().toString()
             const payload = sample.payload()
+            const [encoding, schema] = sample.encoding().toString().split(';')
+
             const message: ZenohMessage = {
               topic,
-              payload: payload.to_string(),
+              payload,
+              encoding,
+              schema,
               timestamp: new Date(),
             }
 

core/frontend/vite.config.js

@@ -58,7 +58,7 @@ export default defineConfig(({ command, mode }) => {
         project: "blueos",
       })
     ],
-    assetsInclude: ['**/*.gif', '**/*.glb', '**/*.png', '**/*.svg', '**/assets/ArduPilot-Parameter-Repository**.json'],
+    assetsInclude: ['**/*.gif', '**/*.glb', '**/*.png', '**/*.svg', '**/assets/ArduPilot-Parameter-Repository**.json', '**/*.msg'],
     resolve: {
       extensions: ['.mjs', '.js', '.ts', '.jsx', '.tsx', '.json', '.vue'],
       alias: {

core/frontend/yarn.lock

@@ -1197,6 +1197,43 @@
   resolved "https://registry.yarnpkg.com/@ffmpeg/util/-/util-0.12.2.tgz#6bab9d8022a5bed9a6ff0933e5235b9b38698599"
   integrity sha512-ouyoW+4JB7WxjeZ2y6KpRvB+dLp7Cp4ro8z0HIVpZVCM7AwFlHa0c4R8Y/a4M3wMqATpYKhC7lSFHQ0T11MEDw==
 
+"@foxglove/cdr@^3.3.0":
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/@foxglove/cdr/-/cdr-3.3.0.tgz#8d273e1d6a11b8b3c508162c41073d4dad78cc5c"
+  integrity sha512-CjeA6ka/0cddVzQZY0qTEbExw849C7dD1aGpa+KeEzx0LZNxsQvvhsqWeGjszVM4BzwPcyWHwsiOZv27zbnnFA==
+
+"@foxglove/message-definition@^0.3.1":
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/@foxglove/message-definition/-/message-definition-0.3.1.tgz#63f48e8f3de47bba8943d8bfa8021af635254f1c"
+  integrity sha512-nkPowiED67LjcKEC77CprkUG3XvSsFHHR9HEwWCuhnIC2wm0W57T1J+WWvteoArZ7SdGGlKzSYSRFyjQkgmITw==
+
+"@foxglove/message-definition@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@foxglove/message-definition/-/message-definition-0.4.0.tgz#ecce0c30a58e0ddda776045e486c64dc29d501c1"
+  integrity sha512-lo98RFt+w7B3s+G3G431xyvXcc49GWRDLCj4NxPK5TXS3BkMKpe4+FVj4I3fSF87xYAFE52SgMqvBVUHnnRWQw==
+
+"@foxglove/rosmsg2-serialization@^3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@foxglove/rosmsg2-serialization/-/rosmsg2-serialization-3.0.2.tgz#e22733913870f265f2d0724d33eb5fea2c050067"
+  integrity sha512-NKMmRqyKWQk5mkWzE6WvYWdOUqnQzySgn+FLPa3Bc6eZt22ARZ9QNtX4MpymmCXNjC8Iocp/jpLC0uCrDPMILw==
+  dependencies:
+    "@foxglove/cdr" "^3.3.0"
+    "@foxglove/message-definition" "^0.4.0"
+    "@foxglove/rostime" "^1.1.2"
+
+"@foxglove/rosmsg@^5.0.4":
+  version "5.0.4"
+  resolved "https://registry.yarnpkg.com/@foxglove/rosmsg/-/rosmsg-5.0.4.tgz#445328d9722dff6960e060113a930f64a76b3ecb"
+  integrity sha512-s0JQLA6Zi0Qh9HtzyAan30sjiMU0u34+fkwccek/Xt+aQjCkSoErP/U4NBP6hCDfp0fBcb7CDSU2ggLFy/rJAQ==
+  dependencies:
+    "@foxglove/message-definition" "^0.3.1"
+    md5-typescript "^1.0.5"
+
+"@foxglove/rostime@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@foxglove/rostime/-/rostime-1.1.2.tgz#f8e1f10bff115c22be8a3e06791c2ac800449ee8"
+  integrity sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw==
+
 "@google/model-viewer@^3.0.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@google/model-viewer/-/model-viewer-3.1.0.tgz#2d6fa32ca1a0e0d30e457a448a49aa41b28de037"
@@ -4968,6 +5005,11 @@ marked@^4.1.1:
   resolved "https://registry.yarnpkg.com/marked/-/marked-4.3.0.tgz#796362821b019f734054582038b116481b456cf3"
   integrity sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==
 
+md5-typescript@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/md5-typescript/-/md5-typescript-1.0.5.tgz#68c0b24dff8e5d3162e498fa9893b63be72e038f"
+  integrity sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw==
+
 median-quickselect@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/median-quickselect/-/median-quickselect-1.0.1.tgz#de3408035a5b2f0438a39b99893faf3e7d6177f8"