Full implementation of WaveSurver Spectrogram

1

Hi, I'm attempting to implement a spectrogram visualization for an audio classification/transcription task. I'm trying to base my solution on this post, but am having no luck. I've tried the modification to the global_css value mentioned in the post as well as various other things, but nothing is working.

The post is also a little vague, e.g. "with something like", incomplete (no HTML included) making it difficult to implement (for a front end noob like myself anyway!)

Does anyone have a complete working example of wavesurfer integration with a audio interface?

2

Answering my own question. Based on this answer, with some serious help from front end friend, we got a spectrogram to load. However, we weren't able to append it to the existing wavesurfer instance available on window.prodigy.wavesurfer (not sure why, and I'm sure it should be possible, but we didn't have the time). We had to create a new wavesurfer instance and add a spectrogram to it, then associate them with separate windows, and then make the second wavesurfer waveform invisible (otherwise there were two).

See below for full hackery, including re-loading the base64 file as a Blob, and altering the height param (endless css commands did nothing).

function base64ToBlob(base64, contentType = "", sliceSize = 512) {
    const byteCharacters = atob(base64.split(",")[1]);
    const byteArrays = [];

    for (let offset = 0; offset < byteCharacters.length;
        offset += sliceSize) {
        const slice = byteCharacters.slice(
            offset, offset + sliceSize);

        const byteNumbers = new Array(slice.length);
        for (let i = 0; i < slice.length; i++) {
            byteNumbers[i] = slice.charCodeAt(i);
        }

        const byteArray = new Uint8Array(byteNumbers);
        byteArrays.push(byteArray);
    }

    const blob = new Blob(byteArrays, { type: contentType });
    return blob;
}

var createSpectro = async function() {
    const spectrogramContainer = document.querySelector('#spectogram-123');
    if (spectrogramContainer.firstChild) {
        spectrogramContainer.removeChild(spectrogramContainer.firstChild);
    }
    const { default: Spectrogram } = await import('https://unpkg.com/wavesurfer.js@7.8.9/dist/plugins/spectrogram.js');
    var wavesurfer = WaveSurfer.create({
        container: '#waveform-123',
        waveColor: '#0078ff',
        progressColor: '#005bb5',
        height: 0,
        responsive: true,
        plugins: [Spectrogram.create({
            labels: true,
            fftSamples: 1024,
            wavesurfer: window.wavesurfer,
            container: '#spectogram-123',
            height: 1024,
        })]
    });
    wavesurfer.loadBlob(base64ToBlob(window.prodigy.content.audio))
        .then(() => {
            document.querySelector('#spectogram-123').children[0].style.height = "256px";
        });
}

function loadScript(url, callback, type) {
    var head = document.head;
    var script = document.createElement('script');
    script.type = type;
    script.src = url;
    script.onreadystatechange = callback;
    script.onload = callback;
    head.appendChild(script);
}


function handleSpectrogramEvent(event) {
    loadScript("https://unpkg.com/wavesurfer.js@7.8.9/dist/wavesurfer.min.js", createSpectro, 'text/javascript');
    console.log(`Spectrogram updated due to event: ${event.type}`);
}

document.addEventListener('prodigyanswer', handleSpectrogramEvent)
document.addEventListener('prodigymount', handleSpectrogramEvent)


<div id="spectogram-123"></div>
<div id="waveform-123" style="height: 0; visibility: hidden;"></div>

This worked for us. I surely think a Prodigy native support for a spectrogram would be very useful, especially for phonetic annotation tasks!

EDIT: Original answer wasn't updating with new tasks, so I've added event listeners and deleted any existing spectrogramContainer.firstChild instance (otherwise, it was appending new spectrograms underneath existing ones).

3

Thanks for sharing your solution @cbjrobertson! We will definitely consider adding spectrogram support natively.