xref: /aosp_15_r20/external/armnn/samples/SpeechRecognition/include/Wav2LetterPreprocessor.hpp (revision 89c4ff92f2867872bb9e2354d150bf0c8c502810)

//
// Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
// SPDX-License-Identifier: MIT
//
#ifndef SPEECH_RECOGNITION_EXAMPLE_WAV2LETTERPREPROCESSOR_HPP
#define SPEECH_RECOGNITION_EXAMPLE_WAV2LETTERPREPROCESSOR_HPP

#include <numeric>
#include "DataStructures.hpp"
#include "SlidingWindow.hpp"
#include "MFCC.hpp"
#include "Wav2LetterMFCC.hpp"
// Class to facilitate pre-processing calculation for Wav2Letter model for ASR
using AudioWindow = SlidingWindow<const float>;

class Wav2LetterPreprocessor
{
public:
    Wav2LetterPreprocessor(uint32_t windowLen, uint32_t windowStride,
                           std::unique_ptr<Wav2LetterMFCC> mfccInst);

    /**
     * @brief       Calculates the features required from audio data. This
     *              includes MFCC, first and second order deltas,
     *              normalisation and finally, quantisation. The tensor is
     *              populated with feature from a given window placed along
     *              in a single row.
     * @param[in]   audioData     pointer to the first element of audio data
     * @param[in]   audioDataLen  number of elements in the audio data
     * @param[in]   tensor        tensor to be populated
     * @return      true if successful, false in case of error.
     */
    bool Invoke(const float* audioData, uint32_t audioDataLen, std::vector<int8_t>& output, int quantOffset,
                float quantScale);

    std::unique_ptr<MFCC> m_mfcc;

    // Actual buffers to be populated
    Array2d<float> m_mfccBuf;         // Contiguous buffer 1D: MFCC
    Array2d<float> m_delta1Buf;       // Contiguous buffer 1D: Delta 1
    Array2d<float> m_delta2Buf;       // Contiguous buffer 1D: Delta 2

    uint32_t m_windowLen;       // Window length for MFCC
    uint32_t m_windowStride;    // Window stride len for MFCC
    AudioWindow m_window;       // Sliding window

protected:
    /**
     * @brief Computes the first and second order deltas for the
     *        MFCC buffers - they are assumed to be populated.
     *
     * @param[in]  mfcc   MFCC buffers
     * @param[out] delta1 result of the first diff computation
     * @param[out] delta2 result of the second diff computation
     *
     * @return true if successful, false otherwise
     */
    static bool ComputeDeltas(Array2d<float>& mfcc,
                              Array2d<float>& delta1,
                              Array2d<float>& delta2);

protected:

    /**
     * @brief      Given a 2D vector of floats, computes the mean
     * @param[in]   vec      vector of vector of floats
     * @return      mean value
     */
    static float GetMean(Array2d<float>& vec);

    /**
     * @brief       Given a 2D vector of floats, computes the stddev
     * @param[in]   vec   vector of vector of floats
     * @param[in]   mean     mean value of the vector passed in
     * @return      stddev value
     */
    static float GetStdDev(Array2d<float>& vec, float mean);

    /**
     * @brief           Given a 2D vector of floats, normalises it using
     *                  the mean and the stddev
     * @param[in/out]   vec      vector of vector of floats
     * @return
     */
    static void NormaliseVec(Array2d<float>& vec);

    /**
     * @brief       Normalises the MFCC and delta buffers
     * @return
     */
    void Normalise();

    /**
     * @brief       Given the quantisation and data type limits, computes
     *              the quantised values of a floating point input data.
     * @param[in]   elem            Element to be quantised
     * @param[in]   quantScale      Scale
     * @param[in]   quantOffset     Offset
     * @param[in]   minVal          Numerical limit - minimum
     * @param[in]   maxVal          Numerical limit - maximum
     * @return      floating point quantised value
     */
    static float GetQuantElem(
            float elem,
            float quantScale,
            int quantOffset,
            float minVal,
            float maxVal);

    /**
     * @brief       Quantises the MFCC and delta buffers, and places them
     *              in the output buffer. While doing so, it transposes
     *              the data. Reason: Buffers in this class are arranged
     *              for "time" axis to be row major. Primary reason for
     *              this being the convolution speed up (as we can use
     *              contiguous memory). The output, however, requires the
     *              time axis to be in column major arrangement.
     * @param[in]   outputBuf       pointer to the output buffer
     * @param[in]   outputBufSz     output buffer's size
     * @param[in]   quantScale      quantisation scale
     * @param[in]   quantOffset     quantisation offset
     */
    template<typename T>
    bool Quantise(T*outputBuf, int quantOffset, float quantScale)
    {
        // Populate
        T* outputBufMfcc = outputBuf;
        T* outputBufD1 = outputBuf + this->m_mfcc->m_params.m_numMfccFeatures;
        T* outputBufD2 = outputBufD1 + this->m_mfcc->m_params.m_numMfccFeatures;
        const uint32_t ptrIncr = this->m_mfcc->m_params.m_numMfccFeatures * 2; // (3 vectors - 1 vector)

        const float minVal = std::numeric_limits<T>::min();
        const float maxVal = std::numeric_limits<T>::max();

        // We need to do a transpose while copying and concatenating the tensor
        for (uint32_t j = 0; j < this->m_mfcc->m_params.m_numMfccVectors; ++j)
        {
            for (uint32_t i = 0; i < this->m_mfcc->m_params.m_numMfccFeatures; ++i)
            {
                *outputBufMfcc++ = static_cast<T>(Wav2LetterPreprocessor::GetQuantElem(
                        this->m_mfccBuf(i, j), quantScale,
                        quantOffset, minVal, maxVal));
                *outputBufD1++ = static_cast<T>(Wav2LetterPreprocessor::GetQuantElem(
                        this->m_delta1Buf(i, j), quantScale,
                        quantOffset, minVal, maxVal));
                *outputBufD2++ = static_cast<T>(Wav2LetterPreprocessor::GetQuantElem(
                        this->m_delta2Buf(i, j), quantScale,
                        quantOffset, minVal, maxVal));
            }
            outputBufMfcc += ptrIncr;
            outputBufD1 += ptrIncr;
            outputBufD2 += ptrIncr;
        }
        return true;
    }
};

#endif //SPEECH_RECOGNITION_EXAMPLE_WAV2LETTERPREPROCESSOR_HPP