View source for Mozilla deepspeech: python recording example

This example is not complete (too laggy for production), but it's a simple implementation of deepspeech's usage,
mic-to-text.

<source lang="python">
#!/usr/bin/env python
""" Record Audio in one thread, Interpret Audio as text within another thread.

Notes:
    This is super CPU-dependent. On my (slower) laptop, the time between sample
    recordings inflated, resulting in samples being cut-off strangely (hi there, how are you --> hi).
    My guess is that this is because I am using threads instead of independent processes - I
    have not tested this guess).

    On my (faster) desktop computer, the extracted audio samples perfectly encapsulated
    simple sentences -- but there were still issues with lag, and interpretation:

        a) the lag was still quite heavy -- process-pools would help here a lot.
           If this cannot catch-up to realtime, the lag will continually get worse.

        b) the untrained deepspeech model was starting to show it's weakness in interpreting text.
           * training the model would likely help
           * The SpeechRecognition python module has a filter that would
             likely help clean up audio files before passing it to the model.

"""
from __future__ import absolute_import, division, print_function
from deepspeech import client
import deepspeech
import logging
import multiprocessing
import numpy
import os
import pprint
import pyaudio
import resampy
import threading
import wave
import wavio
import time


# =======
# globals
# =======

logger = logging.getLogger(__name__)     # logger setup
pa = pyaudio.PyAudio()                   # init portaudio system
sample_queue = multiprocessing.Queue()   # queue shared by producer/consumer


# =============
# configuration
# =============

# device settings
# ---------------
# first device containing this text in it's name will be chosen
# as the Input Device for recording
audio_device_match = 'HDA Intel PCH: ALC3232 Analog'
audio_device_match = 'HD Pro'

# audio with volume registered between following will be counted as silence.
# (samplewidth * silence_threshold)
# and (samplewidth * silence_threshold * -1)
silence_threshold = 0.1


# audio settings
# --------------
channels = 1
target_rate = 16000                # mozilla-deepspeech requires samplerate-16000 MHz
pyaudio_fmt = pyaudio.paInt16      # sample-width
chunksize_samplerate_ratio = 0.1   # samples will be processed as this fraction-of-a-second

# seconds audio must be blank for N seconds before a command
# counts as complete, and it is processed by the deepspeech model
end_pause_seconds = 1
end_pause_frames = target_rate * end_pause_seconds

# deepspeech settings
# -------------------
deepspeech_models_dir = '/home/will/Downloads/deepspeech-0.5.1-models'


# ===========
# device info
# ===========


def print_input_devices():
    """ Prints all audio devices with their device-indexes.

    Notes:
        To help you determine audio device you want to record wtih.
    """
    print('------------')
    for i in range(pa.get_device_count()):
        device_info = pa.get_device_info_by_index(i)
        if device_info['maxInputChannels']:
            print('{name}: in({maxInputChannels})'.format(**device_info))


def get_input_device_index(device_text):
    """ Get device index from part of it's device-name.

    Args:
        device_text (str): ``(ex: 'HDA Intel PCM: ALC3232 Analog')``
            A string with enough of your desired recording device's `name` field text
            to uniquely select it.

    See Also:
        * :py:func:`print_input_devices`

    Returns:
        int:
            index of audio device
    """
    for i in range(pa.get_device_count()):
        device_info = pa.get_device_info_by_index(i)
        if device_text in device_info['name']:
            pprint.pprint(device_info, indent=2)
            return i
    raise RuntimeError('Unable to find known webcam')


def get_device_samplerate(device_index):
    """ Gets default samplerate for device.

    Returns:
        int: samplerate in Hz
    """
    device_info = pa.get_device_info_by_index(device_index)
    return int(device_info['defaultSampleRate'])


# =================
# Producer/Consumer
# =================


class RecordAudioThread(threading.Thread):
    """ Thread that records audio, queueing audio samples in chunks.
    """
    def __init__(self, input_device_index, input_samplerate):
        super(RecordAudioThread, self).__init__()
        self._device_index = input_device_index
        self._input_rate = input_samplerate

    def run(self):
        """ Records audio on a loop, dumping each chunk into `sample_queue` to be
        processed somewhere else.
        """
        try:
            # record at 44100 MHz (few input devices support recording at 16000 Mhz)
            chunksize = int(self._input_rate * chunksize_samplerate_ratio)
            stream = pa.open(format=pyaudio_fmt,
                             channels=channels,
                             rate=self._input_rate,
                             input=True,
                             frames_per_buffer=chunksize,
                             stream_callback=self._add_wavdata_to_samplequeue)

            # record indefinitely
            while True:
                stream.start_stream()

        finally:
            stream.stop_stream()
            stream.close()

    def _add_wavdata_to_samplequeue(self, raw_wavdata, frame_count, time_info, status):
        """ Dumps each recorded chunk into sample_queue (to be processed by consumer thread).
        """
        sample_queue.put(raw_wavdata)
        return (raw_wavdata, pyaudio.paContinue)


class AudioConsumerThread(threading.Thread):
    """ Retrieves samples from `sample_queue` , assembling non-silent audio until `silence_threshold` is reached.
    Then interprets audio using deepspeech, and prints it as text.

    Notes:
        this could benefit from being a processpool, to make it more responsive.
    """
    def __init__(self, input_samplerate):
        super(AudioConsumerThread, self).__init__()

        self._input_rate = input_samplerate

        self._model = deepspeech.Model(
            aModelPath='{}/output_graph.pb'.format(deepspeech_models_dir),
            aNCep=client.N_FEATURES,       # 26
            aNContext=client.N_CONTEXT,    # 9
            aAlphabetConfigPath='{}/alphabet.txt'.format(deepspeech_models_dir),
            aBeamWidth=client.BEAM_WIDTH,  # 500
        )

    def run(self):
        """ Run in separate thread, interprets speech.
        """
        command_chunks = []
        pause_frames = 0
        i = 0
        chunks_processed = False
        logger.info('Processing Samples Started')

        while True:
            wavdata = sample_queue.get()
            wav_array = self._wavdata_to_16bit_nparray(wavdata, self._input_rate)

            # if sample is not silent, append to command_chunks
            if not self._check_wavarray_is_silent(wav_array):
                logger.debug('Sample Contains Sound')
                command_chunks.append(wav_array)

            # if sample is silent, continue
            else:
                # if no audio recorded to date, ignore and continue
                # processing queue.
                if len(command_chunks) == 0:
                    if chunks_processed is False:
                        logger.debug('First Sample Without Sound, No Text Queued Yet')
                    pause_frames = 0
                    chunks_processed = True
                    continue

                # if audio has been recorded, count seconds of silence to
                # determine we should interpret the command, or continue
                # listening to samples.
                pause_frames += int(target_rate * chunksize_samplerate_ratio)
                logger.debug('Sample Without Sound (pause-{}/{}) (queuesize-{})'.format(pause_frames, end_pause_frames, sample_queue.qsize()))

                if pause_frames >= end_pause_frames:
                    command_array = numpy.concatenate(command_chunks)
                    logger.debug('Command Details: Num Samples: {}, Total Frames: {}'\
                                 .format(len(command_chunks), command_array.size))

                    self._write_audiosample(command_num=i, wav_array=command_array)
                    spoken_text = self._model.stt(command_array, target_rate)
                    logger.info('Spoken Text: {}'.format(spoken_text))

                    # reset command, increment command no#
                    command_chunks = []
                    i += 1

            chunks_processed = True

    def _wavdata_to_16bit_nparray(self, raw_wavdata, input_rate):
        """ Converts raw wavdata recorded at samplerate `input_rate` to
        a 1-dimensional numpy array representing wavedata at 16000Hz (deepspeech requirement).

        Returns:
            numpy.array
        """
        nparray_wavdata = numpy.frombuffer(raw_wavdata, dtype=numpy.int16)
        resampled_nparray_wavdata = resampy.resample(nparray_wavdata, input_rate, target_rate)
        return resampled_nparray_wavdata

    def _check_wavarray_is_silent(self, wav_array):
        """
        Returns:
            bool: True if wavarray is determined to be silent.
        """
        # 16bit int (unsigned) (-32,768 to 32,767)
        # https://stackoverflow.com/questions/28632721/does-16bit-integer-pcm-data-mean-its-signed-or-unsigned
        sample_max = 32767

        # skip processing if sample has no sound above/below a specific threshold
        above_threshold = sample_max * silence_threshold
        below_threshold = sample_max * silence_threshold * -1
        wav_array_wo_silence = numpy.array(list(filter(
            lambda x: x > above_threshold or x < below_threshold,
            wav_array
        )))
        return wav_array_wo_silence.size == 0

    def _write_audiosample(self, command_num, wav_array):
        """ Writes wav-array to ``/var/tmp/speechexe/{}.wav`` , to help debug program,
        and possibly in future, train deepspeech model.
        """
        if not os.path.isdir('/var/tmp/speechexe'):
            os.makedirs('/var/tmp/speechexe')
        wavio.write('/var/tmp/speechexe/{}.wav'.format(str(command_num)), wav_array, rate=target_rate)


if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    def run_listen():
        # choose device, and samplerate
        device_index = get_input_device_index(audio_device_match)
        input_samplerate = get_device_samplerate(device_index)

        # begin recording/interpreting
        producer = RecordAudioThread(device_index, input_samplerate)
        consumer = AudioConsumerThread(input_samplerate)
        consumer.start()
        producer.start()


    #print_input_devices()
    run_listen()

</source>