/* * Copyright (c) 2011- Osmo Antero. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 3 of the License (GPL3), or any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Library General Public License 3 for more details. * * You should have received a copy of the GNU Library General Public * License 3 along with this program; if not, see /usr/share/common-licenses/GPL file * or . */ #include "gst-vad.h" #include "timer.h" #include "audio-sources.h" #include "dconf.h" #include "log.h" #include "utility.h" #include "gst-pipeline.h" #include "gst-recorder.h" #include #include #include #include #include // Some information on how to implement VAD (Voice Activity Detector) using Gstreamer elements. // Ref: https://lists.freedesktop.org/archives/gstreamer-devel/2012-September/037233.html // // We are still using two (2) pipelines; one dummy pipeline for VAD with a level element, and // another pipeline for recording. // GStreamer pipeline for VAD static GstElement *g_vad_pipeline = NULL; // Debug flag (--debug-signal or -d argument) static gboolean g_debug_flag = FALSE; // Parameters for VAD-pipeline static PipelineParms *g_curr_parms = NULL; // Delay between reading values #define TRIGGER_TIME_MS 150 // in milliseconds static GstElement *vad_create_pipeline(PipelineParms *parms); static void vad_save_parms(PipelineParms *new_parms); static gboolean vad_is_running(); static void vad_shutdown_pipeline(); static gboolean vad_message_handler(GstBus * bus, GstMessage * message, gpointer data); void vad_module_init() { LOG_DEBUG("Init gst-vad.c.\n"); // Initialize g_vad_pipeline = NULL; g_curr_parms = NULL; g_debug_flag = FALSE; } void vad_module_exit() { LOG_DEBUG("Clean up gst-vad.c.\n"); vad_stop_VAD(); } void vad_set_debug_flag(gboolean on) { // Set debug flag. Please see arguments: // $ audio-recorder --help // VAD process will now print level values in a terminal window. gboolean active = FALSE; conf_get_boolean_value("timer-active", &active); if (on && (!active)) { // Just remind user about the timer checkbox g_print("Please activate the timer (checkbox) to see the level values.\n"); } g_debug_flag = on; } void vad_save_parms(PipelineParms *new_parms) { if (g_curr_parms) { pipeline_free_parms(g_curr_parms); } g_curr_parms = NULL; if (!new_parms) return; // Save the pointer g_curr_parms = new_parms; } void vad_start_VAD() { PipelineParms *parms = g_malloc0(sizeof(PipelineParms)); gint type = -1; gchar *dev_id = NULL; // These are set in the main window (in device/player combo) conf_get_int_value("audio-device-type", &type); conf_get_string_value("audio-device-id", &dev_id); // Get audio source and device list // parms->source = audio_sources_get_source(dev_id, type); parms->gst_plugin = audio_sources_get_gstreamer_plugin(); parms->gst_plugin_attrib = audio_sources_get_gstreamer_plugin_attrib(); parms->dev_list = audio_sources_get_devices(dev_id, type); g_free(dev_id); gboolean changed = TRUE; if (g_curr_parms) { changed = !str_lists_equal(parms->dev_list, g_curr_parms->dev_list); } // Changed? if (changed) { // Yes, stop VAD. vad_stop_VAD(); } // Already running (or values were unchanged)? if (vad_is_running()) { // Values not used! pipeline_free_parms(parms); parms = NULL; return; } // Create a new GStreamer pipeline for VAD g_vad_pipeline = vad_create_pipeline(parms); // Save values vad_save_parms(parms); } void vad_stop_VAD() { // Shutdown pipeline vad_shutdown_pipeline(); // Reset static variables vad_message_handler(NULL, NULL, NULL); // Clear PipelineParms vad_save_parms(NULL); } static gboolean vad_is_running() { // Valid pipeline? if (!GST_IS_OBJECT(g_vad_pipeline)) return FALSE; // The state is? // Ref: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/GstElement.html#gst-element-get-state GstState status, pending; gst_element_get_state(g_vad_pipeline, &status, &pending, 0); return (status != GST_STATE_NULL); } static void vad_check_triggers(GstClockTime timestamp, GstClockTimeDiff time_diff, gdouble rms_dB_left, gdouble rms_dB_right, gdouble peak_dB) { // From dB (decibel) to normalized value between [0 - 1.0]. // // Normalized value [0 - 1.0] = exp(dB / 20). // Decibel value = log10(normalized value) * 20. // Calc normalized values gdouble rms_left = exp(rms_dB_left/20.0); gdouble rms_right = exp(rms_dB_right/20.0); gdouble peak = exp(peak_dB/20.0); // Calculate average gdouble sum = 0.0; guint count = 0; // Left channel gave a value? if (rms_left > 0.001) { sum += rms_left; count++; } // Right channel gave a value? if (rms_right > 0.001) { sum += rms_right; count++; } // Calc avg gdouble rms_avg = 0.0; if (count > 0) { rms_avg = sum / count; } // Print values in a terminal window? // Got --debug-signal or -d argument? if (g_debug_flag) { gdouble rms_dB_avg = rms_dB_left > -120 ? ((rms_dB_left + rms_dB_right)/2.0) : rms_dB_right; g_print("Audio level. Time:%" GST_TIME_FORMAT ", RMS:%3.2f dB, normalized RMS:%3.2f, peak value:%3.2f.\n", GST_TIME_ARGS(timestamp), rms_dB_avg, rms_avg, peak); } // Evaluate triggers related to RMS value timer_evaluate_triggers(time_diff, rms_avg); } static gboolean vad_message_handler(GstBus * bus, GstMessage * message, gpointer data) { static GstClockTime g_last_timestamp = 0L; if (bus == NULL) { // Reset static variables g_last_timestamp = 0L; // And return return TRUE; } if (GST_MESSAGE_TYPE(message) != GST_MESSAGE_ELEMENT) { return TRUE; } const GstStructure *s = gst_message_get_structure(message); const gchar *name = gst_structure_get_name(s); GstClockTime timestamp; if (!gst_structure_get_clock_time(s, "timestamp", ×tamp)) { LOG_ERROR("vad_message_handler: Cannot read timestamp.\n"); } // Wait at least TRIGGER_TIME_MS milliseconds GstClockTimeDiff diff = GST_CLOCK_DIFF(g_last_timestamp, timestamp); if (diff < TRIGGER_TIME_MS * GST_MSECOND) { goto LBL_1; } g_last_timestamp = timestamp; if (g_str_equal(name, "level")) { // Ref: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/gst-plugins-good-plugins-level.html //Field names: //level field:endtime (GValueArray) //level field:timestamp (GValueArray) //level field:stream-time (GValueArray) //level field:running-time (GValueArray) //level field:duration (GValueArray) //level field:rms (GValueArray) //level field:peak (GValueArray) //level field:decay (GValueArray) const GValue *value = gst_structure_get_value (s, "rms"); GValueArray *array = (GValueArray*)g_value_get_boxed(value); if (!array) goto LBL_1; guint channels = array->n_values; if (channels < 1) goto LBL_1; gdouble rms_dB_left = -G_MAXDOUBLE; // left channel gdouble rms_dB_right = -G_MAXDOUBLE; // right channel if (channels > 0) { // Ref: https://lists.freedesktop.org/archives/gstreamer-devel/2012-October/037538.html rms_dB_left = g_value_get_double(array->values+0); } if (channels > 1) { rms_dB_right = g_value_get_double(array->values+1); } // peak_dB: value = gst_structure_get_value(s, "peak"); array = (GValueArray*)g_value_get_boxed(value); gdouble peak_dB = g_value_get_double(array->values+0); // Evaluate timer triggers vad_check_triggers(timestamp, diff, rms_dB_left, rms_dB_right, peak_dB); } LBL_1: // We handled the message we want, and ignored the ones we didn't want. so the core can unref the message for us return TRUE; } static void vad_shutdown_pipeline() { // Shutdown VAD-pipeline if (!GST_IS_PIPELINE(g_vad_pipeline)) { goto LBL_1; } LOG_VAD("Shutdown VAD pipeline.\n"); // Switch to GST_STATE_NULL gst_element_set_state(g_vad_pipeline, GST_STATE_NULL); // Then destroy it gst_object_unref(GST_OBJECT(g_vad_pipeline)); g_vad_pipeline = NULL; LBL_1: ; } static GstElement *vad_create_pipeline(PipelineParms *parms) { if (!parms) return NULL; #if defined(DEBUG_VAD) || defined(DEBUG_ALL) LOG_VAD("Start VAD for \"%s\"\n", parms->gst_plugin); str_list_print("Monitor devices", parms->dev_list); #endif gchar *err_msg = NULL; GstElement *pipeline = pipeline_create_VAD(parms, &err_msg); // Errors? if (!GST_IS_PIPELINE(pipeline) || err_msg) { goto LBL_1; } // Add message handler GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); gst_bus_add_signal_watch(bus); // Monitor "level" messages g_signal_connect(bus, "message::element", G_CALLBACK(vad_message_handler), NULL); gst_object_unref(bus); // Roll pipeline if (gst_element_set_state(pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { err_msg = g_strdup(_("Cannot start reading from the stream/pipeline.\n")); goto LBL_1; } else { LOG_DEBUG("Pipeline for VAD (Voice Activity Detection) is running and OK.\n"); } // Ok return pipeline; LBL_1: if (!err_msg) { err_msg = g_strdup_printf(_("Cannot create audio pipeline. %s.\n"), "(VAD pipeline)"); } LOG_ERROR(err_msg); g_free(err_msg); // Destroy pipeline if (G_IS_OBJECT(pipeline)) { gst_element_set_state(pipeline, GST_STATE_NULL); gst_object_unref(GST_OBJECT(pipeline)); } // Return NULL return NULL; } gboolean vad_get_debug_flag() { // --debug-signal or -d flag return g_debug_flag; }