Audio: Add audio feature extractor component MFCC

src/arch/host/configs/library_defconfig

@@ -7,3 +7,4 @@ CONFIG_COMP_MULTIBAND_DRC=y
 CONFIG_COMP_CODEC_ADAPTER=y
 CONFIG_COMP_SRC=y
 CONFIG_COMP_SRC_IPC4_FULL_MATRIX=y
+CONFIG_COMP_MFCC=y

src/arch/xtensa/configs/baytrail_defconfig

@@ -20,3 +20,4 @@ CONFIG_PCM_CONVERTER_FORMAT_FLOAT=n
 CONFIG_FORMAT_FLOAT=n
 CONFIG_COMP_BLOB=n
 CONFIG_COMP_DAI_GROUP=n
+CONFIG_COMP_MFCC=n

src/arch/xtensa/configs/baytrail_gcc_defconfig

@@ -20,3 +20,4 @@ CONFIG_PCM_CONVERTER_FORMAT_FLOAT=n
 CONFIG_FORMAT_FLOAT=n
 CONFIG_COMP_BLOB=n
 CONFIG_COMP_DAI_GROUP=n
+CONFIG_COMP_MFCC=n

src/arch/xtensa/configs/broadwell_defconfig

@@ -3,3 +3,4 @@ CONFIG_COMP_ASRC=n
 CONFIG_HAVE_AGENT=n
 CONFIG_COMP_SRC_TINY=y
 CONFIG_COMP_LEGACY_INTERFACE=y
+CONFIG_COMP_MFCC=n

src/arch/xtensa/configs/cherrytrail_defconfig

@@ -21,3 +21,4 @@ CONFIG_PCM_CONVERTER_FORMAT_FLOAT=n
 CONFIG_FORMAT_FLOAT=n
 CONFIG_COMP_BLOB=n
 CONFIG_COMP_DAI_GROUP=n
+CONFIG_COMP_MFCC=n

src/arch/xtensa/configs/cherrytrail_gcc_defconfig

@@ -21,3 +21,4 @@ CONFIG_PCM_CONVERTER_FORMAT_FLOAT=n
 CONFIG_FORMAT_FLOAT=n
 CONFIG_COMP_BLOB=n
 CONFIG_COMP_DAI_GROUP=n
+CONFIG_COMP_MFCC=n

src/arch/xtensa/configs/haswell_defconfig

@@ -2,3 +2,4 @@ CONFIG_HASWELL=y
 CONFIG_COMP_ASRC=n
 CONFIG_HAVE_AGENT=n
 CONFIG_COMP_SRC_TINY=y
+CONFIG_COMP_MFCC=n

src/audio/CMakeLists.txt

@@ -151,6 +151,9 @@ if(NOT CONFIG_LIBRARY)
 	if(CONFIG_COMP_UP_DOWN_MIXER)
 		add_subdirectory(up_down_mixer)
 	endif()
+	if(CONFIG_COMP_MFCC)
+		add_subdirectory(mfcc)
+	endif()
 	subdirs(pipeline)
 
 	return()
@@ -200,7 +203,7 @@ check_optimization(fma -mfma -ftree-vectorize -DOPS_FMA)
 check_optimization(hifi2ep -mhifi2ep "" -DOPS_HIFI2EP)
 check_optimization(hifi3 -mhifi3 "" -DOPS_HIFI3)
 
-set(sof_audio_modules mixer volume src asrc eq-fir eq-iir dcblock crossover tdfb drc multiband_drc)
+set(sof_audio_modules mixer volume src asrc eq-fir eq-iir dcblock crossover tdfb drc multiband_drc mfcc)
 
 # sources for each module
 set(volume_sources module_adapter/module_adapter.c module_adapter/module/generic.c module_adapter/module/volume/volume.c module_adapter/module/volume/volume_generic.c)
@@ -214,6 +217,7 @@ set(crossover_sources crossover/crossover.c crossover/crossover_generic.c)
 set(tdfb_sources tdfb/tdfb.c tdfb/tdfb_generic.c tdfb/tdfb_direction.c)
 set(drc_sources drc/drc.c drc/drc_generic.c drc/drc_math_generic.c)
 set(multiband_drc_sources multiband_drc/multiband_drc_generic.c crossover/crossover.c crossover/crossover_generic.c drc/drc.c drc/drc_generic.c drc/drc_math_generic.c multiband_drc/multiband_drc.c )
+set(mfcc_sources module_adapter/module_adapter.c module_adapter/module/generic.c mfcc/mfcc.c mfcc/mfcc_setup.c mfcc/mfcc_generic.c)
 
 foreach(audio_module ${sof_audio_modules})
 	# first compile with no optimizations

src/audio/Kconfig

@@ -483,6 +483,30 @@ config COMP_RTNR
 	  proprietary binary libSOF_RTK_MA_API.a, libSuite_rename.a, libNet.a and libPreset.a.
 	  Please contact [email protected] for any question about the binary.
 
+config COMP_MFCC
+	bool "MFCC component"
+	depends on COMP_MODULE_ADAPTER
+	depends on !COMP_LEGACY_INTERFACE
+	select CORDIC_FIXED
+	select MATH_16BIT_MEL_FILTERBANK
+	select MATH_AUDITORY
+	select MATH_DCT
+	select MATH_DECIBELS
+	select MATH_FFT
+	select MATH_MATRIX
+	select MATH_WINDOW
+	select NATURAL_LOGARITHM_FIXED
+	select NUMBERS_NORM
+	select SQRT_FIXED
+	default y
+	help
+	  This option enables build of a Mel-frequency cepstral coefficients
+	  (MFCC) audio features extractor component. It converts the input
+	  audio stream into encoded audio data for cepstral coefficients.
+	  The characteristic of the audio features are defined in the binary
+	  control blob. Directory tools/tune/mfcc contains a tool to create
+	  the configurations.
+
 endmenu # "Audio components"
 
 menu "Data formats"

src/audio/mfcc/CMakeLists.txt

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+add_local_sources(sof mfcc.c mfcc_setup.c mfcc_generic.c)

src/audio/mfcc/mfcc.c

@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Copyright(c) 2022 Intel Corporation. All rights reserved.
+//
+// Author: Seppo Ingalsuo <[email protected]>
+
+#include <sof/audio/mfcc/mfcc_comp.h>
+#include <sof/audio/module_adapter/module/generic.h>
+#include <sof/audio/component.h>
+#include <sof/audio/data_blob.h>
+#include <sof/audio/buffer.h>
+#include <sof/audio/format.h>
+#include <sof/audio/pipeline.h>
+#include <sof/audio/ipc-config.h>
+#include <sof/common.h>
+#include <sof/debug/panic.h>
+#include <sof/ipc/msg.h>
+#include <sof/lib/memory.h>
+#include <sof/lib/uuid.h>
+#include <sof/list.h>
+#include <sof/platform.h>
+#include <sof/ut.h>
+#include <sof/trace/trace.h>
+#include <ipc/control.h>
+#include <ipc/stream.h>
+#include <ipc/topology.h>
+#include <user/mfcc.h>
+#include <user/trace.h>
+#include <rtos/string.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+LOG_MODULE_REGISTER(mfcc, CONFIG_SOF_LOG_LEVEL);
+
+/* db10a773-1aa4-4cea-a21f-2d57a5c982eb */
+DECLARE_SOF_RT_UUID("mfcc", mfcc_uuid, 0xdb10a773, 0x1aa4, 0x4cea,
+		    0xa2, 0x1f, 0x2d, 0x57, 0xa5, 0xc9, 0x82, 0xeb);
+
+DECLARE_TR_CTX(mfcc_tr, SOF_UUID(mfcc_uuid), LOG_LEVEL_INFO);
+
+const struct mfcc_func_map mfcc_fm[] = {
+#if CONFIG_FORMAT_S16LE
+	{SOF_IPC_FRAME_S16_LE,  mfcc_s16_default},
+#endif /* CONFIG_FORMAT_S16LE */
+#if CONFIG_FORMAT_S24LE
+	{SOF_IPC_FRAME_S24_4LE, NULL},
+#endif /* CONFIG_FORMAT_S24LE */
+#if CONFIG_FORMAT_S32LE
+	{SOF_IPC_FRAME_S32_LE,  NULL},
+#endif /* CONFIG_FORMAT_S32LE */
+};
+
+static mfcc_func mfcc_find_func(enum sof_ipc_frame source_format,
+				enum sof_ipc_frame sink_format,
+				const struct mfcc_func_map *map,
+				int n)
+{
+	int i;
+
+	/* Find suitable processing function from map. */
+	for (i = 0; i < n; i++) {
+		if (source_format == map[i].source)
+			return map[i].func;
+	}
+
+	return NULL;
+}
+
+/*
+ * End of MFCC setup code. Next the standard component methods.
+ */
+
+static int mfcc_init(struct processing_module *mod)
+{
+	struct module_data *md = &mod->priv;
+	struct comp_dev *dev = mod->dev;
+	struct module_config *cfg = &md->cfg;
+	struct mfcc_comp_data *cd = NULL;
+	size_t bs = cfg->size;
+	int ret;
+
+	comp_info(dev, "mfcc_init()");
+
+	/* Check first that configuration blob size is sane */
+	if (bs > SOF_MFCC_CONFIG_MAX_SIZE) {
+		comp_err(dev, "mfcc_init() error: configuration blob size %u exceeds %d",
+			 bs, SOF_MFCC_CONFIG_MAX_SIZE);
+		return -EINVAL;
+	}
+
+	cd = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*cd));
+	if (!cd)
+		return -ENOMEM;
+
+	/* Handler for configuration data */
+	md->private = cd;
+	cd->model_handler = comp_data_blob_handler_new(dev);
+	if (!cd->model_handler) {
+		comp_err(dev, "mfcc_init(): comp_data_blob_handler_new() failed.");
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	/* Get configuration data */
+	ret = comp_init_data_blob(cd->model_handler, bs, cfg->data);
+	if (ret < 0) {
+		comp_err(mod->dev, "mfcc_init(): comp_init_data_blob() failed.");
+		goto err_init;
+	}
+
+	mod->simple_copy = true;
+	return 0;
+
+err_init:
+	comp_data_blob_handler_free(cd->model_handler);
+
+err:
+	rfree(cd);
+	return ret;
+}
+
+static int mfcc_free(struct processing_module *mod)
+{
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+
+	comp_info(mod->dev, "mfcc_free()");
+	comp_data_blob_handler_free(cd->model_handler);
+	mfcc_free_buffers(cd);
+	rfree(cd);
+	return 0;
+}
+
+static int mfcc_get_config(struct processing_module *mod,
+			   uint32_t config_id, uint32_t *data_offset_size,
+			   uint8_t *fragment, size_t fragment_size)
+{
+	struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+
+	comp_info(mod->dev, "mfcc_get_config()");
+
+	return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
+}
+
+static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
+			   enum module_cfg_fragment_position pos, uint32_t data_offset_size,
+			   const uint8_t *fragment, size_t fragment_size, uint8_t *response,
+			   size_t response_size)
+{
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+
+	comp_info(mod->dev, "mfcc_set_config()");
+
+	return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
+				  fragment, fragment_size);
+}
+
+static int mfcc_process(struct processing_module *mod,
+			struct input_stream_buffer *input_buffers, int num_input_buffers,
+			struct output_stream_buffer *output_buffers, int num_output_buffers)
+{
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+	struct audio_stream __sparse_cache *source = input_buffers->data;
+	struct audio_stream __sparse_cache *sink = output_buffers->data;
+	int frames = input_buffers->size;
+
+	comp_dbg(mod->dev, "mfcc_process(), start");
+
+	frames = MIN(frames, cd->max_frames);
+	cd->mfcc_func(mod, input_buffers, output_buffers, frames);
+
+	/* TODO: use module_update_buffer_position() from #6194 */
+	input_buffers->consumed += audio_stream_frame_bytes(source) * frames;
+	output_buffers->size += audio_stream_frame_bytes(sink) * frames;
+	comp_dbg(mod->dev, "mfcc_process(), done");
+	return 0;
+}
+
+static void mfcc_set_alignment(struct audio_stream *source, struct audio_stream *sink)
+{
+	const uint32_t byte_align = 1;
+	const uint32_t frame_align_req = 1;
+
+	audio_stream_init_alignment_constants(byte_align, frame_align_req, source);
+	audio_stream_init_alignment_constants(byte_align, frame_align_req, sink);
+}
+
+static int mfcc_prepare(struct processing_module *mod)
+{
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+	struct comp_buffer *sourceb;
+	struct comp_buffer *sinkb;
+	struct comp_buffer __sparse_cache *source_c;
+	struct comp_buffer __sparse_cache *sink_c;
+	struct comp_dev *dev = mod->dev;
+	enum sof_ipc_frame source_format;
+	enum sof_ipc_frame sink_format;
+	uint32_t sink_period_bytes;
+	int ret;
+
+	comp_info(dev, "mfcc_prepare()");
+
+	/* MFCC component will only ever have 1 source and 1 sink buffer */
+	sourceb = list_first_item(&dev->bsource_list, struct comp_buffer, sink_list);
+	sinkb = list_first_item(&dev->bsink_list, struct comp_buffer, source_list);
+	source_c = buffer_acquire(sourceb);
+	sink_c = buffer_acquire(sinkb);
+
+	/* get source data format */
+	source_format = source_c->stream.frame_fmt;
+
+	/* set align requirements */
+	mfcc_set_alignment(&source_c->stream, &sink_c->stream);
+
+	/* get sink data format and period bytes */
+	sink_format = sink_c->stream.frame_fmt;
+	sink_period_bytes = audio_stream_period_bytes(&sink_c->stream, dev->frames);
+	comp_info(dev, "mfcc_prepare(), source_format = %d, sink_format = %d",
+		  source_format, sink_format);
+	if (sink_c->stream.size < sink_period_bytes) {
+		comp_err(dev, "mfcc_prepare(): sink buffer size %d is insufficient < %d",
+			 sink_c->stream.size, sink_period_bytes);
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	cd->config = comp_get_data_blob(cd->model_handler, NULL, NULL);
+
+	/* Initialize MFCC, max_frames is set to dev->frames + 4 */
+	if (cd->config) {
+		ret = mfcc_setup(mod, dev->frames + 4, source_c->stream.rate,
+				 source_c->stream.channels);
+		if (ret < 0) {
+			comp_err(dev, "mfcc_prepare(), setup failed.");
+			goto err;
+		}
+	}
+
+	cd->mfcc_func = mfcc_find_func(source_format, sink_format, mfcc_fm, ARRAY_SIZE(mfcc_fm));
+	if (!cd->mfcc_func) {
+		comp_err(dev, "mfcc_prepare(), No proc func");
+		ret = -EINVAL;
+		goto err;
+	}
+
+	buffer_release(sink_c);
+	buffer_release(source_c);
+	return 0;
+
+err:
+	buffer_release(sink_c);
+	buffer_release(source_c);
+	comp_set_state(dev, COMP_TRIGGER_RESET);
+	return ret;
+}
+
+static int mfcc_reset(struct processing_module *mod)
+{
+	struct mfcc_comp_data *cd = module_get_private_data(mod);
+
+	comp_info(mod->dev, "mfcc_reset()");
+
+	/* Reset to similar state as init() */
+	cd->mfcc_func = NULL;
+	return 0;
+}
+
+static struct module_interface mfcc_interface = {
+		.init = mfcc_init,
+		.free = mfcc_free,
+		.set_configuration = mfcc_set_config,
+		.get_configuration = mfcc_get_config,
+		.process = mfcc_process,
+		.prepare = mfcc_prepare,
+		.reset = mfcc_reset,
+};
+
+DECLARE_MODULE_ADAPTER(mfcc_interface, mfcc_uuid, mfcc_tr);