1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2	//
3	// This file is provided under a dual BSD/GPLv2 license. When using or
4	// redistributing this file, you may do so under either license.
5	//
6	// Copyright(c) 2022 Intel Corporation. All rights reserved.
7	//
8	//
9	#include <linux/bitfield.h>
10	#include <uapi/sound/sof/tokens.h>
11	#include <sound/pcm_params.h>
12	#include <sound/sof/ext_manifest4.h>
13	#include <sound/intel-nhlt.h>
14	#include "sof-priv.h"
15	#include "sof-audio.h"
16	#include "ipc4-priv.h"
17	#include "ipc4-topology.h"
18	#include "ops.h"
19
20	/*
21	* The ignore_cpc flag can be used to ignore the CPC value for all modules by
22	* using 0 instead.
23	* The CPC is sent to the firmware along with the SOF_IPC4_MOD_INIT_INSTANCE
24	* message and it is used for clock scaling.
25	* 0 as CPC value will instruct the firmware to use maximum frequency, thus
26	* deactivating the clock scaling.
27	*/
28	static bool ignore_cpc;
29	module_param_named(ipc4_ignore_cpc, ignore_cpc, bool, 0444);
30	MODULE_PARM_DESC(ipc4_ignore_cpc,
31	"Ignore CPC values. This option will disable clock scaling in firmware.");
32
33	#define SOF_IPC4_GAIN_PARAM_ID 0
34	#define SOF_IPC4_TPLG_ABI_SIZE 6
35	#define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2
36
37	static DEFINE_IDA(alh_group_ida);
38	static DEFINE_IDA(pipeline_ida);
39
40	static const struct sof_topology_token ipc4_sched_tokens[] = {
41	{SOF_TKN_SCHED_LP_MODE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
42	offsetof(struct sof_ipc4_pipeline, lp_mode)},
43	{SOF_TKN_SCHED_USE_CHAIN_DMA, SND_SOC_TPLG_TUPLE_TYPE_BOOL, get_token_u16,
44	offsetof(struct sof_ipc4_pipeline, use_chain_dma)},
45	{SOF_TKN_SCHED_CORE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
46	offsetof(struct sof_ipc4_pipeline, core_id)},
47	{SOF_TKN_SCHED_PRIORITY, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
48	offsetof(struct sof_ipc4_pipeline, priority)},
49	};
50
51	static const struct sof_topology_token pipeline_tokens[] = {
52	{SOF_TKN_SCHED_DYNAMIC_PIPELINE, SND_SOC_TPLG_TUPLE_TYPE_BOOL, get_token_u16,
53	offsetof(struct snd_sof_widget, dynamic_pipeline_widget)},
54	};
55
56	static const struct sof_topology_token ipc4_comp_tokens[] = {
57	{SOF_TKN_COMP_IS_PAGES, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
58	offsetof(struct sof_ipc4_base_module_cfg, is_pages)},
59	};
60
61	static const struct sof_topology_token ipc4_in_audio_format_tokens[] = {
62	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
63	offsetof(struct sof_ipc4_pin_format, audio_fmt.sampling_frequency)},
64	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_BIT_DEPTH, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
65	offsetof(struct sof_ipc4_pin_format, audio_fmt.bit_depth)},
66	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_CH_MAP, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
67	offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_map)},
68	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_CH_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
69	offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_cfg)},
70	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_INTERLEAVING_STYLE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
71	get_token_u32, offsetof(struct sof_ipc4_pin_format,
72	audio_fmt.interleaving_style)},
73	{SOF_TKN_CAVS_AUDIO_FORMAT_IN_FMT_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
74	offsetof(struct sof_ipc4_pin_format, audio_fmt.fmt_cfg)},
75	{SOF_TKN_CAVS_AUDIO_FORMAT_INPUT_PIN_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
76	offsetof(struct sof_ipc4_pin_format, pin_index)},
77	{SOF_TKN_CAVS_AUDIO_FORMAT_IBS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
78	offsetof(struct sof_ipc4_pin_format, buffer_size)},
79	};
80
81	static const struct sof_topology_token ipc4_out_audio_format_tokens[] = {
82	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
83	offsetof(struct sof_ipc4_pin_format, audio_fmt.sampling_frequency)},
84	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_BIT_DEPTH, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
85	offsetof(struct sof_ipc4_pin_format, audio_fmt.bit_depth)},
86	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_CH_MAP, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
87	offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_map)},
88	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_CH_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
89	offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_cfg)},
90	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_INTERLEAVING_STYLE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
91	get_token_u32, offsetof(struct sof_ipc4_pin_format,
92	audio_fmt.interleaving_style)},
93	{SOF_TKN_CAVS_AUDIO_FORMAT_OUT_FMT_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
94	offsetof(struct sof_ipc4_pin_format, audio_fmt.fmt_cfg)},
95	{SOF_TKN_CAVS_AUDIO_FORMAT_OUTPUT_PIN_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
96	offsetof(struct sof_ipc4_pin_format, pin_index)},
97	{SOF_TKN_CAVS_AUDIO_FORMAT_OBS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
98	offsetof(struct sof_ipc4_pin_format, buffer_size)},
99	};
100
101	static const struct sof_topology_token ipc4_copier_deep_buffer_tokens[] = {
102	{SOF_TKN_INTEL_COPIER_DEEP_BUFFER_DMA_MS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, 0},
103	};
104
105	static const struct sof_topology_token ipc4_copier_tokens[] = {
106	{SOF_TKN_INTEL_COPIER_NODE_TYPE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, 0},
107	};
108
109	static const struct sof_topology_token ipc4_audio_fmt_num_tokens[] = {
110	{SOF_TKN_COMP_NUM_INPUT_AUDIO_FORMATS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
111	offsetof(struct sof_ipc4_available_audio_format, num_input_formats)},
112	{SOF_TKN_COMP_NUM_OUTPUT_AUDIO_FORMATS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
113	offsetof(struct sof_ipc4_available_audio_format, num_output_formats)},
114	};
115
116	static const struct sof_topology_token dai_tokens[] = {
117	{SOF_TKN_DAI_TYPE, SND_SOC_TPLG_TUPLE_TYPE_STRING, get_token_dai_type,
118	offsetof(struct sof_ipc4_copier, dai_type)},
119	{SOF_TKN_DAI_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
120	offsetof(struct sof_ipc4_copier, dai_index)},
121	};
122
123	/* Component extended tokens */
124	static const struct sof_topology_token comp_ext_tokens[] = {
125	{SOF_TKN_COMP_UUID, SND_SOC_TPLG_TUPLE_TYPE_UUID, get_token_uuid,
126	offsetof(struct snd_sof_widget, uuid)},
127	{SOF_TKN_COMP_CORE_ID, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
128	offsetof(struct snd_sof_widget, core)},
129	};
130
131	static const struct sof_topology_token gain_tokens[] = {
132	{SOF_TKN_GAIN_RAMP_TYPE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
133	get_token_u32, offsetof(struct sof_ipc4_gain_params, curve_type)},
134	{SOF_TKN_GAIN_RAMP_DURATION,
135	SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
136	offsetof(struct sof_ipc4_gain_params, curve_duration_l)},
137	{SOF_TKN_GAIN_VAL, SND_SOC_TPLG_TUPLE_TYPE_WORD,
138	get_token_u32, offsetof(struct sof_ipc4_gain_params, init_val)},
139	};
140
141	/* SRC */
142	static const struct sof_topology_token src_tokens[] = {
143	{SOF_TKN_SRC_RATE_OUT, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
144	offsetof(struct sof_ipc4_src_data, sink_rate)},
145	};
146
147	static const struct sof_token_info ipc4_token_list[SOF_TOKEN_COUNT] = {
148	[SOF_DAI_TOKENS] = {"DAI tokens", dai_tokens, ARRAY_SIZE(dai_tokens)},
149	[SOF_PIPELINE_TOKENS] = {.name: "Pipeline tokens", .tokens: pipeline_tokens, ARRAY_SIZE(pipeline_tokens)},
150	[SOF_SCHED_TOKENS] = {.name: "Scheduler tokens", .tokens: ipc4_sched_tokens,
151	ARRAY_SIZE(ipc4_sched_tokens)},
152	[SOF_COMP_EXT_TOKENS] = {.name: "Comp extended tokens", .tokens: comp_ext_tokens,
153	ARRAY_SIZE(comp_ext_tokens)},
154	[SOF_COMP_TOKENS] = {.name: "IPC4 Component tokens",
155	.tokens: ipc4_comp_tokens, ARRAY_SIZE(ipc4_comp_tokens)},
156	[SOF_IN_AUDIO_FORMAT_TOKENS] = {.name: "IPC4 Input Audio format tokens",
157	.tokens: ipc4_in_audio_format_tokens, ARRAY_SIZE(ipc4_in_audio_format_tokens)},
158	[SOF_OUT_AUDIO_FORMAT_TOKENS] = {.name: "IPC4 Output Audio format tokens",
159	.tokens: ipc4_out_audio_format_tokens, ARRAY_SIZE(ipc4_out_audio_format_tokens)},
160	[SOF_COPIER_DEEP_BUFFER_TOKENS] = {.name: "IPC4 Copier deep buffer tokens",
161	.tokens: ipc4_copier_deep_buffer_tokens, ARRAY_SIZE(ipc4_copier_deep_buffer_tokens)},
162	[SOF_COPIER_TOKENS] = {.name: "IPC4 Copier tokens", .tokens: ipc4_copier_tokens,
163	ARRAY_SIZE(ipc4_copier_tokens)},
164	[SOF_AUDIO_FMT_NUM_TOKENS] = {.name: "IPC4 Audio format number tokens",
165	.tokens: ipc4_audio_fmt_num_tokens, ARRAY_SIZE(ipc4_audio_fmt_num_tokens)},
166	[SOF_GAIN_TOKENS] = {.name: "Gain tokens", .tokens: gain_tokens, ARRAY_SIZE(gain_tokens)},
167	[SOF_SRC_TOKENS] = {.name: "SRC tokens", .tokens: src_tokens, ARRAY_SIZE(src_tokens)},
168	};
169
170	struct snd_sof_widget *sof_ipc4_find_swidget_by_ids(struct snd_sof_dev *sdev,
171	u32 module_id, int instance_id)
172	{
173	struct snd_sof_widget *swidget;
174
175	list_for_each_entry(swidget, &sdev->widget_list, list) {
176	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
177
178	/* Only active module instances have valid instance_id */
179	if (!swidget->use_count)
180	continue;
181
182	if (fw_module && fw_module->man4_module_entry.id == module_id &&
183	swidget->instance_id == instance_id)
184	return swidget;
185	}
186
187	return NULL;
188	}
189
190	static void sof_ipc4_dbg_audio_format(struct device *dev, struct sof_ipc4_pin_format *pin_fmt,
191	int num_formats)
192	{
193	int i;
194
195	for (i = 0; i < num_formats; i++) {
196	struct sof_ipc4_audio_format *fmt = &pin_fmt[i].audio_fmt;
197	dev_dbg(dev,
198	"Pin index #%d: %uHz, %ubit (ch_map %#x ch_cfg %u interleaving_style %u fmt_cfg %#x) buffer size %d\n",
199	pin_fmt[i].pin_index, fmt->sampling_frequency, fmt->bit_depth, fmt->ch_map,
200	fmt->ch_cfg, fmt->interleaving_style, fmt->fmt_cfg,
201	pin_fmt[i].buffer_size);
202	}
203	}
204
205	static const struct sof_ipc4_audio_format *
206	sof_ipc4_get_input_pin_audio_fmt(struct snd_sof_widget *swidget, int pin_index)
207	{
208	struct sof_ipc4_base_module_cfg_ext *base_cfg_ext;
209	struct sof_ipc4_process *process;
210	int i;
211
212	if (swidget->id != snd_soc_dapm_effect) {
213	struct sof_ipc4_base_module_cfg *base = swidget->private;
214
215	/* For non-process modules, base module config format is used for all input pins */
216	return &base->audio_fmt;
217	}
218
219	process = swidget->private;
220	base_cfg_ext = process->base_config_ext;
221
222	/*
223	* If there are multiple input formats available for a pin, the first available format
224	* is chosen.
225	*/
226	for (i = 0; i < base_cfg_ext->num_input_pin_fmts; i++) {
227	struct sof_ipc4_pin_format *pin_format = &base_cfg_ext->pin_formats[i];
228
229	if (pin_format->pin_index == pin_index)
230	return &pin_format->audio_fmt;
231	}
232
233	return NULL;
234	}
235
236	/**
237	* sof_ipc4_get_audio_fmt - get available audio formats from swidget->tuples
238	* @scomp: pointer to pointer to SOC component
239	* @swidget: pointer to struct snd_sof_widget containing tuples
240	* @available_fmt: pointer to struct sof_ipc4_available_audio_format being filling in
241	* @module_base_cfg: Pointer to the base_config in the module init IPC payload
242	*
243	* Return: 0 if successful
244	*/
245	static int sof_ipc4_get_audio_fmt(struct snd_soc_component *scomp,
246	struct snd_sof_widget *swidget,
247	struct sof_ipc4_available_audio_format *available_fmt,
248	struct sof_ipc4_base_module_cfg *module_base_cfg)
249	{
250	struct sof_ipc4_pin_format *in_format = NULL;
251	struct sof_ipc4_pin_format *out_format;
252	int ret;
253
254	ret = sof_update_ipc_object(scomp, object: available_fmt,
255	token_id: SOF_AUDIO_FMT_NUM_TOKENS, tuples: swidget->tuples,
256	num_tuples: swidget->num_tuples, object_size: sizeof(*available_fmt), token_instance_num: 1);
257	if (ret) {
258	dev_err(scomp->dev, "Failed to parse audio format token count\n");
259	return ret;
260	}
261
262	if (!available_fmt->num_input_formats && !available_fmt->num_output_formats) {
263	dev_err(scomp->dev, "No input/output pin formats set in topology\n");
264	return -EINVAL;
265	}
266
267	dev_dbg(scomp->dev,
268	"Number of input audio formats: %d. Number of output audio formats: %d\n",
269	available_fmt->num_input_formats, available_fmt->num_output_formats);
270
271	/* set is_pages in the module's base_config */
272	ret = sof_update_ipc_object(scomp, object: module_base_cfg, token_id: SOF_COMP_TOKENS, tuples: swidget->tuples,
273	num_tuples: swidget->num_tuples, object_size: sizeof(*module_base_cfg), token_instance_num: 1);
274	if (ret) {
275	dev_err(scomp->dev, "parse comp tokens for %s failed, error: %d\n",
276	swidget->widget->name, ret);
277	return ret;
278	}
279
280	dev_dbg(scomp->dev, "widget %s: is_pages: %d\n", swidget->widget->name,
281	module_base_cfg->is_pages);
282
283	if (available_fmt->num_input_formats) {
284	in_format = kcalloc(n: available_fmt->num_input_formats,
285	size: sizeof(*in_format), GFP_KERNEL);
286	if (!in_format)
287	return -ENOMEM;
288	available_fmt->input_pin_fmts = in_format;
289
290	ret = sof_update_ipc_object(scomp, object: in_format,
291	token_id: SOF_IN_AUDIO_FORMAT_TOKENS, tuples: swidget->tuples,
292	num_tuples: swidget->num_tuples, object_size: sizeof(*in_format),
293	token_instance_num: available_fmt->num_input_formats);
294	if (ret) {
295	dev_err(scomp->dev, "parse input audio fmt tokens failed %d\n", ret);
296	goto err_in;
297	}
298
299	dev_dbg(scomp->dev, "Input audio formats for %s\n", swidget->widget->name);
300	sof_ipc4_dbg_audio_format(dev: scomp->dev, pin_fmt: in_format,
301	num_formats: available_fmt->num_input_formats);
302	}
303
304	if (available_fmt->num_output_formats) {
305	out_format = kcalloc(n: available_fmt->num_output_formats, size: sizeof(*out_format),
306	GFP_KERNEL);
307	if (!out_format) {
308	ret = -ENOMEM;
309	goto err_in;
310	}
311
312	ret = sof_update_ipc_object(scomp, object: out_format,
313	token_id: SOF_OUT_AUDIO_FORMAT_TOKENS, tuples: swidget->tuples,
314	num_tuples: swidget->num_tuples, object_size: sizeof(*out_format),
315	token_instance_num: available_fmt->num_output_formats);
316	if (ret) {
317	dev_err(scomp->dev, "parse output audio fmt tokens failed\n");
318	goto err_out;
319	}
320
321	available_fmt->output_pin_fmts = out_format;
322	dev_dbg(scomp->dev, "Output audio formats for %s\n", swidget->widget->name);
323	sof_ipc4_dbg_audio_format(dev: scomp->dev, pin_fmt: out_format,
324	num_formats: available_fmt->num_output_formats);
325	}
326
327	return 0;
328
329	err_out:
330	kfree(objp: out_format);
331	err_in:
332	kfree(objp: in_format);
333	available_fmt->input_pin_fmts = NULL;
334	return ret;
335	}
336
337	/* release the memory allocated in sof_ipc4_get_audio_fmt */
338	static void sof_ipc4_free_audio_fmt(struct sof_ipc4_available_audio_format *available_fmt)
339
340	{
341	kfree(objp: available_fmt->output_pin_fmts);
342	available_fmt->output_pin_fmts = NULL;
343	kfree(objp: available_fmt->input_pin_fmts);
344	available_fmt->input_pin_fmts = NULL;
345	}
346
347	static void sof_ipc4_widget_free_comp_pipeline(struct snd_sof_widget *swidget)
348	{
349	kfree(objp: swidget->private);
350	}
351
352	static int sof_ipc4_widget_set_module_info(struct snd_sof_widget *swidget)
353	{
354	struct snd_soc_component *scomp = swidget->scomp;
355	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
356
357	swidget->module_info = sof_ipc4_find_module_by_uuid(sdev, uuid: &swidget->uuid);
358
359	if (swidget->module_info)
360	return 0;
361
362	dev_err(sdev->dev, "failed to find module info for widget %s with UUID %pUL\n",
363	swidget->widget->name, &swidget->uuid);
364	return -EINVAL;
365	}
366
367	static int sof_ipc4_widget_setup_msg(struct snd_sof_widget *swidget, struct sof_ipc4_msg *msg)
368	{
369	struct sof_ipc4_fw_module *fw_module;
370	uint32_t type;
371	int ret;
372
373	ret = sof_ipc4_widget_set_module_info(swidget);
374	if (ret)
375	return ret;
376
377	fw_module = swidget->module_info;
378
379	msg->primary = fw_module->man4_module_entry.id;
380	msg->primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_INIT_INSTANCE);
381	msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
382	msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
383
384	msg->extension = SOF_IPC4_MOD_EXT_CORE_ID(swidget->core);
385
386	type = (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP) ? 1 : 0;
387	msg->extension |= SOF_IPC4_MOD_EXT_DOMAIN(type);
388
389	return 0;
390	}
391
392	static void sof_ipc4_widget_update_kcontrol_module_id(struct snd_sof_widget *swidget)
393	{
394	struct snd_soc_component *scomp = swidget->scomp;
395	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
396	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
397	struct snd_sof_control *scontrol;
398
399	/* update module ID for all kcontrols for this widget */
400	list_for_each_entry(scontrol, &sdev->kcontrol_list, list) {
401	if (scontrol->comp_id == swidget->comp_id) {
402	struct sof_ipc4_control_data *cdata = scontrol->ipc_control_data;
403	struct sof_ipc4_msg *msg = &cdata->msg;
404
405	msg->primary |= fw_module->man4_module_entry.id;
406	}
407	}
408	}
409
410	static int sof_ipc4_widget_setup_pcm(struct snd_sof_widget *swidget)
411	{
412	struct sof_ipc4_available_audio_format *available_fmt;
413	struct snd_soc_component *scomp = swidget->scomp;
414	struct sof_ipc4_copier *ipc4_copier;
415	struct snd_sof_pcm *spcm;
416	int node_type = 0;
417	int ret, dir;
418
419	ipc4_copier = kzalloc(size: sizeof(*ipc4_copier), GFP_KERNEL);
420	if (!ipc4_copier)
421	return -ENOMEM;
422
423	swidget->private = ipc4_copier;
424	available_fmt = &ipc4_copier->available_fmt;
425
426	dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
427
428	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt,
429	module_base_cfg: &ipc4_copier->data.base_config);
430	if (ret)
431	goto free_copier;
432
433	/*
434	* This callback is used by host copier and module-to-module copier,
435	* and only host copier needs to set gtw_cfg.
436	*/
437	if (!WIDGET_IS_AIF(swidget->id))
438	goto skip_gtw_cfg;
439
440	ret = sof_update_ipc_object(scomp, object: &node_type,
441	token_id: SOF_COPIER_TOKENS, tuples: swidget->tuples,
442	num_tuples: swidget->num_tuples, object_size: sizeof(node_type), token_instance_num: 1);
443
444	if (ret) {
445	dev_err(scomp->dev, "parse host copier node type token failed %d\n",
446	ret);
447	goto free_available_fmt;
448	}
449	dev_dbg(scomp->dev, "host copier '%s' node_type %u\n", swidget->widget->name, node_type);
450
451	spcm = snd_sof_find_spcm_comp(scomp, comp_id: swidget->comp_id, direction: &dir);
452	if (!spcm)
453	goto skip_gtw_cfg;
454
455	if (dir == SNDRV_PCM_STREAM_PLAYBACK) {
456	struct snd_sof_pcm_stream *sps = &spcm->stream[dir];
457
458	sof_update_ipc_object(scomp, object: &sps->dsp_max_burst_size_in_ms,
459	token_id: SOF_COPIER_DEEP_BUFFER_TOKENS,
460	tuples: swidget->tuples,
461	num_tuples: swidget->num_tuples, object_size: sizeof(u32), token_instance_num: 1);
462	/* Set default DMA buffer size if it is not specified in topology */
463	if (!sps->dsp_max_burst_size_in_ms)
464	sps->dsp_max_burst_size_in_ms = SOF_IPC4_MIN_DMA_BUFFER_SIZE;
465	} else {
466	/* Capture data is copied from DSP to host in 1ms bursts */
467	spcm->stream[dir].dsp_max_burst_size_in_ms = 1;
468	}
469
470	skip_gtw_cfg:
471	ipc4_copier->gtw_attr = kzalloc(size: sizeof(*ipc4_copier->gtw_attr), GFP_KERNEL);
472	if (!ipc4_copier->gtw_attr) {
473	ret = -ENOMEM;
474	goto free_available_fmt;
475	}
476
477	ipc4_copier->copier_config = (uint32_t *)ipc4_copier->gtw_attr;
478	ipc4_copier->data.gtw_cfg.config_length =
479	sizeof(struct sof_ipc4_gtw_attributes) >> 2;
480
481	switch (swidget->id) {
482	case snd_soc_dapm_aif_in:
483	case snd_soc_dapm_aif_out:
484	ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_NODE_TYPE(node_type);
485	break;
486	case snd_soc_dapm_buffer:
487	ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID;
488	ipc4_copier->ipc_config_size = 0;
489	break;
490	default:
491	dev_err(scomp->dev, "invalid widget type %d\n", swidget->id);
492	ret = -EINVAL;
493	goto free_gtw_attr;
494	}
495
496	/* set up module info and message header */
497	ret = sof_ipc4_widget_setup_msg(swidget, msg: &ipc4_copier->msg);
498	if (ret)
499	goto free_gtw_attr;
500
501	return 0;
502
503	free_gtw_attr:
504	kfree(objp: ipc4_copier->gtw_attr);
505	free_available_fmt:
506	sof_ipc4_free_audio_fmt(available_fmt);
507	free_copier:
508	kfree(objp: ipc4_copier);
509	swidget->private = NULL;
510	return ret;
511	}
512
513	static void sof_ipc4_widget_free_comp_pcm(struct snd_sof_widget *swidget)
514	{
515	struct sof_ipc4_copier *ipc4_copier = swidget->private;
516	struct sof_ipc4_available_audio_format *available_fmt;
517
518	if (!ipc4_copier)
519	return;
520
521	available_fmt = &ipc4_copier->available_fmt;
522	kfree(objp: available_fmt->output_pin_fmts);
523	kfree(objp: ipc4_copier->gtw_attr);
524	kfree(objp: ipc4_copier);
525	swidget->private = NULL;
526	}
527
528	static int sof_ipc4_widget_setup_comp_dai(struct snd_sof_widget *swidget)
529	{
530	struct sof_ipc4_available_audio_format *available_fmt;
531	struct snd_soc_component *scomp = swidget->scomp;
532	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
533	struct snd_sof_dai *dai = swidget->private;
534	struct sof_ipc4_copier *ipc4_copier;
535	struct snd_sof_widget *pipe_widget;
536	struct sof_ipc4_pipeline *pipeline;
537	int node_type = 0;
538	int ret;
539
540	ipc4_copier = kzalloc(size: sizeof(*ipc4_copier), GFP_KERNEL);
541	if (!ipc4_copier)
542	return -ENOMEM;
543
544	available_fmt = &ipc4_copier->available_fmt;
545
546	dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
547
548	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt,
549	module_base_cfg: &ipc4_copier->data.base_config);
550	if (ret)
551	goto free_copier;
552
553	ret = sof_update_ipc_object(scomp, object: &node_type,
554	token_id: SOF_COPIER_TOKENS, tuples: swidget->tuples,
555	num_tuples: swidget->num_tuples, object_size: sizeof(node_type), token_instance_num: 1);
556	if (ret) {
557	dev_err(scomp->dev, "parse dai node type failed %d\n", ret);
558	goto free_available_fmt;
559	}
560
561	ret = sof_update_ipc_object(scomp, object: ipc4_copier,
562	token_id: SOF_DAI_TOKENS, tuples: swidget->tuples,
563	num_tuples: swidget->num_tuples, object_size: sizeof(u32), token_instance_num: 1);
564	if (ret) {
565	dev_err(scomp->dev, "parse dai copier node token failed %d\n", ret);
566	goto free_available_fmt;
567	}
568
569	dev_dbg(scomp->dev, "dai %s node_type %u dai_type %u dai_index %d\n", swidget->widget->name,
570	node_type, ipc4_copier->dai_type, ipc4_copier->dai_index);
571
572	dai->type = ipc4_copier->dai_type;
573	ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_NODE_TYPE(node_type);
574
575	pipe_widget = swidget->spipe->pipe_widget;
576	pipeline = pipe_widget->private;
577
578	if (pipeline->use_chain_dma &&
579	!snd_sof_is_chain_dma_supported(sdev, dai_type: ipc4_copier->dai_type)) {
580	dev_err(scomp->dev, "Bad DAI type '%d', Chain DMA is not supported\n",
581	ipc4_copier->dai_type);
582	ret = -ENODEV;
583	goto free_available_fmt;
584	}
585
586	switch (ipc4_copier->dai_type) {
587	case SOF_DAI_INTEL_ALH:
588	{
589	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
590	struct sof_ipc4_alh_configuration_blob *blob;
591	struct snd_soc_dapm_path *p;
592	struct snd_sof_widget *w;
593	int src_num = 0;
594
595	snd_soc_dapm_widget_for_each_source_path(swidget->widget, p)
596	src_num++;
597
598	if (swidget->id == snd_soc_dapm_dai_in && src_num == 0) {
599	/*
600	* The blob will not be used if the ALH copier is playback direction
601	* and doesn't connect to any source.
602	* It is fine to call kfree(ipc4_copier->copier_config) since
603	* ipc4_copier->copier_config is null.
604	*/
605	ret = 0;
606	break;
607	}
608
609	blob = kzalloc(size: sizeof(*blob), GFP_KERNEL);
610	if (!blob) {
611	ret = -ENOMEM;
612	goto free_available_fmt;
613	}
614
615	list_for_each_entry(w, &sdev->widget_list, list) {
616	if (w->widget->sname &&
617	strcmp(w->widget->sname, swidget->widget->sname))
618	continue;
619
620	blob->alh_cfg.device_count++;
621	}
622
623	ipc4_copier->copier_config = (uint32_t *)blob;
624	/* set data.gtw_cfg.config_length based on device_count */
625	ipc4_copier->data.gtw_cfg.config_length = (sizeof(blob->gw_attr) +
626	sizeof(blob->alh_cfg.device_count) +
627	sizeof(*blob->alh_cfg.mapping) *
628	blob->alh_cfg.device_count) >> 2;
629	break;
630	}
631	case SOF_DAI_INTEL_SSP:
632	/* set SSP DAI index as the node_id */
633	ipc4_copier->data.gtw_cfg.node_id |=
634	SOF_IPC4_NODE_INDEX_INTEL_SSP(ipc4_copier->dai_index);
635	break;
636	case SOF_DAI_INTEL_DMIC:
637	/* set DMIC DAI index as the node_id */
638	ipc4_copier->data.gtw_cfg.node_id |=
639	SOF_IPC4_NODE_INDEX_INTEL_DMIC(ipc4_copier->dai_index);
640	break;
641	default:
642	ipc4_copier->gtw_attr = kzalloc(size: sizeof(*ipc4_copier->gtw_attr), GFP_KERNEL);
643	if (!ipc4_copier->gtw_attr) {
644	ret = -ENOMEM;
645	goto free_available_fmt;
646	}
647
648	ipc4_copier->copier_config = (uint32_t *)ipc4_copier->gtw_attr;
649	ipc4_copier->data.gtw_cfg.config_length =
650	sizeof(struct sof_ipc4_gtw_attributes) >> 2;
651	break;
652	}
653
654	dai->scomp = scomp;
655	dai->private = ipc4_copier;
656
657	/* set up module info and message header */
658	ret = sof_ipc4_widget_setup_msg(swidget, msg: &ipc4_copier->msg);
659	if (ret)
660	goto free_copier_config;
661
662	return 0;
663
664	free_copier_config:
665	kfree(objp: ipc4_copier->copier_config);
666	free_available_fmt:
667	sof_ipc4_free_audio_fmt(available_fmt);
668	free_copier:
669	kfree(objp: ipc4_copier);
670	dai->private = NULL;
671	dai->scomp = NULL;
672	return ret;
673	}
674
675	static void sof_ipc4_widget_free_comp_dai(struct snd_sof_widget *swidget)
676	{
677	struct sof_ipc4_available_audio_format *available_fmt;
678	struct snd_sof_dai *dai = swidget->private;
679	struct sof_ipc4_copier *ipc4_copier;
680
681	if (!dai)
682	return;
683
684	if (!dai->private) {
685	kfree(objp: dai);
686	swidget->private = NULL;
687	return;
688	}
689
690	ipc4_copier = dai->private;
691	available_fmt = &ipc4_copier->available_fmt;
692
693	kfree(objp: available_fmt->output_pin_fmts);
694	if (ipc4_copier->dai_type != SOF_DAI_INTEL_SSP &&
695	ipc4_copier->dai_type != SOF_DAI_INTEL_DMIC)
696	kfree(objp: ipc4_copier->copier_config);
697	kfree(objp: dai->private);
698	kfree(objp: dai);
699	swidget->private = NULL;
700	}
701
702	static int sof_ipc4_widget_setup_comp_pipeline(struct snd_sof_widget *swidget)
703	{
704	struct snd_soc_component *scomp = swidget->scomp;
705	struct sof_ipc4_pipeline *pipeline;
706	struct snd_sof_pipeline *spipe = swidget->spipe;
707	int ret;
708
709	pipeline = kzalloc(size: sizeof(*pipeline), GFP_KERNEL);
710	if (!pipeline)
711	return -ENOMEM;
712
713	ret = sof_update_ipc_object(scomp, object: pipeline, token_id: SOF_SCHED_TOKENS, tuples: swidget->tuples,
714	num_tuples: swidget->num_tuples, object_size: sizeof(*pipeline), token_instance_num: 1);
715	if (ret) {
716	dev_err(scomp->dev, "parsing scheduler tokens failed\n");
717	goto err;
718	}
719
720	swidget->core = pipeline->core_id;
721	spipe->core_mask |= BIT(pipeline->core_id);
722
723	if (pipeline->use_chain_dma) {
724	dev_dbg(scomp->dev, "Set up chain DMA for %s\n", swidget->widget->name);
725	swidget->private = pipeline;
726	return 0;
727	}
728
729	/* parse one set of pipeline tokens */
730	ret = sof_update_ipc_object(scomp, object: swidget, token_id: SOF_PIPELINE_TOKENS, tuples: swidget->tuples,
731	num_tuples: swidget->num_tuples, object_size: sizeof(*swidget), token_instance_num: 1);
732	if (ret) {
733	dev_err(scomp->dev, "parsing pipeline tokens failed\n");
734	goto err;
735	}
736
737	dev_dbg(scomp->dev, "pipeline '%s': id %d, pri %d, core_id %u, lp mode %d\n",
738	swidget->widget->name, swidget->pipeline_id,
739	pipeline->priority, pipeline->core_id, pipeline->lp_mode);
740
741	swidget->private = pipeline;
742
743	pipeline->msg.primary = SOF_IPC4_GLB_PIPE_PRIORITY(pipeline->priority);
744	pipeline->msg.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_CREATE_PIPELINE);
745	pipeline->msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
746	pipeline->msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
747
748	pipeline->msg.extension = pipeline->lp_mode;
749	pipeline->msg.extension |= SOF_IPC4_GLB_PIPE_EXT_CORE_ID(pipeline->core_id);
750	pipeline->state = SOF_IPC4_PIPE_UNINITIALIZED;
751
752	return 0;
753	err:
754	kfree(objp: pipeline);
755	return ret;
756	}
757
758	static int sof_ipc4_widget_setup_comp_pga(struct snd_sof_widget *swidget)
759	{
760	struct snd_soc_component *scomp = swidget->scomp;
761	struct sof_ipc4_gain *gain;
762	int ret;
763
764	gain = kzalloc(size: sizeof(*gain), GFP_KERNEL);
765	if (!gain)
766	return -ENOMEM;
767
768	swidget->private = gain;
769
770	gain->data.params.channels = SOF_IPC4_GAIN_ALL_CHANNELS_MASK;
771	gain->data.params.init_val = SOF_IPC4_VOL_ZERO_DB;
772
773	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt: &gain->available_fmt, module_base_cfg: &gain->data.base_config);
774	if (ret)
775	goto err;
776
777	ret = sof_update_ipc_object(scomp, object: &gain->data.params, token_id: SOF_GAIN_TOKENS,
778	tuples: swidget->tuples, num_tuples: swidget->num_tuples, object_size: sizeof(gain->data), token_instance_num: 1);
779	if (ret) {
780	dev_err(scomp->dev, "Parsing gain tokens failed\n");
781	goto err;
782	}
783
784	dev_dbg(scomp->dev,
785	"pga widget %s: ramp type: %d, ramp duration %d, initial gain value: %#x\n",
786	swidget->widget->name, gain->data.params.curve_type,
787	gain->data.params.curve_duration_l, gain->data.params.init_val);
788
789	ret = sof_ipc4_widget_setup_msg(swidget, msg: &gain->msg);
790	if (ret)
791	goto err;
792
793	sof_ipc4_widget_update_kcontrol_module_id(swidget);
794
795	return 0;
796	err:
797	sof_ipc4_free_audio_fmt(available_fmt: &gain->available_fmt);
798	kfree(objp: gain);
799	swidget->private = NULL;
800	return ret;
801	}
802
803	static void sof_ipc4_widget_free_comp_pga(struct snd_sof_widget *swidget)
804	{
805	struct sof_ipc4_gain *gain = swidget->private;
806
807	if (!gain)
808	return;
809
810	sof_ipc4_free_audio_fmt(available_fmt: &gain->available_fmt);
811	kfree(objp: swidget->private);
812	swidget->private = NULL;
813	}
814
815	static int sof_ipc4_widget_setup_comp_mixer(struct snd_sof_widget *swidget)
816	{
817	struct snd_soc_component *scomp = swidget->scomp;
818	struct sof_ipc4_mixer *mixer;
819	int ret;
820
821	dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
822
823	mixer = kzalloc(size: sizeof(*mixer), GFP_KERNEL);
824	if (!mixer)
825	return -ENOMEM;
826
827	swidget->private = mixer;
828
829	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt: &mixer->available_fmt,
830	module_base_cfg: &mixer->base_config);
831	if (ret)
832	goto err;
833
834	ret = sof_ipc4_widget_setup_msg(swidget, msg: &mixer->msg);
835	if (ret)
836	goto err;
837
838	return 0;
839	err:
840	sof_ipc4_free_audio_fmt(available_fmt: &mixer->available_fmt);
841	kfree(objp: mixer);
842	swidget->private = NULL;
843	return ret;
844	}
845
846	static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget)
847	{
848	struct snd_soc_component *scomp = swidget->scomp;
849	struct snd_sof_pipeline *spipe = swidget->spipe;
850	struct sof_ipc4_src *src;
851	int ret;
852
853	dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
854
855	src = kzalloc(size: sizeof(*src), GFP_KERNEL);
856	if (!src)
857	return -ENOMEM;
858
859	swidget->private = src;
860
861	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt: &src->available_fmt,
862	module_base_cfg: &src->data.base_config);
863	if (ret)
864	goto err;
865
866	ret = sof_update_ipc_object(scomp, object: &src->data, token_id: SOF_SRC_TOKENS, tuples: swidget->tuples,
867	num_tuples: swidget->num_tuples, object_size: sizeof(*src), token_instance_num: 1);
868	if (ret) {
869	dev_err(scomp->dev, "Parsing SRC tokens failed\n");
870	goto err;
871	}
872
873	spipe->core_mask |= BIT(swidget->core);
874
875	dev_dbg(scomp->dev, "SRC sink rate %d\n", src->data.sink_rate);
876
877	ret = sof_ipc4_widget_setup_msg(swidget, msg: &src->msg);
878	if (ret)
879	goto err;
880
881	return 0;
882	err:
883	sof_ipc4_free_audio_fmt(available_fmt: &src->available_fmt);
884	kfree(objp: src);
885	swidget->private = NULL;
886	return ret;
887	}
888
889	static void sof_ipc4_widget_free_comp_src(struct snd_sof_widget *swidget)
890	{
891	struct sof_ipc4_src *src = swidget->private;
892
893	if (!src)
894	return;
895
896	sof_ipc4_free_audio_fmt(available_fmt: &src->available_fmt);
897	kfree(objp: swidget->private);
898	swidget->private = NULL;
899	}
900
901	static void sof_ipc4_widget_free_comp_mixer(struct snd_sof_widget *swidget)
902	{
903	struct sof_ipc4_mixer *mixer = swidget->private;
904
905	if (!mixer)
906	return;
907
908	sof_ipc4_free_audio_fmt(available_fmt: &mixer->available_fmt);
909	kfree(objp: swidget->private);
910	swidget->private = NULL;
911	}
912
913	/*
914	* Add the process modules support. The process modules are defined as snd_soc_dapm_effect modules.
915	*/
916	static int sof_ipc4_widget_setup_comp_process(struct snd_sof_widget *swidget)
917	{
918	struct snd_soc_component *scomp = swidget->scomp;
919	struct sof_ipc4_fw_module *fw_module;
920	struct snd_sof_pipeline *spipe = swidget->spipe;
921	struct sof_ipc4_process *process;
922	void *cfg;
923	int ret;
924
925	process = kzalloc(size: sizeof(*process), GFP_KERNEL);
926	if (!process)
927	return -ENOMEM;
928
929	swidget->private = process;
930
931	ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt: &process->available_fmt,
932	module_base_cfg: &process->base_config);
933	if (ret)
934	goto err;
935
936	ret = sof_ipc4_widget_setup_msg(swidget, msg: &process->msg);
937	if (ret)
938	goto err;
939
940	/* parse process init module payload config type from module info */
941	fw_module = swidget->module_info;
942	process->init_config = FIELD_GET(SOF_IPC4_MODULE_INIT_CONFIG_MASK,
943	fw_module->man4_module_entry.type);
944
945	process->ipc_config_size = sizeof(struct sof_ipc4_base_module_cfg);
946
947	/* allocate memory for base config extension if needed */
948	if (process->init_config == SOF_IPC4_MODULE_INIT_CONFIG_TYPE_BASE_CFG_WITH_EXT) {
949	struct sof_ipc4_base_module_cfg_ext *base_cfg_ext;
950	u32 ext_size = struct_size(base_cfg_ext, pin_formats,
951	size_add(swidget->num_input_pins,
952	swidget->num_output_pins));
953
954	base_cfg_ext = kzalloc(size: ext_size, GFP_KERNEL);
955	if (!base_cfg_ext) {
956	ret = -ENOMEM;
957	goto free_available_fmt;
958	}
959
960	base_cfg_ext->num_input_pin_fmts = swidget->num_input_pins;
961	base_cfg_ext->num_output_pin_fmts = swidget->num_output_pins;
962	process->base_config_ext = base_cfg_ext;
963	process->base_config_ext_size = ext_size;
964	process->ipc_config_size += ext_size;
965	}
966
967	cfg = kzalloc(size: process->ipc_config_size, GFP_KERNEL);
968	if (!cfg) {
969	ret = -ENOMEM;
970	goto free_base_cfg_ext;
971	}
972
973	process->ipc_config_data = cfg;
974
975	sof_ipc4_widget_update_kcontrol_module_id(swidget);
976
977	/* set pipeline core mask to keep track of the core the module is scheduled to run on */
978	spipe->core_mask |= BIT(swidget->core);
979
980	return 0;
981	free_base_cfg_ext:
982	kfree(objp: process->base_config_ext);
983	process->base_config_ext = NULL;
984	free_available_fmt:
985	sof_ipc4_free_audio_fmt(available_fmt: &process->available_fmt);
986	err:
987	kfree(objp: process);
988	swidget->private = NULL;
989	return ret;
990	}
991
992	static void sof_ipc4_widget_free_comp_process(struct snd_sof_widget *swidget)
993	{
994	struct sof_ipc4_process *process = swidget->private;
995
996	if (!process)
997	return;
998
999	kfree(objp: process->ipc_config_data);
1000	kfree(objp: process->base_config_ext);
1001	sof_ipc4_free_audio_fmt(available_fmt: &process->available_fmt);
1002	kfree(objp: swidget->private);
1003	swidget->private = NULL;
1004	}
1005
1006	static void
1007	sof_ipc4_update_resource_usage(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget,
1008	struct sof_ipc4_base_module_cfg *base_config)
1009	{
1010	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
1011	struct snd_sof_widget *pipe_widget;
1012	struct sof_ipc4_pipeline *pipeline;
1013	int task_mem, queue_mem;
1014	int ibs, bss, total;
1015
1016	ibs = base_config->ibs;
1017	bss = base_config->is_pages;
1018
1019	task_mem = SOF_IPC4_PIPELINE_OBJECT_SIZE;
1020	task_mem += SOF_IPC4_MODULE_INSTANCE_LIST_ITEM_SIZE + bss;
1021
1022	if (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_LL) {
1023	task_mem += SOF_IPC4_FW_ROUNDUP(SOF_IPC4_LL_TASK_OBJECT_SIZE);
1024	task_mem += SOF_IPC4_FW_MAX_QUEUE_COUNT * SOF_IPC4_MODULE_INSTANCE_LIST_ITEM_SIZE;
1025	task_mem += SOF_IPC4_LL_TASK_LIST_ITEM_SIZE;
1026	} else {
1027	task_mem += SOF_IPC4_FW_ROUNDUP(SOF_IPC4_DP_TASK_OBJECT_SIZE);
1028	task_mem += SOF_IPC4_DP_TASK_LIST_SIZE;
1029	}
1030
1031	ibs = SOF_IPC4_FW_ROUNDUP(ibs);
1032	queue_mem = SOF_IPC4_FW_MAX_QUEUE_COUNT * (SOF_IPC4_DATA_QUEUE_OBJECT_SIZE + ibs);
1033
1034	total = SOF_IPC4_FW_PAGE(task_mem + queue_mem);
1035
1036	pipe_widget = swidget->spipe->pipe_widget;
1037	pipeline = pipe_widget->private;
1038	pipeline->mem_usage += total;
1039
1040	/* Update base_config->cpc from the module manifest */
1041	sof_ipc4_update_cpc_from_manifest(sdev, fw_module, basecfg: base_config);
1042
1043	if (ignore_cpc) {
1044	dev_dbg(sdev->dev, "%s: ibs / obs: %u / %u, forcing cpc to 0 from %u\n",
1045	swidget->widget->name, base_config->ibs, base_config->obs,
1046	base_config->cpc);
1047	base_config->cpc = 0;
1048	} else {
1049	dev_dbg(sdev->dev, "%s: ibs / obs / cpc: %u / %u / %u\n",
1050	swidget->widget->name, base_config->ibs, base_config->obs,
1051	base_config->cpc);
1052	}
1053	}
1054
1055	static int sof_ipc4_widget_assign_instance_id(struct snd_sof_dev *sdev,
1056	struct snd_sof_widget *swidget)
1057	{
1058	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
1059	int max_instances = fw_module->man4_module_entry.instance_max_count;
1060
1061	swidget->instance_id = ida_alloc_max(ida: &fw_module->m_ida, max: max_instances, GFP_KERNEL);
1062	if (swidget->instance_id < 0) {
1063	dev_err(sdev->dev, "failed to assign instance id for widget %s",
1064	swidget->widget->name);
1065	return swidget->instance_id;
1066	}
1067
1068	return 0;
1069	}
1070
1071	/* update hw_params based on the audio stream format */
1072	static int sof_ipc4_update_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_hw_params *params,
1073	struct sof_ipc4_audio_format *fmt)
1074	{
1075	snd_pcm_format_t snd_fmt;
1076	struct snd_interval *i;
1077	struct snd_mask *m;
1078	int valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1079	unsigned int channels, rate;
1080
1081	switch (valid_bits) {
1082	case 16:
1083	snd_fmt = SNDRV_PCM_FORMAT_S16_LE;
1084	break;
1085	case 24:
1086	snd_fmt = SNDRV_PCM_FORMAT_S24_LE;
1087	break;
1088	case 32:
1089	snd_fmt = SNDRV_PCM_FORMAT_S32_LE;
1090	break;
1091	default:
1092	dev_err(sdev->dev, "invalid PCM valid_bits %d\n", valid_bits);
1093	return -EINVAL;
1094	}
1095
1096	m = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
1097	snd_mask_none(mask: m);
1098	snd_mask_set_format(mask: m, format: snd_fmt);
1099
1100	rate = fmt->sampling_frequency;
1101	i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
1102	i->min = rate;
1103	i->max = rate;
1104
1105	channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1106	i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS);
1107	i->min = channels;
1108	i->max = channels;
1109
1110	return 0;
1111	}
1112
1113	static bool sof_ipc4_is_single_format(struct snd_sof_dev *sdev,
1114	struct sof_ipc4_pin_format *pin_fmts, u32 pin_fmts_size)
1115	{
1116	struct sof_ipc4_audio_format *fmt;
1117	u32 rate, channels, valid_bits;
1118	int i;
1119
1120	fmt = &pin_fmts[0].audio_fmt;
1121	rate = fmt->sampling_frequency;
1122	channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1123	valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1124
1125	/* check if all output formats in topology are the same */
1126	for (i = 1; i < pin_fmts_size; i++) {
1127	u32 _rate, _channels, _valid_bits;
1128
1129	fmt = &pin_fmts[i].audio_fmt;
1130	_rate = fmt->sampling_frequency;
1131	_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1132	_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1133
1134	if (_rate != rate || _channels != channels || _valid_bits != valid_bits)
1135	return false;
1136	}
1137
1138	return true;
1139	}
1140
1141	static int sof_ipc4_init_output_audio_fmt(struct snd_sof_dev *sdev,
1142	struct sof_ipc4_base_module_cfg *base_config,
1143	struct sof_ipc4_available_audio_format *available_fmt,
1144	u32 out_ref_rate, u32 out_ref_channels,
1145	u32 out_ref_valid_bits)
1146	{
1147	struct sof_ipc4_audio_format *out_fmt;
1148	bool single_format;
1149	int i;
1150
1151	if (!available_fmt->num_output_formats)
1152	return -EINVAL;
1153
1154	single_format = sof_ipc4_is_single_format(sdev, pin_fmts: available_fmt->output_pin_fmts,
1155	pin_fmts_size: available_fmt->num_output_formats);
1156
1157	/* pick the first format if there's only one available or if all formats are the same */
1158	if (single_format) {
1159	base_config->obs = available_fmt->output_pin_fmts[0].buffer_size;
1160	return 0;
1161	}
1162
1163	/*
1164	* if there are multiple output formats, then choose the output format that matches
1165	* the reference params
1166	*/
1167	for (i = 0; i < available_fmt->num_output_formats; i++) {
1168	u32 _out_rate, _out_channels, _out_valid_bits;
1169
1170	out_fmt = &available_fmt->output_pin_fmts[i].audio_fmt;
1171	_out_rate = out_fmt->sampling_frequency;
1172	_out_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(out_fmt->fmt_cfg);
1173	_out_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(out_fmt->fmt_cfg);
1174
1175	if (_out_rate == out_ref_rate && _out_channels == out_ref_channels &&
1176	_out_valid_bits == out_ref_valid_bits) {
1177	base_config->obs = available_fmt->output_pin_fmts[i].buffer_size;
1178	return i;
1179	}
1180	}
1181
1182	return -EINVAL;
1183	}
1184
1185	static int sof_ipc4_get_valid_bits(struct snd_sof_dev *sdev, struct snd_pcm_hw_params *params)
1186	{
1187	switch (params_format(p: params)) {
1188	case SNDRV_PCM_FORMAT_S16_LE:
1189	return 16;
1190	case SNDRV_PCM_FORMAT_S24_LE:
1191	return 24;
1192	case SNDRV_PCM_FORMAT_S32_LE:
1193	return 32;
1194	default:
1195	dev_err(sdev->dev, "invalid pcm frame format %d\n", params_format(params));
1196	return -EINVAL;
1197	}
1198	}
1199
1200	static int sof_ipc4_init_input_audio_fmt(struct snd_sof_dev *sdev,
1201	struct snd_sof_widget *swidget,
1202	struct sof_ipc4_base_module_cfg *base_config,
1203	struct snd_pcm_hw_params *params,
1204	struct sof_ipc4_available_audio_format *available_fmt)
1205	{
1206	struct sof_ipc4_pin_format *pin_fmts = available_fmt->input_pin_fmts;
1207	u32 pin_fmts_size = available_fmt->num_input_formats;
1208	u32 valid_bits;
1209	u32 channels;
1210	u32 rate;
1211	bool single_format;
1212	int sample_valid_bits;
1213	int i = 0;
1214
1215	if (!available_fmt->num_input_formats) {
1216	dev_err(sdev->dev, "no input formats for %s\n", swidget->widget->name);
1217	return -EINVAL;
1218	}
1219
1220	single_format = sof_ipc4_is_single_format(sdev, pin_fmts: available_fmt->input_pin_fmts,
1221	pin_fmts_size: available_fmt->num_input_formats);
1222	if (single_format)
1223	goto in_fmt;
1224
1225	sample_valid_bits = sof_ipc4_get_valid_bits(sdev, params);
1226	if (sample_valid_bits < 0)
1227	return sample_valid_bits;
1228
1229	/*
1230	* Search supported input audio formats with pin index 0 to match rate, channels and
1231	* sample_valid_bits from reference params
1232	*/
1233	for (i = 0; i < pin_fmts_size; i++) {
1234	struct sof_ipc4_audio_format *fmt = &pin_fmts[i].audio_fmt;
1235
1236	if (pin_fmts[i].pin_index)
1237	continue;
1238
1239	rate = fmt->sampling_frequency;
1240	channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1241	valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1242	if (params_rate(p: params) == rate && params_channels(p: params) == channels &&
1243	sample_valid_bits == valid_bits) {
1244	dev_dbg(sdev->dev, "matched audio format index for %uHz, %ubit, %u channels: %d\n",
1245	rate, valid_bits, channels, i);
1246	break;
1247	}
1248	}
1249
1250	if (i == pin_fmts_size) {
1251	dev_err(sdev->dev, "%s: Unsupported audio format: %uHz, %ubit, %u channels\n",
1252	__func__, params_rate(params), sample_valid_bits, params_channels(params));
1253	return -EINVAL;
1254	}
1255
1256	in_fmt:
1257	/* copy input format */
1258	if (available_fmt->num_input_formats && i < available_fmt->num_input_formats) {
1259	memcpy(&base_config->audio_fmt, &available_fmt->input_pin_fmts[i].audio_fmt,
1260	sizeof(struct sof_ipc4_audio_format));
1261
1262	/* set base_cfg ibs/obs */
1263	base_config->ibs = available_fmt->input_pin_fmts[i].buffer_size;
1264
1265	dev_dbg(sdev->dev, "Init input audio formats for %s\n", swidget->widget->name);
1266	sof_ipc4_dbg_audio_format(dev: sdev->dev, pin_fmt: &available_fmt->input_pin_fmts[i], num_formats: 1);
1267	}
1268
1269	return i;
1270	}
1271
1272	static void sof_ipc4_unprepare_copier_module(struct snd_sof_widget *swidget)
1273	{
1274	struct sof_ipc4_copier *ipc4_copier = NULL;
1275	struct snd_sof_widget *pipe_widget;
1276	struct sof_ipc4_pipeline *pipeline;
1277
1278	/* reset pipeline memory usage */
1279	pipe_widget = swidget->spipe->pipe_widget;
1280	pipeline = pipe_widget->private;
1281	pipeline->mem_usage = 0;
1282
1283	if (WIDGET_IS_AIF(swidget->id) || swidget->id == snd_soc_dapm_buffer) {
1284	if (pipeline->use_chain_dma) {
1285	pipeline->msg.primary = 0;
1286	pipeline->msg.extension = 0;
1287	}
1288	ipc4_copier = swidget->private;
1289	} else if (WIDGET_IS_DAI(swidget->id)) {
1290	struct snd_sof_dai *dai = swidget->private;
1291
1292	ipc4_copier = dai->private;
1293
1294	if (pipeline->use_chain_dma) {
1295	pipeline->msg.primary = 0;
1296	pipeline->msg.extension = 0;
1297	}
1298
1299	if (ipc4_copier->dai_type == SOF_DAI_INTEL_ALH) {
1300	struct sof_ipc4_copier_data *copier_data = &ipc4_copier->data;
1301	struct sof_ipc4_alh_configuration_blob *blob;
1302	unsigned int group_id;
1303
1304	blob = (struct sof_ipc4_alh_configuration_blob *)ipc4_copier->copier_config;
1305	if (blob->alh_cfg.device_count > 1) {
1306	group_id = SOF_IPC4_NODE_INDEX(ipc4_copier->data.gtw_cfg.node_id) -
1307	ALH_MULTI_GTW_BASE;
1308	ida_free(&alh_group_ida, id: group_id);
1309	}
1310
1311	/* clear the node ID */
1312	copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1313	}
1314	}
1315
1316	if (ipc4_copier) {
1317	kfree(objp: ipc4_copier->ipc_config_data);
1318	ipc4_copier->ipc_config_data = NULL;
1319	ipc4_copier->ipc_config_size = 0;
1320	}
1321	}
1322
1323	#if IS_ENABLED(CONFIG_ACPI) && IS_ENABLED(CONFIG_SND_INTEL_NHLT)
1324	static int snd_sof_get_hw_config_params(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1325	int *sample_rate, int *channel_count, int *bit_depth)
1326	{
1327	struct snd_soc_tplg_hw_config *hw_config;
1328	struct snd_sof_dai_link *slink;
1329	bool dai_link_found = false;
1330	bool hw_cfg_found = false;
1331	int i;
1332
1333	/* get current hw_config from link */
1334	list_for_each_entry(slink, &sdev->dai_link_list, list) {
1335	if (!strcmp(slink->link->name, dai->name)) {
1336	dai_link_found = true;
1337	break;
1338	}
1339	}
1340
1341	if (!dai_link_found) {
1342	dev_err(sdev->dev, "%s: no DAI link found for DAI %s\n", __func__, dai->name);
1343	return -EINVAL;
1344	}
1345
1346	for (i = 0; i < slink->num_hw_configs; i++) {
1347	hw_config = &slink->hw_configs[i];
1348	if (dai->current_config == le32_to_cpu(hw_config->id)) {
1349	hw_cfg_found = true;
1350	break;
1351	}
1352	}
1353
1354	if (!hw_cfg_found) {
1355	dev_err(sdev->dev, "%s: no matching hw_config found for DAI %s\n", __func__,
1356	dai->name);
1357	return -EINVAL;
1358	}
1359
1360	*bit_depth = le32_to_cpu(hw_config->tdm_slot_width);
1361	*channel_count = le32_to_cpu(hw_config->tdm_slots);
1362	*sample_rate = le32_to_cpu(hw_config->fsync_rate);
1363
1364	dev_dbg(sdev->dev, "sample rate: %d sample width: %d channels: %d\n",
1365	*sample_rate, *bit_depth, *channel_count);
1366
1367	return 0;
1368	}
1369
1370	static int snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1371	struct snd_pcm_hw_params *params, u32 dai_index,
1372	u32 linktype, u8 dir, u32 **dst, u32 *len)
1373	{
1374	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
1375	struct nhlt_specific_cfg *cfg;
1376	int sample_rate, channel_count;
1377	int bit_depth, ret;
1378	u32 nhlt_type;
1379	int dev_type = 0;
1380
1381	/* convert to NHLT type */
1382	switch (linktype) {
1383	case SOF_DAI_INTEL_DMIC:
1384	nhlt_type = NHLT_LINK_DMIC;
1385	bit_depth = params_width(p: params);
1386	channel_count = params_channels(p: params);
1387	sample_rate = params_rate(p: params);
1388	break;
1389	case SOF_DAI_INTEL_SSP:
1390	nhlt_type = NHLT_LINK_SSP;
1391	ret = snd_sof_get_hw_config_params(sdev, dai, sample_rate: &sample_rate, channel_count: &channel_count,
1392	bit_depth: &bit_depth);
1393	if (ret < 0)
1394	return ret;
1395
1396	/*
1397	* We need to know the type of the external device attached to a SSP
1398	* port to retrieve the blob from NHLT. However, device type is not
1399	* specified in topology.
1400	* Query the type for the port and then pass that information back
1401	* to the blob lookup function.
1402	*/
1403	dev_type = intel_nhlt_ssp_device_type(dev: sdev->dev, nhlt: ipc4_data->nhlt,
1404	virtual_bus_id: dai_index);
1405	if (dev_type < 0)
1406	return dev_type;
1407	break;
1408	default:
1409	return 0;
1410	}
1411
1412	dev_dbg(sdev->dev, "dai index %d nhlt type %d direction %d dev type %d\n",
1413	dai_index, nhlt_type, dir, dev_type);
1414
1415	/* find NHLT blob with matching params */
1416	cfg = intel_nhlt_get_endpoint_blob(dev: sdev->dev, nhlt: ipc4_data->nhlt, bus_id: dai_index, link_type: nhlt_type,
1417	vbps: bit_depth, bps: bit_depth, num_ch: channel_count, rate: sample_rate,
1418	dir, dev_type);
1419
1420	if (!cfg) {
1421	dev_err(sdev->dev,
1422	"no matching blob for sample rate: %d sample width: %d channels: %d\n",
1423	sample_rate, bit_depth, channel_count);
1424	return -EINVAL;
1425	}
1426
1427	/* config length should be in dwords */
1428	*len = cfg->size >> 2;
1429	*dst = (u32 *)cfg->caps;
1430
1431	return 0;
1432	}
1433	#else
1434	static int snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1435	struct snd_pcm_hw_params *params, u32 dai_index,
1436	u32 linktype, u8 dir, u32 **dst, u32 *len)
1437	{
1438	return 0;
1439	}
1440	#endif
1441
1442	bool sof_ipc4_copier_is_single_format(struct snd_sof_dev *sdev,
1443	struct sof_ipc4_pin_format *pin_fmts,
1444	u32 pin_fmts_size)
1445	{
1446	struct sof_ipc4_audio_format *fmt;
1447	u32 valid_bits;
1448	int i;
1449
1450	fmt = &pin_fmts[0].audio_fmt;
1451	valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1452
1453	/* check if all formats in topology are the same */
1454	for (i = 1; i < pin_fmts_size; i++) {
1455	u32 _valid_bits;
1456
1457	fmt = &pin_fmts[i].audio_fmt;
1458	_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1459
1460	if (_valid_bits != valid_bits)
1461	return false;
1462	}
1463
1464	return true;
1465	}
1466
1467	static int
1468	sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget,
1469	struct snd_pcm_hw_params *fe_params,
1470	struct snd_sof_platform_stream_params *platform_params,
1471	struct snd_pcm_hw_params *pipeline_params, int dir)
1472	{
1473	struct sof_ipc4_available_audio_format *available_fmt;
1474	struct snd_soc_component *scomp = swidget->scomp;
1475	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
1476	struct sof_ipc4_copier_data *copier_data;
1477	struct snd_pcm_hw_params *ref_params;
1478	struct sof_ipc4_copier *ipc4_copier;
1479	struct snd_sof_dai *dai;
1480	u32 gtw_cfg_config_length;
1481	u32 dma_config_tlv_size = 0;
1482	void **ipc_config_data;
1483	int *ipc_config_size;
1484	u32 **data;
1485	int ipc_size, ret, out_ref_valid_bits;
1486	u32 out_ref_rate, out_ref_channels;
1487	u32 deep_buffer_dma_ms = 0;
1488	int output_fmt_index;
1489	bool single_output_format;
1490
1491	dev_dbg(sdev->dev, "copier %s, type %d", swidget->widget->name, swidget->id);
1492
1493	switch (swidget->id) {
1494	case snd_soc_dapm_aif_in:
1495	case snd_soc_dapm_aif_out:
1496	{
1497	struct sof_ipc4_gtw_attributes *gtw_attr;
1498	struct snd_sof_widget *pipe_widget;
1499	struct sof_ipc4_pipeline *pipeline;
1500
1501	/* parse the deep buffer dma size */
1502	ret = sof_update_ipc_object(scomp, object: &deep_buffer_dma_ms,
1503	token_id: SOF_COPIER_DEEP_BUFFER_TOKENS, tuples: swidget->tuples,
1504	num_tuples: swidget->num_tuples, object_size: sizeof(u32), token_instance_num: 1);
1505	if (ret) {
1506	dev_err(scomp->dev, "Failed to parse deep buffer dma size for %s\n",
1507	swidget->widget->name);
1508	return ret;
1509	}
1510
1511	ipc4_copier = (struct sof_ipc4_copier *)swidget->private;
1512	gtw_attr = ipc4_copier->gtw_attr;
1513	copier_data = &ipc4_copier->data;
1514	available_fmt = &ipc4_copier->available_fmt;
1515
1516	pipe_widget = swidget->spipe->pipe_widget;
1517	pipeline = pipe_widget->private;
1518
1519	if (pipeline->use_chain_dma) {
1520	u32 host_dma_id;
1521	u32 fifo_size;
1522
1523	host_dma_id = platform_params->stream_tag - 1;
1524	pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_HOST_ID(host_dma_id);
1525
1526	/* Set SCS bit for S16_LE format only */
1527	if (params_format(p: fe_params) == SNDRV_PCM_FORMAT_S16_LE)
1528	pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_SCS_MASK;
1529
1530	/*
1531	* Despite its name the bitfield 'fifo_size' is used to define DMA buffer
1532	* size. The expression calculates 2ms buffer size.
1533	*/
1534	fifo_size = DIV_ROUND_UP((SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS *
1535	params_rate(fe_params) *
1536	params_channels(fe_params) *
1537	params_physical_width(fe_params)), 8000);
1538	pipeline->msg.extension |= SOF_IPC4_GLB_EXT_CHAIN_DMA_FIFO_SIZE(fifo_size);
1539
1540	/*
1541	* Chain DMA does not support stream timestamping, set node_id to invalid
1542	* to skip the code in sof_ipc4_get_stream_start_offset().
1543	*/
1544	copier_data->gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID;
1545
1546	return 0;
1547	}
1548
1549	/*
1550	* Use the input_pin_fmts to match pcm params for playback and the output_pin_fmts
1551	* for capture.
1552	*/
1553	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
1554	ref_params = fe_params;
1555	else
1556	ref_params = pipeline_params;
1557
1558	copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1559	copier_data->gtw_cfg.node_id |=
1560	SOF_IPC4_NODE_INDEX(platform_params->stream_tag - 1);
1561
1562	/* set gateway attributes */
1563	gtw_attr->lp_buffer_alloc = pipeline->lp_mode;
1564	break;
1565	}
1566	case snd_soc_dapm_dai_in:
1567	case snd_soc_dapm_dai_out:
1568	{
1569	struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
1570	struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
1571
1572	if (pipeline->use_chain_dma)
1573	return 0;
1574
1575	dai = swidget->private;
1576
1577	ipc4_copier = (struct sof_ipc4_copier *)dai->private;
1578	copier_data = &ipc4_copier->data;
1579	available_fmt = &ipc4_copier->available_fmt;
1580
1581	/*
1582	* When there is format conversion within a pipeline, the number of supported
1583	* output formats is typically limited to just 1 for the DAI copiers. But when there
1584	* is no format conversion, the DAI copiers input format must match that of the
1585	* FE hw_params for capture and the pipeline params for playback.
1586	*/
1587	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
1588	ref_params = pipeline_params;
1589	else
1590	ref_params = fe_params;
1591
1592	ret = snd_sof_get_nhlt_endpoint_data(sdev, dai, params: fe_params, dai_index: ipc4_copier->dai_index,
1593	linktype: ipc4_copier->dai_type, dir,
1594	dst: &ipc4_copier->copier_config,
1595	len: &copier_data->gtw_cfg.config_length);
1596	if (ret < 0)
1597	return ret;
1598
1599	break;
1600	}
1601	case snd_soc_dapm_buffer:
1602	{
1603	ipc4_copier = (struct sof_ipc4_copier *)swidget->private;
1604	copier_data = &ipc4_copier->data;
1605	available_fmt = &ipc4_copier->available_fmt;
1606	ref_params = pipeline_params;
1607
1608	break;
1609	}
1610	default:
1611	dev_err(sdev->dev, "unsupported type %d for copier %s",
1612	swidget->id, swidget->widget->name);
1613	return -EINVAL;
1614	}
1615
1616	/* set input and output audio formats */
1617	ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, base_config: &copier_data->base_config, params: ref_params,
1618	available_fmt);
1619	if (ret < 0)
1620	return ret;
1621
1622	/* set the reference params for output format selection */
1623	single_output_format = sof_ipc4_copier_is_single_format(sdev,
1624	pin_fmts: available_fmt->output_pin_fmts,
1625	pin_fmts_size: available_fmt->num_output_formats);
1626	switch (swidget->id) {
1627	case snd_soc_dapm_aif_in:
1628	case snd_soc_dapm_dai_out:
1629	case snd_soc_dapm_buffer:
1630	{
1631	struct sof_ipc4_audio_format *in_fmt;
1632
1633	in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1634	out_ref_rate = in_fmt->sampling_frequency;
1635	out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1636
1637	if (!single_output_format)
1638	out_ref_valid_bits =
1639	SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1640	break;
1641	}
1642	case snd_soc_dapm_aif_out:
1643	case snd_soc_dapm_dai_in:
1644	out_ref_rate = params_rate(p: fe_params);
1645	out_ref_channels = params_channels(p: fe_params);
1646	if (!single_output_format) {
1647	out_ref_valid_bits = sof_ipc4_get_valid_bits(sdev, params: fe_params);
1648	if (out_ref_valid_bits < 0)
1649	return out_ref_valid_bits;
1650	}
1651	break;
1652	default:
1653	/*
1654	* Unsupported type should be caught by the former switch default
1655	* case, this should never happen in reality.
1656	*/
1657	return -EINVAL;
1658	}
1659
1660	/*
1661	* if the output format is the same across all available output formats, choose
1662	* that as the reference.
1663	*/
1664	if (single_output_format) {
1665	struct sof_ipc4_audio_format *out_fmt;
1666
1667	out_fmt = &available_fmt->output_pin_fmts[0].audio_fmt;
1668	out_ref_valid_bits =
1669	SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(out_fmt->fmt_cfg);
1670	}
1671
1672	dev_dbg(sdev->dev, "copier %s: reference output rate %d, channels %d valid_bits %d\n",
1673	swidget->widget->name, out_ref_rate, out_ref_channels, out_ref_valid_bits);
1674
1675	output_fmt_index = sof_ipc4_init_output_audio_fmt(sdev, base_config: &copier_data->base_config,
1676	available_fmt, out_ref_rate,
1677	out_ref_channels, out_ref_valid_bits);
1678	if (output_fmt_index < 0) {
1679	dev_err(sdev->dev, "Failed to initialize output format for %s",
1680	swidget->widget->name);
1681	return output_fmt_index;
1682	}
1683
1684	/*
1685	* Set the output format. Current topology defines pin 0 input and output formats in pairs.
1686	* This assumes that the pin 0 formats are defined before all other pins.
1687	* So pick the output audio format with the same index as the chosen
1688	* input format. This logic will need to be updated when the format definitions
1689	* in topology change.
1690	*/
1691	memcpy(&copier_data->out_format,
1692	&available_fmt->output_pin_fmts[output_fmt_index].audio_fmt,
1693	sizeof(struct sof_ipc4_audio_format));
1694	dev_dbg(sdev->dev, "Output audio format for %s\n", swidget->widget->name);
1695	sof_ipc4_dbg_audio_format(dev: sdev->dev, pin_fmt: &available_fmt->output_pin_fmts[output_fmt_index], num_formats: 1);
1696
1697	switch (swidget->id) {
1698	case snd_soc_dapm_dai_in:
1699	case snd_soc_dapm_dai_out:
1700	{
1701	/*
1702	* Only SOF_DAI_INTEL_ALH needs copier_data to set blob.
1703	* That's why only ALH dai's blob is set after sof_ipc4_init_input_audio_fmt
1704	*/
1705	if (ipc4_copier->dai_type == SOF_DAI_INTEL_ALH) {
1706	struct sof_ipc4_alh_configuration_blob *blob;
1707	struct sof_ipc4_copier_data *alh_data;
1708	struct sof_ipc4_copier *alh_copier;
1709	struct snd_sof_widget *w;
1710	u32 ch_count = 0;
1711	u32 ch_mask = 0;
1712	u32 ch_map;
1713	u32 step;
1714	u32 mask;
1715	int i;
1716
1717	blob = (struct sof_ipc4_alh_configuration_blob *)ipc4_copier->copier_config;
1718
1719	blob->gw_attr.lp_buffer_alloc = 0;
1720
1721	/* Get channel_mask from ch_map */
1722	ch_map = copier_data->base_config.audio_fmt.ch_map;
1723	for (i = 0; ch_map; i++) {
1724	if ((ch_map & 0xf) != 0xf) {
1725	ch_mask |= BIT(i);
1726	ch_count++;
1727	}
1728	ch_map >>= 4;
1729	}
1730
1731	step = ch_count / blob->alh_cfg.device_count;
1732	mask = GENMASK(step - 1, 0);
1733	/*
1734	* Set each gtw_cfg.node_id to blob->alh_cfg.mapping[]
1735	* for all widgets with the same stream name
1736	*/
1737	i = 0;
1738	list_for_each_entry(w, &sdev->widget_list, list) {
1739	if (w->widget->sname &&
1740	strcmp(w->widget->sname, swidget->widget->sname))
1741	continue;
1742
1743	dai = w->private;
1744	alh_copier = (struct sof_ipc4_copier *)dai->private;
1745	alh_data = &alh_copier->data;
1746	blob->alh_cfg.mapping[i].device = alh_data->gtw_cfg.node_id;
1747	/*
1748	* Set the same channel mask for playback as the audio data is
1749	* duplicated for all speakers. For capture, split the channels
1750	* among the aggregated DAIs. For example, with 4 channels on 2
1751	* aggregated DAIs, the channel_mask should be 0x3 and 0xc for the
1752	* two DAI's.
1753	* The channel masks used depend on the cpu_dais used in the
1754	* dailink at the machine driver level, which actually comes from
1755	* the tables in soc_acpi files depending on the _ADR and devID
1756	* registers for each codec.
1757	*/
1758	if (w->id == snd_soc_dapm_dai_in)
1759	blob->alh_cfg.mapping[i].channel_mask = ch_mask;
1760	else
1761	blob->alh_cfg.mapping[i].channel_mask = mask << (step * i);
1762
1763	i++;
1764	}
1765	if (blob->alh_cfg.device_count > 1) {
1766	int group_id;
1767
1768	group_id = ida_alloc_max(ida: &alh_group_ida, ALH_MULTI_GTW_COUNT - 1,
1769	GFP_KERNEL);
1770
1771	if (group_id < 0)
1772	return group_id;
1773
1774	/* add multi-gateway base */
1775	group_id += ALH_MULTI_GTW_BASE;
1776	copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1777	copier_data->gtw_cfg.node_id |= SOF_IPC4_NODE_INDEX(group_id);
1778	}
1779	}
1780	}
1781	}
1782
1783	/* modify the input params for the next widget */
1784	ret = sof_ipc4_update_hw_params(sdev, params: pipeline_params, fmt: &copier_data->out_format);
1785	if (ret)
1786	return ret;
1787
1788	/*
1789	* Set the gateway dma_buffer_size to 2ms buffer size to meet the FW expectation. In the
1790	* deep buffer case, set the dma_buffer_size depending on the deep_buffer_dma_ms set
1791	* in topology.
1792	*/
1793	switch (swidget->id) {
1794	case snd_soc_dapm_dai_in:
1795	copier_data->gtw_cfg.dma_buffer_size =
1796	SOF_IPC4_MIN_DMA_BUFFER_SIZE * copier_data->base_config.ibs;
1797	break;
1798	case snd_soc_dapm_aif_in:
1799	copier_data->gtw_cfg.dma_buffer_size =
1800	max((u32)SOF_IPC4_MIN_DMA_BUFFER_SIZE, deep_buffer_dma_ms) *
1801	copier_data->base_config.ibs;
1802	dev_dbg(sdev->dev, "copier %s, dma buffer%s: %u ms (%u bytes)",
1803	swidget->widget->name,
1804	deep_buffer_dma_ms ? " (using Deep Buffer)" : "",
1805	max((u32)SOF_IPC4_MIN_DMA_BUFFER_SIZE, deep_buffer_dma_ms),
1806	copier_data->gtw_cfg.dma_buffer_size);
1807	break;
1808	case snd_soc_dapm_dai_out:
1809	case snd_soc_dapm_aif_out:
1810	copier_data->gtw_cfg.dma_buffer_size =
1811	SOF_IPC4_MIN_DMA_BUFFER_SIZE * copier_data->base_config.obs;
1812	break;
1813	default:
1814	break;
1815	}
1816
1817	data = &ipc4_copier->copier_config;
1818	ipc_config_size = &ipc4_copier->ipc_config_size;
1819	ipc_config_data = &ipc4_copier->ipc_config_data;
1820
1821	/* config_length is DWORD based */
1822	gtw_cfg_config_length = copier_data->gtw_cfg.config_length * 4;
1823	ipc_size = sizeof(*copier_data) + gtw_cfg_config_length;
1824
1825	if (ipc4_copier->dma_config_tlv.type == SOF_IPC4_GTW_DMA_CONFIG_ID &&
1826	ipc4_copier->dma_config_tlv.length) {
1827	dma_config_tlv_size = sizeof(ipc4_copier->dma_config_tlv) +
1828	ipc4_copier->dma_config_tlv.dma_config.dma_priv_config_size;
1829
1830	/* paranoia check on TLV size/length */
1831	if (dma_config_tlv_size != ipc4_copier->dma_config_tlv.length +
1832	sizeof(uint32_t) * 2) {
1833	dev_err(sdev->dev, "Invalid configuration, TLV size %d length %d\n",
1834	dma_config_tlv_size, ipc4_copier->dma_config_tlv.length);
1835	return -EINVAL;
1836	}
1837
1838	ipc_size += dma_config_tlv_size;
1839
1840	/* we also need to increase the size at the gtw level */
1841	copier_data->gtw_cfg.config_length += dma_config_tlv_size / 4;
1842	}
1843
1844	dev_dbg(sdev->dev, "copier %s, IPC size is %d", swidget->widget->name, ipc_size);
1845
1846	*ipc_config_data = kzalloc(size: ipc_size, GFP_KERNEL);
1847	if (!*ipc_config_data)
1848	return -ENOMEM;
1849
1850	*ipc_config_size = ipc_size;
1851
1852	/* update pipeline memory usage */
1853	sof_ipc4_update_resource_usage(sdev, swidget, base_config: &copier_data->base_config);
1854
1855	/* copy IPC data */
1856	memcpy(*ipc_config_data, (void *)copier_data, sizeof(*copier_data));
1857	if (gtw_cfg_config_length)
1858	memcpy(*ipc_config_data + sizeof(*copier_data),
1859	*data, gtw_cfg_config_length);
1860
1861	/* add DMA Config TLV, if configured */
1862	if (dma_config_tlv_size)
1863	memcpy(*ipc_config_data + sizeof(*copier_data) +
1864	gtw_cfg_config_length,
1865	&ipc4_copier->dma_config_tlv, dma_config_tlv_size);
1866
1867	/*
1868	* Restore gateway config length now that IPC payload is prepared. This avoids
1869	* counting the DMA CONFIG TLV multiple times
1870	*/
1871	copier_data->gtw_cfg.config_length = gtw_cfg_config_length / 4;
1872
1873	return 0;
1874	}
1875
1876	static int sof_ipc4_prepare_gain_module(struct snd_sof_widget *swidget,
1877	struct snd_pcm_hw_params *fe_params,
1878	struct snd_sof_platform_stream_params *platform_params,
1879	struct snd_pcm_hw_params *pipeline_params, int dir)
1880	{
1881	struct snd_soc_component *scomp = swidget->scomp;
1882	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
1883	struct sof_ipc4_gain *gain = swidget->private;
1884	struct sof_ipc4_available_audio_format *available_fmt = &gain->available_fmt;
1885	struct sof_ipc4_audio_format *in_fmt;
1886	u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1887	int ret;
1888
1889	ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, base_config: &gain->data.base_config,
1890	params: pipeline_params, available_fmt);
1891	if (ret < 0)
1892	return ret;
1893
1894	in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1895	out_ref_rate = in_fmt->sampling_frequency;
1896	out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1897	out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1898
1899	ret = sof_ipc4_init_output_audio_fmt(sdev, base_config: &gain->data.base_config, available_fmt,
1900	out_ref_rate, out_ref_channels, out_ref_valid_bits);
1901	if (ret < 0) {
1902	dev_err(sdev->dev, "Failed to initialize output format for %s",
1903	swidget->widget->name);
1904	return ret;
1905	}
1906
1907	/* update pipeline memory usage */
1908	sof_ipc4_update_resource_usage(sdev, swidget, base_config: &gain->data.base_config);
1909
1910	return 0;
1911	}
1912
1913	static int sof_ipc4_prepare_mixer_module(struct snd_sof_widget *swidget,
1914	struct snd_pcm_hw_params *fe_params,
1915	struct snd_sof_platform_stream_params *platform_params,
1916	struct snd_pcm_hw_params *pipeline_params, int dir)
1917	{
1918	struct snd_soc_component *scomp = swidget->scomp;
1919	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
1920	struct sof_ipc4_mixer *mixer = swidget->private;
1921	struct sof_ipc4_available_audio_format *available_fmt = &mixer->available_fmt;
1922	struct sof_ipc4_audio_format *in_fmt;
1923	u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1924	int ret;
1925
1926	ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, base_config: &mixer->base_config,
1927	params: pipeline_params, available_fmt);
1928	if (ret < 0)
1929	return ret;
1930
1931	in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1932	out_ref_rate = in_fmt->sampling_frequency;
1933	out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1934	out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1935
1936	ret = sof_ipc4_init_output_audio_fmt(sdev, base_config: &mixer->base_config, available_fmt,
1937	out_ref_rate, out_ref_channels, out_ref_valid_bits);
1938	if (ret < 0) {
1939	dev_err(sdev->dev, "Failed to initialize output format for %s",
1940	swidget->widget->name);
1941	return ret;
1942	}
1943
1944	/* update pipeline memory usage */
1945	sof_ipc4_update_resource_usage(sdev, swidget, base_config: &mixer->base_config);
1946
1947	return 0;
1948	}
1949
1950	static int sof_ipc4_prepare_src_module(struct snd_sof_widget *swidget,
1951	struct snd_pcm_hw_params *fe_params,
1952	struct snd_sof_platform_stream_params *platform_params,
1953	struct snd_pcm_hw_params *pipeline_params, int dir)
1954	{
1955	struct snd_soc_component *scomp = swidget->scomp;
1956	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
1957	struct sof_ipc4_src *src = swidget->private;
1958	struct sof_ipc4_available_audio_format *available_fmt = &src->available_fmt;
1959	struct sof_ipc4_audio_format *out_audio_fmt;
1960	struct sof_ipc4_audio_format *in_audio_fmt;
1961	u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1962	int output_format_index, input_format_index;
1963
1964	input_format_index = sof_ipc4_init_input_audio_fmt(sdev, swidget, base_config: &src->data.base_config,
1965	params: pipeline_params, available_fmt);
1966	if (input_format_index < 0)
1967	return input_format_index;
1968
1969	/*
1970	* For playback, the SRC sink rate will be configured based on the requested output
1971	* format, which is restricted to only deal with DAI's with a single format for now.
1972	*/
1973	if (dir == SNDRV_PCM_STREAM_PLAYBACK && available_fmt->num_output_formats > 1) {
1974	dev_err(sdev->dev, "Invalid number of output formats: %d for SRC %s\n",
1975	available_fmt->num_output_formats, swidget->widget->name);
1976	return -EINVAL;
1977	}
1978
1979	/*
1980	* SRC does not perform format conversion, so the output channels and valid bit depth must
1981	* be the same as that of the input.
1982	*/
1983	in_audio_fmt = &available_fmt->input_pin_fmts[input_format_index].audio_fmt;
1984	out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_audio_fmt->fmt_cfg);
1985	out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_audio_fmt->fmt_cfg);
1986
1987	/*
1988	* For capture, the SRC module should convert the rate to match the rate requested by the
1989	* PCM hw_params. Set the reference params based on the fe_params unconditionally as it
1990	* will be ignored for playback anyway.
1991	*/
1992	out_ref_rate = params_rate(p: fe_params);
1993
1994	output_format_index = sof_ipc4_init_output_audio_fmt(sdev, base_config: &src->data.base_config,
1995	available_fmt, out_ref_rate,
1996	out_ref_channels, out_ref_valid_bits);
1997	if (output_format_index < 0) {
1998	dev_err(sdev->dev, "Failed to initialize output format for %s",
1999	swidget->widget->name);
2000	return output_format_index;
2001	}
2002
2003	/* update pipeline memory usage */
2004	sof_ipc4_update_resource_usage(sdev, swidget, base_config: &src->data.base_config);
2005
2006	out_audio_fmt = &available_fmt->output_pin_fmts[output_format_index].audio_fmt;
2007	src->data.sink_rate = out_audio_fmt->sampling_frequency;
2008
2009	/* update pipeline_params for sink widgets */
2010	return sof_ipc4_update_hw_params(sdev, params: pipeline_params, fmt: out_audio_fmt);
2011	}
2012
2013	static int
2014	sof_ipc4_process_set_pin_formats(struct snd_sof_widget *swidget, int pin_type)
2015	{
2016	struct sof_ipc4_process *process = swidget->private;
2017	struct sof_ipc4_base_module_cfg_ext *base_cfg_ext = process->base_config_ext;
2018	struct sof_ipc4_available_audio_format *available_fmt = &process->available_fmt;
2019	struct sof_ipc4_pin_format *pin_format, *format_list_to_search;
2020	struct snd_soc_component *scomp = swidget->scomp;
2021	int num_pins, format_list_count;
2022	int pin_format_offset = 0;
2023	int i, j;
2024
2025	/* set number of pins, offset of pin format and format list to search based on pin type */
2026	if (pin_type == SOF_PIN_TYPE_INPUT) {
2027	num_pins = swidget->num_input_pins;
2028	format_list_to_search = available_fmt->input_pin_fmts;
2029	format_list_count = available_fmt->num_input_formats;
2030	} else {
2031	num_pins = swidget->num_output_pins;
2032	pin_format_offset = swidget->num_input_pins;
2033	format_list_to_search = available_fmt->output_pin_fmts;
2034	format_list_count = available_fmt->num_output_formats;
2035	}
2036
2037	for (i = pin_format_offset; i < num_pins + pin_format_offset; i++) {
2038	pin_format = &base_cfg_ext->pin_formats[i];
2039
2040	/* Pin 0 audio formats are derived from the base config input/output format */
2041	if (i == pin_format_offset) {
2042	if (pin_type == SOF_PIN_TYPE_INPUT) {
2043	pin_format->buffer_size = process->base_config.ibs;
2044	pin_format->audio_fmt = process->base_config.audio_fmt;
2045	} else {
2046	pin_format->buffer_size = process->base_config.obs;
2047	pin_format->audio_fmt = process->output_format;
2048	}
2049	continue;
2050	}
2051
2052	/*
2053	* For all other pins, find the pin formats from those set in topology. If there
2054	* is more than one format specified for a pin, this will pick the first available
2055	* one.
2056	*/
2057	for (j = 0; j < format_list_count; j++) {
2058	struct sof_ipc4_pin_format *pin_format_item = &format_list_to_search[j];
2059
2060	if (pin_format_item->pin_index == i - pin_format_offset) {
2061	*pin_format = *pin_format_item;
2062	break;
2063	}
2064	}
2065
2066	if (j == format_list_count) {
2067	dev_err(scomp->dev, "%s pin %d format not found for %s\n",
2068	(pin_type == SOF_PIN_TYPE_INPUT) ? "input" : "output",
2069	i - pin_format_offset, swidget->widget->name);
2070	return -EINVAL;
2071	}
2072	}
2073
2074	return 0;
2075	}
2076
2077	static int sof_ipc4_process_add_base_cfg_extn(struct snd_sof_widget *swidget)
2078	{
2079	int ret, i;
2080
2081	/* copy input and output pin formats */
2082	for (i = 0; i <= SOF_PIN_TYPE_OUTPUT; i++) {
2083	ret = sof_ipc4_process_set_pin_formats(swidget, pin_type: i);
2084	if (ret < 0)
2085	return ret;
2086	}
2087
2088	return 0;
2089	}
2090
2091	static int sof_ipc4_prepare_process_module(struct snd_sof_widget *swidget,
2092	struct snd_pcm_hw_params *fe_params,
2093	struct snd_sof_platform_stream_params *platform_params,
2094	struct snd_pcm_hw_params *pipeline_params, int dir)
2095	{
2096	struct snd_soc_component *scomp = swidget->scomp;
2097	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
2098	struct sof_ipc4_process *process = swidget->private;
2099	struct sof_ipc4_available_audio_format *available_fmt = &process->available_fmt;
2100	struct sof_ipc4_audio_format *in_fmt;
2101	u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
2102	void *cfg = process->ipc_config_data;
2103	int output_fmt_index;
2104	int ret;
2105
2106	ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, base_config: &process->base_config,
2107	params: pipeline_params, available_fmt);
2108	if (ret < 0)
2109	return ret;
2110
2111	in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
2112	out_ref_rate = in_fmt->sampling_frequency;
2113	out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
2114	out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
2115
2116	output_fmt_index = sof_ipc4_init_output_audio_fmt(sdev, base_config: &process->base_config,
2117	available_fmt, out_ref_rate,
2118	out_ref_channels, out_ref_valid_bits);
2119	if (output_fmt_index < 0 && available_fmt->num_output_formats) {
2120	dev_err(sdev->dev, "Failed to initialize output format for %s",
2121	swidget->widget->name);
2122	return output_fmt_index;
2123	}
2124
2125	/* copy Pin 0 output format */
2126	if (available_fmt->num_output_formats &&
2127	output_fmt_index < available_fmt->num_output_formats &&
2128	!available_fmt->output_pin_fmts[output_fmt_index].pin_index) {
2129	memcpy(&process->output_format,
2130	&available_fmt->output_pin_fmts[output_fmt_index].audio_fmt,
2131	sizeof(struct sof_ipc4_audio_format));
2132
2133	/* modify the pipeline params with the pin 0 output format */
2134	ret = sof_ipc4_update_hw_params(sdev, params: pipeline_params, fmt: &process->output_format);
2135	if (ret)
2136	return ret;
2137	}
2138
2139	/* update pipeline memory usage */
2140	sof_ipc4_update_resource_usage(sdev, swidget, base_config: &process->base_config);
2141
2142	/* ipc_config_data is composed of the base_config followed by an optional extension */
2143	memcpy(cfg, &process->base_config, sizeof(struct sof_ipc4_base_module_cfg));
2144	cfg += sizeof(struct sof_ipc4_base_module_cfg);
2145
2146	if (process->init_config == SOF_IPC4_MODULE_INIT_CONFIG_TYPE_BASE_CFG_WITH_EXT) {
2147	struct sof_ipc4_base_module_cfg_ext *base_cfg_ext = process->base_config_ext;
2148
2149	ret = sof_ipc4_process_add_base_cfg_extn(swidget);
2150	if (ret < 0)
2151	return ret;
2152
2153	memcpy(cfg, base_cfg_ext, process->base_config_ext_size);
2154	}
2155
2156	return 0;
2157	}
2158
2159	static int sof_ipc4_control_load_volume(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2160	{
2161	struct sof_ipc4_control_data *control_data;
2162	struct sof_ipc4_msg *msg;
2163	int i;
2164
2165	scontrol->size = struct_size(control_data, chanv, scontrol->num_channels);
2166
2167	/* scontrol->ipc_control_data will be freed in sof_control_unload */
2168	scontrol->ipc_control_data = kzalloc(size: scontrol->size, GFP_KERNEL);
2169	if (!scontrol->ipc_control_data)
2170	return -ENOMEM;
2171
2172	control_data = scontrol->ipc_control_data;
2173	control_data->index = scontrol->index;
2174
2175	msg = &control_data->msg;
2176	msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2177	msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2178	msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2179
2180	/* volume controls with range 0-1 (off/on) are switch controls */
2181	if (scontrol->max == 1)
2182	msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID);
2183	else
2184	msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_GAIN_PARAM_ID);
2185
2186	for (i = 0; i < scontrol->num_channels; i++) {
2187	control_data->chanv[i].channel = i;
2188	/*
2189	* Default, initial values:
2190	* - 0dB for volume controls
2191	* - off (0) for switch controls - value already zero after
2192	* memory allocation
2193	*/
2194	if (scontrol->max > 1)
2195	control_data->chanv[i].value = SOF_IPC4_VOL_ZERO_DB;
2196	}
2197
2198	return 0;
2199	}
2200
2201	static int sof_ipc4_control_load_enum(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2202	{
2203	struct sof_ipc4_control_data *control_data;
2204	struct sof_ipc4_msg *msg;
2205	int i;
2206
2207	scontrol->size = struct_size(control_data, chanv, scontrol->num_channels);
2208
2209	/* scontrol->ipc_control_data will be freed in sof_control_unload */
2210	scontrol->ipc_control_data = kzalloc(size: scontrol->size, GFP_KERNEL);
2211	if (!scontrol->ipc_control_data)
2212	return -ENOMEM;
2213
2214	control_data = scontrol->ipc_control_data;
2215	control_data->index = scontrol->index;
2216
2217	msg = &control_data->msg;
2218	msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2219	msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2220	msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2221
2222	msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID);
2223
2224	/* Default, initial value for enums: first enum entry is selected (0) */
2225	for (i = 0; i < scontrol->num_channels; i++)
2226	control_data->chanv[i].channel = i;
2227
2228	return 0;
2229	}
2230
2231	static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2232	{
2233	struct sof_ipc4_control_data *control_data;
2234	struct sof_ipc4_msg *msg;
2235	int ret;
2236
2237	if (scontrol->max_size < (sizeof(*control_data) + sizeof(struct sof_abi_hdr))) {
2238	dev_err(sdev->dev, "insufficient size for a bytes control %s: %zu.\n",
2239	scontrol->name, scontrol->max_size);
2240	return -EINVAL;
2241	}
2242
2243	if (scontrol->priv_size > scontrol->max_size - sizeof(*control_data)) {
2244	dev_err(sdev->dev, "scontrol %s bytes data size %zu exceeds max %zu.\n",
2245	scontrol->name, scontrol->priv_size,
2246	scontrol->max_size - sizeof(*control_data));
2247	return -EINVAL;
2248	}
2249
2250	scontrol->size = sizeof(struct sof_ipc4_control_data) + scontrol->priv_size;
2251
2252	scontrol->ipc_control_data = kzalloc(size: scontrol->max_size, GFP_KERNEL);
2253	if (!scontrol->ipc_control_data)
2254	return -ENOMEM;
2255
2256	control_data = scontrol->ipc_control_data;
2257	control_data->index = scontrol->index;
2258	if (scontrol->priv_size > 0) {
2259	memcpy(control_data->data, scontrol->priv, scontrol->priv_size);
2260	kfree(objp: scontrol->priv);
2261	scontrol->priv = NULL;
2262
2263	if (control_data->data->magic != SOF_IPC4_ABI_MAGIC) {
2264	dev_err(sdev->dev, "Wrong ABI magic (%#x) for control: %s\n",
2265	control_data->data->magic, scontrol->name);
2266	ret = -EINVAL;
2267	goto err;
2268	}
2269
2270	/* TODO: check the ABI version */
2271
2272	if (control_data->data->size + sizeof(struct sof_abi_hdr) !=
2273	scontrol->priv_size) {
2274	dev_err(sdev->dev, "Control %s conflict in bytes %zu vs. priv size %zu.\n",
2275	scontrol->name,
2276	control_data->data->size + sizeof(struct sof_abi_hdr),
2277	scontrol->priv_size);
2278	ret = -EINVAL;
2279	goto err;
2280	}
2281	}
2282
2283	msg = &control_data->msg;
2284	msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2285	msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2286	msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2287
2288	return 0;
2289
2290	err:
2291	kfree(objp: scontrol->ipc_control_data);
2292	scontrol->ipc_control_data = NULL;
2293	return ret;
2294	}
2295
2296	static int sof_ipc4_control_setup(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2297	{
2298	switch (scontrol->info_type) {
2299	case SND_SOC_TPLG_CTL_VOLSW:
2300	case SND_SOC_TPLG_CTL_VOLSW_SX:
2301	case SND_SOC_TPLG_CTL_VOLSW_XR_SX:
2302	return sof_ipc4_control_load_volume(sdev, scontrol);
2303	case SND_SOC_TPLG_CTL_BYTES:
2304	return sof_ipc4_control_load_bytes(sdev, scontrol);
2305	case SND_SOC_TPLG_CTL_ENUM:
2306	case SND_SOC_TPLG_CTL_ENUM_VALUE:
2307	return sof_ipc4_control_load_enum(sdev, scontrol);
2308	default:
2309	break;
2310	}
2311
2312	return 0;
2313	}
2314
2315	static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
2316	{
2317	struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2318	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2319	struct sof_ipc4_pipeline *pipeline;
2320	struct sof_ipc4_msg *msg;
2321	void *ipc_data = NULL;
2322	u32 ipc_size = 0;
2323	int ret;
2324
2325	switch (swidget->id) {
2326	case snd_soc_dapm_scheduler:
2327	pipeline = swidget->private;
2328
2329	if (pipeline->use_chain_dma) {
2330	dev_warn(sdev->dev, "use_chain_dma set for scheduler %s",
2331	swidget->widget->name);
2332	return 0;
2333	}
2334
2335	dev_dbg(sdev->dev, "pipeline: %d memory pages: %d\n", swidget->pipeline_id,
2336	pipeline->mem_usage);
2337
2338	msg = &pipeline->msg;
2339	msg->primary |= pipeline->mem_usage;
2340
2341	swidget->instance_id = ida_alloc_max(ida: &pipeline_ida, max: ipc4_data->max_num_pipelines,
2342	GFP_KERNEL);
2343	if (swidget->instance_id < 0) {
2344	dev_err(sdev->dev, "failed to assign pipeline id for %s: %d\n",
2345	swidget->widget->name, swidget->instance_id);
2346	return swidget->instance_id;
2347	}
2348	msg->primary &= ~SOF_IPC4_GLB_PIPE_INSTANCE_MASK;
2349	msg->primary |= SOF_IPC4_GLB_PIPE_INSTANCE_ID(swidget->instance_id);
2350	break;
2351	case snd_soc_dapm_aif_in:
2352	case snd_soc_dapm_aif_out:
2353	case snd_soc_dapm_buffer:
2354	{
2355	struct sof_ipc4_copier *ipc4_copier = swidget->private;
2356
2357	pipeline = pipe_widget->private;
2358	if (pipeline->use_chain_dma)
2359	return 0;
2360
2361	ipc_size = ipc4_copier->ipc_config_size;
2362	ipc_data = ipc4_copier->ipc_config_data;
2363
2364	msg = &ipc4_copier->msg;
2365	break;
2366	}
2367	case snd_soc_dapm_dai_in:
2368	case snd_soc_dapm_dai_out:
2369	{
2370	struct snd_sof_dai *dai = swidget->private;
2371	struct sof_ipc4_copier *ipc4_copier = dai->private;
2372
2373	pipeline = pipe_widget->private;
2374	if (pipeline->use_chain_dma)
2375	return 0;
2376
2377	ipc_size = ipc4_copier->ipc_config_size;
2378	ipc_data = ipc4_copier->ipc_config_data;
2379
2380	msg = &ipc4_copier->msg;
2381	break;
2382	}
2383	case snd_soc_dapm_pga:
2384	{
2385	struct sof_ipc4_gain *gain = swidget->private;
2386
2387	ipc_size = sizeof(gain->data);
2388	ipc_data = &gain->data;
2389
2390	msg = &gain->msg;
2391	break;
2392	}
2393	case snd_soc_dapm_mixer:
2394	{
2395	struct sof_ipc4_mixer *mixer = swidget->private;
2396
2397	ipc_size = sizeof(mixer->base_config);
2398	ipc_data = &mixer->base_config;
2399
2400	msg = &mixer->msg;
2401	break;
2402	}
2403	case snd_soc_dapm_src:
2404	{
2405	struct sof_ipc4_src *src = swidget->private;
2406
2407	ipc_size = sizeof(src->data);
2408	ipc_data = &src->data;
2409
2410	msg = &src->msg;
2411	break;
2412	}
2413	case snd_soc_dapm_effect:
2414	{
2415	struct sof_ipc4_process *process = swidget->private;
2416
2417	if (!process->ipc_config_size) {
2418	dev_err(sdev->dev, "module %s has no config data!\n",
2419	swidget->widget->name);
2420	return -EINVAL;
2421	}
2422
2423	ipc_size = process->ipc_config_size;
2424	ipc_data = process->ipc_config_data;
2425
2426	msg = &process->msg;
2427	break;
2428	}
2429	default:
2430	dev_err(sdev->dev, "widget type %d not supported", swidget->id);
2431	return -EINVAL;
2432	}
2433
2434	if (swidget->id != snd_soc_dapm_scheduler) {
2435	int module_id = msg->primary & SOF_IPC4_MOD_ID_MASK;
2436
2437	ret = sof_ipc4_widget_assign_instance_id(sdev, swidget);
2438	if (ret < 0) {
2439	dev_err(sdev->dev, "failed to assign instance id for %s\n",
2440	swidget->widget->name);
2441	return ret;
2442	}
2443
2444	msg->primary &= ~SOF_IPC4_MOD_INSTANCE_MASK;
2445	msg->primary |= SOF_IPC4_MOD_INSTANCE(swidget->instance_id);
2446
2447	msg->extension &= ~SOF_IPC4_MOD_EXT_PARAM_SIZE_MASK;
2448	msg->extension |= ipc_size >> 2;
2449
2450	msg->extension &= ~SOF_IPC4_MOD_EXT_PPL_ID_MASK;
2451	msg->extension |= SOF_IPC4_MOD_EXT_PPL_ID(pipe_widget->instance_id);
2452
2453	dev_dbg(sdev->dev, "Create widget %s (pipe %d) - ID %d, instance %d, core %d\n",
2454	swidget->widget->name, swidget->pipeline_id, module_id,
2455	swidget->instance_id, swidget->core);
2456	} else {
2457	dev_dbg(sdev->dev, "Create pipeline %s (pipe %d) - instance %d, core %d\n",
2458	swidget->widget->name, swidget->pipeline_id,
2459	swidget->instance_id, swidget->core);
2460	}
2461
2462	msg->data_size = ipc_size;
2463	msg->data_ptr = ipc_data;
2464
2465	ret = sof_ipc_tx_message_no_reply(ipc: sdev->ipc, msg_data: msg, msg_bytes: ipc_size);
2466	if (ret < 0) {
2467	dev_err(sdev->dev, "failed to create module %s\n", swidget->widget->name);
2468
2469	if (swidget->id != snd_soc_dapm_scheduler) {
2470	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
2471
2472	ida_free(&fw_module->m_ida, id: swidget->instance_id);
2473	} else {
2474	ida_free(&pipeline_ida, id: swidget->instance_id);
2475	}
2476	}
2477
2478	return ret;
2479	}
2480
2481	static int sof_ipc4_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
2482	{
2483	struct sof_ipc4_fw_module *fw_module = swidget->module_info;
2484	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2485	int ret = 0;
2486
2487	mutex_lock(&ipc4_data->pipeline_state_mutex);
2488
2489	/* freeing a pipeline frees all the widgets associated with it */
2490	if (swidget->id == snd_soc_dapm_scheduler) {
2491	struct sof_ipc4_pipeline *pipeline = swidget->private;
2492	struct sof_ipc4_msg msg = {{ 0 }};
2493	u32 header;
2494
2495	if (pipeline->use_chain_dma) {
2496	dev_warn(sdev->dev, "use_chain_dma set for scheduler %s",
2497	swidget->widget->name);
2498	mutex_unlock(lock: &ipc4_data->pipeline_state_mutex);
2499	return 0;
2500	}
2501
2502	header = SOF_IPC4_GLB_PIPE_INSTANCE_ID(swidget->instance_id);
2503	header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_DELETE_PIPELINE);
2504	header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2505	header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
2506
2507	msg.primary = header;
2508
2509	ret = sof_ipc_tx_message_no_reply(ipc: sdev->ipc, msg_data: &msg, msg_bytes: 0);
2510	if (ret < 0)
2511	dev_err(sdev->dev, "failed to free pipeline widget %s\n",
2512	swidget->widget->name);
2513
2514	pipeline->mem_usage = 0;
2515	pipeline->state = SOF_IPC4_PIPE_UNINITIALIZED;
2516	ida_free(&pipeline_ida, id: swidget->instance_id);
2517	swidget->instance_id = -EINVAL;
2518	} else {
2519	struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2520	struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
2521
2522	if (!pipeline->use_chain_dma)
2523	ida_free(&fw_module->m_ida, id: swidget->instance_id);
2524	}
2525
2526	mutex_unlock(lock: &ipc4_data->pipeline_state_mutex);
2527
2528	return ret;
2529	}
2530
2531	static int sof_ipc4_get_queue_id(struct snd_sof_widget *src_widget,
2532	struct snd_sof_widget *sink_widget, bool pin_type)
2533	{
2534	struct snd_sof_widget *current_swidget;
2535	struct snd_soc_component *scomp;
2536	struct ida *queue_ida;
2537	const char *buddy_name;
2538	char **pin_binding;
2539	u32 num_pins;
2540	int i;
2541
2542	if (pin_type == SOF_PIN_TYPE_OUTPUT) {
2543	current_swidget = src_widget;
2544	pin_binding = src_widget->output_pin_binding;
2545	queue_ida = &src_widget->output_queue_ida;
2546	num_pins = src_widget->num_output_pins;
2547	buddy_name = sink_widget->widget->name;
2548	} else {
2549	current_swidget = sink_widget;
2550	pin_binding = sink_widget->input_pin_binding;
2551	queue_ida = &sink_widget->input_queue_ida;
2552	num_pins = sink_widget->num_input_pins;
2553	buddy_name = src_widget->widget->name;
2554	}
2555
2556	scomp = current_swidget->scomp;
2557
2558	if (num_pins < 1) {
2559	dev_err(scomp->dev, "invalid %s num_pins: %d for queue allocation for %s\n",
2560	(pin_type == SOF_PIN_TYPE_OUTPUT ? "output" : "input"),
2561	num_pins, current_swidget->widget->name);
2562	return -EINVAL;
2563	}
2564
2565	/* If there is only one input/output pin, queue id must be 0 */
2566	if (num_pins == 1)
2567	return 0;
2568
2569	/* Allocate queue ID from pin binding array if it is defined in topology. */
2570	if (pin_binding) {
2571	for (i = 0; i < num_pins; i++) {
2572	if (!strcmp(pin_binding[i], buddy_name))
2573	return i;
2574	}
2575	/*
2576	* Fail if no queue ID found from pin binding array, so that we don't
2577	* mixed use pin binding array and ida for queue ID allocation.
2578	*/
2579	dev_err(scomp->dev, "no %s queue id found from pin binding array for %s\n",
2580	(pin_type == SOF_PIN_TYPE_OUTPUT ? "output" : "input"),
2581	current_swidget->widget->name);
2582	return -EINVAL;
2583	}
2584
2585	/* If no pin binding array specified in topology, use ida to allocate one */
2586	return ida_alloc_max(ida: queue_ida, max: num_pins, GFP_KERNEL);
2587	}
2588
2589	static void sof_ipc4_put_queue_id(struct snd_sof_widget *swidget, int queue_id,
2590	bool pin_type)
2591	{
2592	struct ida *queue_ida;
2593	char **pin_binding;
2594	int num_pins;
2595
2596	if (pin_type == SOF_PIN_TYPE_OUTPUT) {
2597	pin_binding = swidget->output_pin_binding;
2598	queue_ida = &swidget->output_queue_ida;
2599	num_pins = swidget->num_output_pins;
2600	} else {
2601	pin_binding = swidget->input_pin_binding;
2602	queue_ida = &swidget->input_queue_ida;
2603	num_pins = swidget->num_input_pins;
2604	}
2605
2606	/* Nothing to free if queue ID is not allocated with ida. */
2607	if (num_pins == 1 || pin_binding)
2608	return;
2609
2610	ida_free(queue_ida, id: queue_id);
2611	}
2612
2613	static int sof_ipc4_set_copier_sink_format(struct snd_sof_dev *sdev,
2614	struct snd_sof_widget *src_widget,
2615	struct snd_sof_widget *sink_widget,
2616	int sink_id)
2617	{
2618	struct sof_ipc4_copier_config_set_sink_format format;
2619	const struct sof_ipc_ops *iops = sdev->ipc->ops;
2620	struct sof_ipc4_base_module_cfg *src_config;
2621	const struct sof_ipc4_audio_format *pin_fmt;
2622	struct sof_ipc4_fw_module *fw_module;
2623	struct sof_ipc4_msg msg = {{ 0 }};
2624
2625	dev_dbg(sdev->dev, "%s set copier sink %d format\n",
2626	src_widget->widget->name, sink_id);
2627
2628	if (WIDGET_IS_DAI(src_widget->id)) {
2629	struct snd_sof_dai *dai = src_widget->private;
2630
2631	src_config = dai->private;
2632	} else {
2633	src_config = src_widget->private;
2634	}
2635
2636	fw_module = src_widget->module_info;
2637
2638	format.sink_id = sink_id;
2639	memcpy(&format.source_fmt, &src_config->audio_fmt, sizeof(format.source_fmt));
2640
2641	pin_fmt = sof_ipc4_get_input_pin_audio_fmt(swidget: sink_widget, pin_index: sink_id);
2642	if (!pin_fmt) {
2643	dev_err(sdev->dev, "Unable to get pin %d format for %s",
2644	sink_id, sink_widget->widget->name);
2645	return -EINVAL;
2646	}
2647
2648	memcpy(&format.sink_fmt, pin_fmt, sizeof(format.sink_fmt));
2649
2650	msg.data_size = sizeof(format);
2651	msg.data_ptr = &format;
2652
2653	msg.primary = fw_module->man4_module_entry.id;
2654	msg.primary |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2655	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2656	msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2657
2658	msg.extension =
2659	SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_COPIER_MODULE_CFG_PARAM_SET_SINK_FORMAT);
2660
2661	return iops->set_get_data(sdev, &msg, msg.data_size, true);
2662	}
2663
2664	static int sof_ipc4_route_setup(struct snd_sof_dev *sdev, struct snd_sof_route *sroute)
2665	{
2666	struct snd_sof_widget *src_widget = sroute->src_widget;
2667	struct snd_sof_widget *sink_widget = sroute->sink_widget;
2668	struct snd_sof_widget *src_pipe_widget = src_widget->spipe->pipe_widget;
2669	struct snd_sof_widget *sink_pipe_widget = sink_widget->spipe->pipe_widget;
2670	struct sof_ipc4_fw_module *src_fw_module = src_widget->module_info;
2671	struct sof_ipc4_fw_module *sink_fw_module = sink_widget->module_info;
2672	struct sof_ipc4_pipeline *src_pipeline = src_pipe_widget->private;
2673	struct sof_ipc4_pipeline *sink_pipeline = sink_pipe_widget->private;
2674	struct sof_ipc4_msg msg = {{ 0 }};
2675	u32 header, extension;
2676	int ret;
2677
2678	/* no route set up if chain DMA is used */
2679	if (src_pipeline->use_chain_dma || sink_pipeline->use_chain_dma) {
2680	if (!src_pipeline->use_chain_dma || !sink_pipeline->use_chain_dma) {
2681	dev_err(sdev->dev,
2682	"use_chain_dma must be set for both src %s and sink %s pipelines\n",
2683	src_widget->widget->name, sink_widget->widget->name);
2684	return -EINVAL;
2685	}
2686	return 0;
2687	}
2688
2689	if (!src_fw_module || !sink_fw_module) {
2690	dev_err(sdev->dev,
2691	"cannot bind %s -> %s, no firmware module for: %s%s\n",
2692	src_widget->widget->name, sink_widget->widget->name,
2693	src_fw_module ? "" : " source",
2694	sink_fw_module ? "" : " sink");
2695
2696	return -ENODEV;
2697	}
2698
2699	sroute->src_queue_id = sof_ipc4_get_queue_id(src_widget, sink_widget,
2700	SOF_PIN_TYPE_OUTPUT);
2701	if (sroute->src_queue_id < 0) {
2702	dev_err(sdev->dev, "failed to get queue ID for source widget: %s\n",
2703	src_widget->widget->name);
2704	return sroute->src_queue_id;
2705	}
2706
2707	sroute->dst_queue_id = sof_ipc4_get_queue_id(src_widget, sink_widget,
2708	SOF_PIN_TYPE_INPUT);
2709	if (sroute->dst_queue_id < 0) {
2710	dev_err(sdev->dev, "failed to get queue ID for sink widget: %s\n",
2711	sink_widget->widget->name);
2712	sof_ipc4_put_queue_id(swidget: src_widget, queue_id: sroute->src_queue_id,
2713	SOF_PIN_TYPE_OUTPUT);
2714	return sroute->dst_queue_id;
2715	}
2716
2717	/* Pin 0 format is already set during copier module init */
2718	if (sroute->src_queue_id > 0 && WIDGET_IS_COPIER(src_widget->id)) {
2719	ret = sof_ipc4_set_copier_sink_format(sdev, src_widget, sink_widget,
2720	sink_id: sroute->src_queue_id);
2721	if (ret < 0) {
2722	dev_err(sdev->dev, "failed to set sink format for %s source queue ID %d\n",
2723	src_widget->widget->name, sroute->src_queue_id);
2724	goto out;
2725	}
2726	}
2727
2728	dev_dbg(sdev->dev, "bind %s:%d -> %s:%d\n",
2729	src_widget->widget->name, sroute->src_queue_id,
2730	sink_widget->widget->name, sroute->dst_queue_id);
2731
2732	header = src_fw_module->man4_module_entry.id;
2733	header |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2734	header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_BIND);
2735	header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2736	header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2737
2738	extension = sink_fw_module->man4_module_entry.id;
2739	extension |= SOF_IPC4_MOD_EXT_DST_MOD_INSTANCE(sink_widget->instance_id);
2740	extension |= SOF_IPC4_MOD_EXT_DST_MOD_QUEUE_ID(sroute->dst_queue_id);
2741	extension |= SOF_IPC4_MOD_EXT_SRC_MOD_QUEUE_ID(sroute->src_queue_id);
2742
2743	msg.primary = header;
2744	msg.extension = extension;
2745
2746	ret = sof_ipc_tx_message_no_reply(ipc: sdev->ipc, msg_data: &msg, msg_bytes: 0);
2747	if (ret < 0) {
2748	dev_err(sdev->dev, "failed to bind modules %s:%d -> %s:%d\n",
2749	src_widget->widget->name, sroute->src_queue_id,
2750	sink_widget->widget->name, sroute->dst_queue_id);
2751	goto out;
2752	}
2753
2754	return ret;
2755
2756	out:
2757	sof_ipc4_put_queue_id(swidget: src_widget, queue_id: sroute->src_queue_id, SOF_PIN_TYPE_OUTPUT);
2758	sof_ipc4_put_queue_id(swidget: sink_widget, queue_id: sroute->dst_queue_id, SOF_PIN_TYPE_INPUT);
2759	return ret;
2760	}
2761
2762	static int sof_ipc4_route_free(struct snd_sof_dev *sdev, struct snd_sof_route *sroute)
2763	{
2764	struct snd_sof_widget *src_widget = sroute->src_widget;
2765	struct snd_sof_widget *sink_widget = sroute->sink_widget;
2766	struct sof_ipc4_fw_module *src_fw_module = src_widget->module_info;
2767	struct sof_ipc4_fw_module *sink_fw_module = sink_widget->module_info;
2768	struct sof_ipc4_msg msg = {{ 0 }};
2769	struct snd_sof_widget *src_pipe_widget = src_widget->spipe->pipe_widget;
2770	struct snd_sof_widget *sink_pipe_widget = sink_widget->spipe->pipe_widget;
2771	struct sof_ipc4_pipeline *src_pipeline = src_pipe_widget->private;
2772	struct sof_ipc4_pipeline *sink_pipeline = sink_pipe_widget->private;
2773	u32 header, extension;
2774	int ret = 0;
2775
2776	/* no route is set up if chain DMA is used */
2777	if (src_pipeline->use_chain_dma || sink_pipeline->use_chain_dma)
2778	return 0;
2779
2780	dev_dbg(sdev->dev, "unbind modules %s:%d -> %s:%d\n",
2781	src_widget->widget->name, sroute->src_queue_id,
2782	sink_widget->widget->name, sroute->dst_queue_id);
2783
2784	/*
2785	* routes belonging to the same pipeline will be disconnected by the FW when the pipeline
2786	* is freed. So avoid sending this IPC which will be ignored by the FW anyway.
2787	*/
2788	if (src_widget->spipe->pipe_widget == sink_widget->spipe->pipe_widget)
2789	goto out;
2790
2791	header = src_fw_module->man4_module_entry.id;
2792	header |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2793	header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_UNBIND);
2794	header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2795	header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2796
2797	extension = sink_fw_module->man4_module_entry.id;
2798	extension |= SOF_IPC4_MOD_EXT_DST_MOD_INSTANCE(sink_widget->instance_id);
2799	extension |= SOF_IPC4_MOD_EXT_DST_MOD_QUEUE_ID(sroute->dst_queue_id);
2800	extension |= SOF_IPC4_MOD_EXT_SRC_MOD_QUEUE_ID(sroute->src_queue_id);
2801
2802	msg.primary = header;
2803	msg.extension = extension;
2804
2805	ret = sof_ipc_tx_message_no_reply(ipc: sdev->ipc, msg_data: &msg, msg_bytes: 0);
2806	if (ret < 0)
2807	dev_err(sdev->dev, "failed to unbind modules %s:%d -> %s:%d\n",
2808	src_widget->widget->name, sroute->src_queue_id,
2809	sink_widget->widget->name, sroute->dst_queue_id);
2810	out:
2811	sof_ipc4_put_queue_id(swidget: sink_widget, queue_id: sroute->dst_queue_id, SOF_PIN_TYPE_INPUT);
2812	sof_ipc4_put_queue_id(swidget: src_widget, queue_id: sroute->src_queue_id, SOF_PIN_TYPE_OUTPUT);
2813
2814	return ret;
2815	}
2816
2817	static int sof_ipc4_dai_config(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget,
2818	unsigned int flags, struct snd_sof_dai_config_data *data)
2819	{
2820	struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2821	struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
2822	struct snd_sof_dai *dai = swidget->private;
2823	struct sof_ipc4_gtw_attributes *gtw_attr;
2824	struct sof_ipc4_copier_data *copier_data;
2825	struct sof_ipc4_copier *ipc4_copier;
2826
2827	if (!dai || !dai->private) {
2828	dev_err(sdev->dev, "Invalid DAI or DAI private data for %s\n",
2829	swidget->widget->name);
2830	return -EINVAL;
2831	}
2832
2833	ipc4_copier = (struct sof_ipc4_copier *)dai->private;
2834	copier_data = &ipc4_copier->data;
2835
2836	if (!data)
2837	return 0;
2838
2839	if (pipeline->use_chain_dma) {
2840	pipeline->msg.primary &= ~SOF_IPC4_GLB_CHAIN_DMA_LINK_ID_MASK;
2841	pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_LINK_ID(data->dai_data);
2842	return 0;
2843	}
2844
2845	switch (ipc4_copier->dai_type) {
2846	case SOF_DAI_INTEL_HDA:
2847	gtw_attr = ipc4_copier->gtw_attr;
2848	gtw_attr->lp_buffer_alloc = pipeline->lp_mode;
2849	fallthrough;
2850	case SOF_DAI_INTEL_ALH:
2851	/*
2852	* Do not clear the node ID when this op is invoked with
2853	* SOF_DAI_CONFIG_FLAGS_HW_FREE. It is needed to free the group_ida during
2854	* unprepare.
2855	*/
2856	if (flags & SOF_DAI_CONFIG_FLAGS_HW_PARAMS) {
2857	copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
2858	copier_data->gtw_cfg.node_id |= SOF_IPC4_NODE_INDEX(data->dai_data);
2859	}
2860	break;
2861	case SOF_DAI_INTEL_DMIC:
2862	case SOF_DAI_INTEL_SSP:
2863	/* nothing to do for SSP/DMIC */
2864	break;
2865	default:
2866	dev_err(sdev->dev, "%s: unsupported dai type %d\n", __func__,
2867	ipc4_copier->dai_type);
2868	return -EINVAL;
2869	}
2870
2871	return 0;
2872	}
2873
2874	static int sof_ipc4_parse_manifest(struct snd_soc_component *scomp, int index,
2875	struct snd_soc_tplg_manifest *man)
2876	{
2877	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(c: scomp);
2878	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2879	struct sof_manifest_tlv *manifest_tlv;
2880	struct sof_manifest *manifest;
2881	u32 size = le32_to_cpu(man->priv.size);
2882	u8 *man_ptr = man->priv.data;
2883	u32 len_check;
2884	int i;
2885
2886	if (!size || size < SOF_IPC4_TPLG_ABI_SIZE) {
2887	dev_err(scomp->dev, "%s: Invalid topology ABI size: %u\n",
2888	__func__, size);
2889	return -EINVAL;
2890	}
2891
2892	manifest = (struct sof_manifest *)man_ptr;
2893
2894	dev_info(scomp->dev,
2895	"Topology: ABI %d:%d:%d Kernel ABI %u:%u:%u\n",
2896	le16_to_cpu(manifest->abi_major), le16_to_cpu(manifest->abi_minor),
2897	le16_to_cpu(manifest->abi_patch),
2898	SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH);
2899
2900	/* TODO: Add ABI compatibility check */
2901
2902	/* no more data after the ABI version */
2903	if (size <= SOF_IPC4_TPLG_ABI_SIZE)
2904	return 0;
2905
2906	manifest_tlv = manifest->items;
2907	len_check = sizeof(struct sof_manifest);
2908	for (i = 0; i < le16_to_cpu(manifest->count); i++) {
2909	len_check += sizeof(struct sof_manifest_tlv) + le32_to_cpu(manifest_tlv->size);
2910	if (len_check > size)
2911	return -EINVAL;
2912
2913	switch (le32_to_cpu(manifest_tlv->type)) {
2914	case SOF_MANIFEST_DATA_TYPE_NHLT:
2915	/* no NHLT in BIOS, so use the one from topology manifest */
2916	if (ipc4_data->nhlt)
2917	break;
2918	ipc4_data->nhlt = devm_kmemdup(dev: sdev->dev, src: manifest_tlv->data,
2919	le32_to_cpu(manifest_tlv->size), GFP_KERNEL);
2920	if (!ipc4_data->nhlt)
2921	return -ENOMEM;
2922	break;
2923	default:
2924	dev_warn(scomp->dev, "Skipping unknown manifest data type %d\n",
2925	manifest_tlv->type);
2926	break;
2927	}
2928	man_ptr += sizeof(struct sof_manifest_tlv) + le32_to_cpu(manifest_tlv->size);
2929	manifest_tlv = (struct sof_manifest_tlv *)man_ptr;
2930	}
2931
2932	return 0;
2933	}
2934
2935	static int sof_ipc4_dai_get_clk(struct snd_sof_dev *sdev, struct snd_sof_dai *dai, int clk_type)
2936	{
2937	struct sof_ipc4_copier *ipc4_copier = dai->private;
2938	struct snd_soc_tplg_hw_config *hw_config;
2939	struct snd_sof_dai_link *slink;
2940	bool dai_link_found = false;
2941	bool hw_cfg_found = false;
2942	int i;
2943
2944	if (!ipc4_copier)
2945	return 0;
2946
2947	list_for_each_entry(slink, &sdev->dai_link_list, list) {
2948	if (!strcmp(slink->link->name, dai->name)) {
2949	dai_link_found = true;
2950	break;
2951	}
2952	}
2953
2954	if (!dai_link_found) {
2955	dev_err(sdev->dev, "no DAI link found for DAI %s\n", dai->name);
2956	return -EINVAL;
2957	}
2958
2959	for (i = 0; i < slink->num_hw_configs; i++) {
2960	hw_config = &slink->hw_configs[i];
2961	if (dai->current_config == le32_to_cpu(hw_config->id)) {
2962	hw_cfg_found = true;
2963	break;
2964	}
2965	}
2966
2967	if (!hw_cfg_found) {
2968	dev_err(sdev->dev, "no matching hw_config found for DAI %s\n", dai->name);
2969	return -EINVAL;
2970	}
2971
2972	switch (ipc4_copier->dai_type) {
2973	case SOF_DAI_INTEL_SSP:
2974	switch (clk_type) {
2975	case SOF_DAI_CLK_INTEL_SSP_MCLK:
2976	return le32_to_cpu(hw_config->mclk_rate);
2977	case SOF_DAI_CLK_INTEL_SSP_BCLK:
2978	return le32_to_cpu(hw_config->bclk_rate);
2979	default:
2980	dev_err(sdev->dev, "Invalid clk type for SSP %d\n", clk_type);
2981	break;
2982	}
2983	break;
2984	default:
2985	dev_err(sdev->dev, "DAI type %d not supported yet!\n", ipc4_copier->dai_type);
2986	break;
2987	}
2988
2989	return -EINVAL;
2990	}
2991
2992	static int sof_ipc4_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verify)
2993	{
2994	struct snd_sof_pcm *spcm;
2995	int dir, ret;
2996
2997	/*
2998	* This function is called during system suspend, we need to make sure
2999	* that all streams have been freed up.
3000	* Freeing might have been skipped when xrun happened just at the start
3001	* of the suspend and it sent a SNDRV_PCM_TRIGGER_STOP to the active
3002	* stream. This will call sof_pcm_stream_free() with
3003	* free_widget_list = false which will leave the kernel and firmware out
3004	* of sync during suspend/resume.
3005	*
3006	* This will also make sure that paused streams handled correctly.
3007	*/
3008	list_for_each_entry(spcm, &sdev->pcm_list, list) {
3009	for_each_pcm_streams(dir) {
3010	struct snd_pcm_substream *substream = spcm->stream[dir].substream;
3011
3012	if (!substream || !substream->runtime || spcm->stream[dir].suspend_ignored)
3013	continue;
3014
3015	if (spcm->stream[dir].list) {
3016	ret = sof_pcm_stream_free(sdev, substream, spcm, dir, free_widget_list: true);
3017	if (ret < 0)
3018	return ret;
3019	}
3020	}
3021	}
3022	return 0;
3023	}
3024
3025	static int sof_ipc4_link_setup(struct snd_sof_dev *sdev, struct snd_soc_dai_link *link)
3026	{
3027	if (link->no_pcm)
3028	return 0;
3029
3030	/*
3031	* set default trigger order for all links. Exceptions to
3032	* the rule will be handled in sof_pcm_dai_link_fixup()
3033	* For playback, the sequence is the following: start BE,
3034	* start FE, stop FE, stop BE; for Capture the sequence is
3035	* inverted start FE, start BE, stop BE, stop FE
3036	*/
3037	link->trigger[SNDRV_PCM_STREAM_PLAYBACK] = SND_SOC_DPCM_TRIGGER_POST;
3038	link->trigger[SNDRV_PCM_STREAM_CAPTURE] = SND_SOC_DPCM_TRIGGER_PRE;
3039
3040	return 0;
3041	}
3042
3043	static enum sof_tokens common_copier_token_list[] = {
3044	SOF_COMP_TOKENS,
3045	SOF_AUDIO_FMT_NUM_TOKENS,
3046	SOF_IN_AUDIO_FORMAT_TOKENS,
3047	SOF_OUT_AUDIO_FORMAT_TOKENS,
3048	SOF_COPIER_DEEP_BUFFER_TOKENS,
3049	SOF_COPIER_TOKENS,
3050	SOF_COMP_EXT_TOKENS,
3051	};
3052
3053	static enum sof_tokens pipeline_token_list[] = {
3054	SOF_SCHED_TOKENS,
3055	SOF_PIPELINE_TOKENS,
3056	};
3057
3058	static enum sof_tokens dai_token_list[] = {
3059	SOF_COMP_TOKENS,
3060	SOF_AUDIO_FMT_NUM_TOKENS,
3061	SOF_IN_AUDIO_FORMAT_TOKENS,
3062	SOF_OUT_AUDIO_FORMAT_TOKENS,
3063	SOF_COPIER_TOKENS,
3064	SOF_DAI_TOKENS,
3065	SOF_COMP_EXT_TOKENS,
3066	};
3067
3068	static enum sof_tokens pga_token_list[] = {
3069	SOF_COMP_TOKENS,
3070	SOF_GAIN_TOKENS,
3071	SOF_AUDIO_FMT_NUM_TOKENS,
3072	SOF_IN_AUDIO_FORMAT_TOKENS,
3073	SOF_OUT_AUDIO_FORMAT_TOKENS,
3074	SOF_COMP_EXT_TOKENS,
3075	};
3076
3077	static enum sof_tokens mixer_token_list[] = {
3078	SOF_COMP_TOKENS,
3079	SOF_AUDIO_FMT_NUM_TOKENS,
3080	SOF_IN_AUDIO_FORMAT_TOKENS,
3081	SOF_OUT_AUDIO_FORMAT_TOKENS,
3082	SOF_COMP_EXT_TOKENS,
3083	};
3084
3085	static enum sof_tokens src_token_list[] = {
3086	SOF_COMP_TOKENS,
3087	SOF_SRC_TOKENS,
3088	SOF_AUDIO_FMT_NUM_TOKENS,
3089	SOF_IN_AUDIO_FORMAT_TOKENS,
3090	SOF_OUT_AUDIO_FORMAT_TOKENS,
3091	SOF_COMP_EXT_TOKENS,
3092	};
3093
3094	static enum sof_tokens process_token_list[] = {
3095	SOF_COMP_TOKENS,
3096	SOF_AUDIO_FMT_NUM_TOKENS,
3097	SOF_IN_AUDIO_FORMAT_TOKENS,
3098	SOF_OUT_AUDIO_FORMAT_TOKENS,
3099	SOF_COMP_EXT_TOKENS,
3100	};
3101
3102	static const struct sof_ipc_tplg_widget_ops tplg_ipc4_widget_ops[SND_SOC_DAPM_TYPE_COUNT] = {
3103	[snd_soc_dapm_aif_in] = {sof_ipc4_widget_setup_pcm, sof_ipc4_widget_free_comp_pcm,
3104	common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3105	NULL, sof_ipc4_prepare_copier_module,
3106	sof_ipc4_unprepare_copier_module},
3107	[snd_soc_dapm_aif_out] = {.ipc_setup: sof_ipc4_widget_setup_pcm, .ipc_free: sof_ipc4_widget_free_comp_pcm,
3108	.token_list: common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3109	NULL, .ipc_prepare: sof_ipc4_prepare_copier_module,
3110	.ipc_unprepare: sof_ipc4_unprepare_copier_module},
3111	[snd_soc_dapm_dai_in] = {.ipc_setup: sof_ipc4_widget_setup_comp_dai, .ipc_free: sof_ipc4_widget_free_comp_dai,
3112	.token_list: dai_token_list, ARRAY_SIZE(dai_token_list), NULL,
3113	.ipc_prepare: sof_ipc4_prepare_copier_module,
3114	.ipc_unprepare: sof_ipc4_unprepare_copier_module},
3115	[snd_soc_dapm_dai_out] = {.ipc_setup: sof_ipc4_widget_setup_comp_dai, .ipc_free: sof_ipc4_widget_free_comp_dai,
3116	.token_list: dai_token_list, ARRAY_SIZE(dai_token_list), NULL,
3117	.ipc_prepare: sof_ipc4_prepare_copier_module,
3118	.ipc_unprepare: sof_ipc4_unprepare_copier_module},
3119	[snd_soc_dapm_buffer] = {.ipc_setup: sof_ipc4_widget_setup_pcm, .ipc_free: sof_ipc4_widget_free_comp_pcm,
3120	.token_list: common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3121	NULL, .ipc_prepare: sof_ipc4_prepare_copier_module,
3122	.ipc_unprepare: sof_ipc4_unprepare_copier_module},
3123	[snd_soc_dapm_scheduler] = {.ipc_setup: sof_ipc4_widget_setup_comp_pipeline,
3124	.ipc_free: sof_ipc4_widget_free_comp_pipeline,
3125	.token_list: pipeline_token_list, ARRAY_SIZE(pipeline_token_list), NULL,
3126	NULL, NULL},
3127	[snd_soc_dapm_pga] = {.ipc_setup: sof_ipc4_widget_setup_comp_pga, .ipc_free: sof_ipc4_widget_free_comp_pga,
3128	.token_list: pga_token_list, ARRAY_SIZE(pga_token_list), NULL,
3129	.ipc_prepare: sof_ipc4_prepare_gain_module,
3130	NULL},
3131	[snd_soc_dapm_mixer] = {.ipc_setup: sof_ipc4_widget_setup_comp_mixer, .ipc_free: sof_ipc4_widget_free_comp_mixer,
3132	.token_list: mixer_token_list, ARRAY_SIZE(mixer_token_list),
3133	NULL, .ipc_prepare: sof_ipc4_prepare_mixer_module,
3134	NULL},
3135	[snd_soc_dapm_src] = {.ipc_setup: sof_ipc4_widget_setup_comp_src, .ipc_free: sof_ipc4_widget_free_comp_src,
3136	.token_list: src_token_list, ARRAY_SIZE(src_token_list),
3137	NULL, .ipc_prepare: sof_ipc4_prepare_src_module,
3138	NULL},
3139	[snd_soc_dapm_effect] = {.ipc_setup: sof_ipc4_widget_setup_comp_process,
3140	.ipc_free: sof_ipc4_widget_free_comp_process,
3141	.token_list: process_token_list, ARRAY_SIZE(process_token_list),
3142	NULL, .ipc_prepare: sof_ipc4_prepare_process_module,
3143	NULL},
3144	};
3145
3146	const struct sof_ipc_tplg_ops ipc4_tplg_ops = {
3147	.widget = tplg_ipc4_widget_ops,
3148	.token_list = ipc4_token_list,
3149	.control_setup = sof_ipc4_control_setup,
3150	.control = &tplg_ipc4_control_ops,
3151	.widget_setup = sof_ipc4_widget_setup,
3152	.widget_free = sof_ipc4_widget_free,
3153	.route_setup = sof_ipc4_route_setup,
3154	.route_free = sof_ipc4_route_free,
3155	.dai_config = sof_ipc4_dai_config,
3156	.parse_manifest = sof_ipc4_parse_manifest,
3157	.dai_get_clk = sof_ipc4_dai_get_clk,
3158	.tear_down_all_pipelines = sof_ipc4_tear_down_all_pipelines,
3159	.link_setup = sof_ipc4_link_setup,
3160	};
3161