1// SPDX-License-Identifier: GPL-2.0-only
2//
3// Copyright(c) 2021-2022 Intel Corporation
4//
5// Authors: Cezary Rojewski <cezary.rojewski@intel.com>
6// Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
7//
8
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/slab.h>
11#include <sound/hdaudio_ext.h>
12#include "avs.h"
13#include "debug.h"
14#include "messages.h"
15#include "registers.h"
16#include "trace.h"
17
18#define AVS_IPC_TIMEOUT_MS 300
19#define AVS_D0IX_DELAY_MS 300
20
21static int
22avs_dsp_set_d0ix(struct avs_dev *adev, bool enable)
23{
24 struct avs_ipc *ipc = adev->ipc;
25 int ret;
26
27 /* Is transition required? */
28 if (ipc->in_d0ix == enable)
29 return 0;
30
31 ret = avs_dsp_op(adev, set_d0ix, enable);
32 if (ret) {
33 /* Prevent further d0ix attempts on conscious IPC failure. */
34 if (ret == -AVS_EIPC)
35 atomic_inc(v: &ipc->d0ix_disable_depth);
36
37 ipc->in_d0ix = false;
38 return ret;
39 }
40
41 ipc->in_d0ix = enable;
42 return 0;
43}
44
45static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx)
46{
47 if (atomic_read(v: &adev->ipc->d0ix_disable_depth))
48 return;
49
50 mod_delayed_work(wq: system_power_efficient_wq, dwork: &adev->ipc->d0ix_work,
51 delay: msecs_to_jiffies(AVS_D0IX_DELAY_MS));
52}
53
54static void avs_dsp_d0ix_work(struct work_struct *work)
55{
56 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work);
57
58 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), enable: true);
59}
60
61static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx)
62{
63 struct avs_ipc *ipc = adev->ipc;
64
65 if (!atomic_read(v: &ipc->d0ix_disable_depth)) {
66 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
67 return avs_dsp_set_d0ix(adev, enable: false);
68 }
69
70 return 0;
71}
72
73int avs_dsp_disable_d0ix(struct avs_dev *adev)
74{
75 struct avs_ipc *ipc = adev->ipc;
76
77 /* Prevent PG only on the first disable. */
78 if (atomic_inc_return(v: &ipc->d0ix_disable_depth) == 1) {
79 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
80 return avs_dsp_set_d0ix(adev, enable: false);
81 }
82
83 return 0;
84}
85
86int avs_dsp_enable_d0ix(struct avs_dev *adev)
87{
88 struct avs_ipc *ipc = adev->ipc;
89
90 if (atomic_dec_and_test(v: &ipc->d0ix_disable_depth))
91 queue_delayed_work(wq: system_power_efficient_wq, dwork: &ipc->d0ix_work,
92 delay: msecs_to_jiffies(AVS_D0IX_DELAY_MS));
93 return 0;
94}
95
96static void avs_dsp_recovery(struct avs_dev *adev)
97{
98 struct avs_soc_component *acomp;
99 unsigned int core_mask;
100 int ret;
101
102 mutex_lock(&adev->comp_list_mutex);
103 /* disconnect all running streams */
104 list_for_each_entry(acomp, &adev->comp_list, node) {
105 struct snd_soc_pcm_runtime *rtd;
106 struct snd_soc_card *card;
107
108 card = acomp->base.card;
109 if (!card)
110 continue;
111
112 for_each_card_rtds(card, rtd) {
113 struct snd_pcm *pcm;
114 int dir;
115
116 pcm = rtd->pcm;
117 if (!pcm || rtd->dai_link->no_pcm)
118 continue;
119
120 for_each_pcm_streams(dir) {
121 struct snd_pcm_substream *substream;
122
123 substream = pcm->streams[dir].substream;
124 if (!substream || !substream->runtime)
125 continue;
126
127 /* No need for _irq() as we are in nonatomic context. */
128 snd_pcm_stream_lock(substream);
129 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
130 snd_pcm_stream_unlock(substream);
131 }
132 }
133 }
134 mutex_unlock(lock: &adev->comp_list_mutex);
135
136 /* forcibly shutdown all cores */
137 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);
138 avs_dsp_core_disable(adev, core_mask);
139
140 /* attempt dsp reboot */
141 ret = avs_dsp_boot_firmware(adev, purge: true);
142 if (ret < 0)
143 dev_err(adev->dev, "dsp reboot failed: %d\n", ret);
144
145 pm_runtime_enable(dev: adev->dev);
146 pm_request_autosuspend(dev: adev->dev);
147
148 atomic_set(v: &adev->ipc->recovering, i: 0);
149}
150
151static void avs_dsp_recovery_work(struct work_struct *work)
152{
153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work);
154
155 avs_dsp_recovery(to_avs_dev(ipc->dev));
156}
157
158static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg)
159{
160 struct avs_ipc *ipc = adev->ipc;
161
162 /* Account for the double-exception case. */
163 ipc->ready = false;
164
165 if (!atomic_add_unless(v: &ipc->recovering, a: 1, u: 1)) {
166 dev_err(adev->dev, "dsp recovery is already in progress\n");
167 return;
168 }
169
170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n");
171
172 /* Avoid deadlock as the exception may be the response to SET_D0IX. */
173 if (current_work() != &ipc->d0ix_work.work)
174 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
175 ipc->in_d0ix = false;
176 /* Re-enabled on recovery completion. */
177 pm_runtime_disable(dev: adev->dev);
178
179 /* Process received notification. */
180 avs_dsp_op(adev, coredump, msg);
181
182 schedule_work(work: &ipc->recovery_work);
183}
184
185static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header)
186{
187 struct avs_ipc *ipc = adev->ipc;
188 union avs_reply_msg msg = AVS_MSG(header);
189 u32 sts, lec;
190
191 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev));
192 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev));
193 trace_avs_ipc_reply_msg(header, sts, lec);
194
195 ipc->rx.header = header;
196 /* Abort copying payload if request processing was unsuccessful. */
197 if (!msg.status) {
198 /* update size in case of LARGE_CONFIG_GET */
199 if (msg.msg_target == AVS_MOD_MSG &&
200 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET)
201 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE,
202 msg.ext.large_config.data_off_size);
203
204 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size);
205 trace_avs_msg_payload(data: ipc->rx.data, size: ipc->rx.size);
206 }
207}
208
209static void avs_dsp_process_notification(struct avs_dev *adev, u64 header)
210{
211 struct avs_notify_mod_data mod_data;
212 union avs_notify_msg msg = AVS_MSG(header);
213 size_t data_size = 0;
214 void *data = NULL;
215 u32 sts, lec;
216
217 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev));
218 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev));
219 trace_avs_ipc_notify_msg(header, sts, lec);
220
221 /* Ignore spurious notifications until handshake is established. */
222 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) {
223 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary);
224 return;
225 }
226
227 /* Calculate notification payload size. */
228 switch (msg.notify_msg_type) {
229 case AVS_NOTIFY_FW_READY:
230 break;
231
232 case AVS_NOTIFY_PHRASE_DETECTED:
233 data_size = sizeof(struct avs_notify_voice_data);
234 break;
235
236 case AVS_NOTIFY_RESOURCE_EVENT:
237 data_size = sizeof(struct avs_notify_res_data);
238 break;
239
240 case AVS_NOTIFY_LOG_BUFFER_STATUS:
241 case AVS_NOTIFY_EXCEPTION_CAUGHT:
242 break;
243
244 case AVS_NOTIFY_MODULE_EVENT:
245 /* To know the total payload size, header needs to be read first. */
246 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data));
247 data_size = sizeof(mod_data) + mod_data.data_size;
248 break;
249
250 default:
251 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary);
252 break;
253 }
254
255 if (data_size) {
256 data = kmalloc(data_size, GFP_KERNEL);
257 if (!data)
258 return;
259
260 memcpy_fromio(data, avs_uplink_addr(adev), data_size);
261 trace_avs_msg_payload(data, size: data_size);
262 }
263
264 /* Perform notification-specific operations. */
265 switch (msg.notify_msg_type) {
266 case AVS_NOTIFY_FW_READY:
267 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary);
268 adev->ipc->ready = true;
269 complete(&adev->fw_ready);
270 break;
271
272 case AVS_NOTIFY_LOG_BUFFER_STATUS:
273 avs_log_buffer_status_locked(adev, msg: &msg);
274 break;
275
276 case AVS_NOTIFY_EXCEPTION_CAUGHT:
277 avs_dsp_exception_caught(adev, msg: &msg);
278 break;
279
280 default:
281 break;
282 }
283
284 kfree(objp: data);
285}
286
287void avs_dsp_process_response(struct avs_dev *adev, u64 header)
288{
289 struct avs_ipc *ipc = adev->ipc;
290
291 /*
292 * Response may either be solicited - a reply for a request that has
293 * been sent beforehand - or unsolicited (notification).
294 */
295 if (avs_msg_is_reply(header)) {
296 /* Response processing is invoked from IRQ thread. */
297 spin_lock_irq(lock: &ipc->rx_lock);
298 avs_dsp_receive_rx(adev, header);
299 ipc->rx_completed = true;
300 spin_unlock_irq(lock: &ipc->rx_lock);
301 } else {
302 avs_dsp_process_notification(adev, header);
303 }
304
305 complete(&ipc->busy_completion);
306}
307
308static bool avs_ipc_is_busy(struct avs_ipc *ipc)
309{
310 struct avs_dev *adev = to_avs_dev(ipc->dev);
311 const struct avs_spec *const spec = adev->spec;
312 u32 hipc_rsp;
313
314 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
315 return hipc_rsp & spec->hipc->rsp_busy_mask;
316}
317
318static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout)
319{
320 u32 repeats_left = 128; /* to avoid infinite looping */
321 int ret;
322
323again:
324 ret = wait_for_completion_timeout(x: &ipc->busy_completion, timeout: msecs_to_jiffies(m: timeout));
325
326 /* DSP could be unresponsive at this point. */
327 if (!ipc->ready)
328 return -EPERM;
329
330 if (!ret) {
331 if (!avs_ipc_is_busy(ipc))
332 return -ETIMEDOUT;
333 /*
334 * Firmware did its job, either notification or reply
335 * has been received - now wait until it's processed.
336 */
337 wait_for_completion_killable(x: &ipc->busy_completion);
338 }
339
340 /* Ongoing notification's bottom-half may cause early wakeup */
341 spin_lock(lock: &ipc->rx_lock);
342 if (!ipc->rx_completed) {
343 if (repeats_left) {
344 /* Reply delayed due to notification. */
345 repeats_left--;
346 reinit_completion(x: &ipc->busy_completion);
347 spin_unlock(lock: &ipc->rx_lock);
348 goto again;
349 }
350
351 spin_unlock(lock: &ipc->rx_lock);
352 return -ETIMEDOUT;
353 }
354
355 spin_unlock(lock: &ipc->rx_lock);
356 return 0;
357}
358
359static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply)
360{
361 lockdep_assert_held(&ipc->rx_lock);
362
363 ipc->rx.header = 0;
364 ipc->rx.size = reply ? reply->size : 0;
365 ipc->rx_completed = false;
366
367 reinit_completion(x: &ipc->done_completion);
368 reinit_completion(x: &ipc->busy_completion);
369}
370
371static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs)
372{
373 const struct avs_spec *const spec = adev->spec;
374 u32 sts = UINT_MAX;
375 u32 lec = UINT_MAX;
376
377 tx->header |= spec->hipc->req_busy_mask;
378 if (read_fwregs) {
379 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev));
380 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev));
381 }
382
383 trace_avs_request(tx, sts, lec);
384
385 if (tx->size)
386 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size);
387 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32);
388 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX);
389}
390
391static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
392 struct avs_ipc_msg *reply, int timeout, const char *name)
393{
394 struct avs_ipc *ipc = adev->ipc;
395 int ret;
396
397 if (!ipc->ready)
398 return -EPERM;
399
400 mutex_lock(&ipc->msg_mutex);
401
402 spin_lock(lock: &ipc->rx_lock);
403 avs_ipc_msg_init(ipc, reply);
404 avs_dsp_send_tx(adev, tx: request, read_fwregs: true);
405 spin_unlock(lock: &ipc->rx_lock);
406
407 ret = avs_ipc_wait_busy_completion(ipc, timeout);
408 if (ret) {
409 if (ret == -ETIMEDOUT) {
410 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT);
411
412 /* Same treatment as on exception, just stack_dump=0. */
413 avs_dsp_exception_caught(adev, msg: &msg);
414 }
415 goto exit;
416 }
417
418 ret = ipc->rx.rsp.status;
419 /*
420 * If IPC channel is blocked e.g.: due to ongoing recovery,
421 * -EPERM error code is expected and thus it's not an actual error.
422 *
423 * Unsupported IPCs are of no harm either.
424 */
425 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED)
426 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
427 name, request->glb.primary, request->glb.ext.val, ret);
428 else if (ret)
429 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
430 name, request->glb.primary, request->glb.ext.val, ret);
431
432 if (reply) {
433 reply->header = ipc->rx.header;
434 reply->size = ipc->rx.size;
435 if (reply->data && ipc->rx.size)
436 memcpy(reply->data, ipc->rx.data, reply->size);
437 }
438
439exit:
440 mutex_unlock(lock: &ipc->msg_mutex);
441 return ret;
442}
443
444static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request,
445 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
446 bool schedule_d0ix, const char *name)
447{
448 int ret;
449
450 trace_avs_d0ix(op: "wake", proceed: wake_d0i0, header: request->header);
451 if (wake_d0i0) {
452 ret = avs_dsp_wake_d0i0(adev, tx: request);
453 if (ret)
454 return ret;
455 }
456
457 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name);
458 if (ret)
459 return ret;
460
461 trace_avs_d0ix(op: "schedule", proceed: schedule_d0ix, header: request->header);
462 if (schedule_d0ix)
463 avs_dsp_schedule_d0ix(adev, tx: request);
464
465 return 0;
466}
467
468int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
469 struct avs_ipc_msg *reply, int timeout, const char *name)
470{
471 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true);
472 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false);
473
474 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix,
475 name);
476}
477
478int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
479 struct avs_ipc_msg *reply, const char *name)
480{
481 return avs_dsp_send_msg_timeout(adev, request, reply, timeout: adev->ipc->default_timeout_ms, name);
482}
483
484int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
485 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
486 const char *name)
487{
488 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix: false, name);
489}
490
491int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
492 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name)
493{
494 return avs_dsp_send_pm_msg_timeout(adev, request, reply, timeout: adev->ipc->default_timeout_ms,
495 wake_d0i0, name);
496}
497
498static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
499 const char *name)
500{
501 struct avs_ipc *ipc = adev->ipc;
502 int ret;
503
504 mutex_lock(&ipc->msg_mutex);
505
506 spin_lock(lock: &ipc->rx_lock);
507 avs_ipc_msg_init(ipc, NULL);
508 /*
509 * with hw still stalled, memory windows may not be
510 * configured properly so avoid accessing SRAM
511 */
512 avs_dsp_send_tx(adev, tx: request, read_fwregs: false);
513 spin_unlock(lock: &ipc->rx_lock);
514
515 /* ROM messages must be sent before main core is unstalled */
516 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false);
517 if (!ret) {
518 ret = wait_for_completion_timeout(x: &ipc->done_completion, timeout: msecs_to_jiffies(m: timeout));
519 ret = ret ? 0 : -ETIMEDOUT;
520 }
521 if (ret)
522 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
523 name, request->glb.primary, request->glb.ext.val, ret);
524
525 mutex_unlock(lock: &ipc->msg_mutex);
526
527 return ret;
528}
529
530int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
531 const char *name)
532{
533 return avs_dsp_do_send_rom_msg(adev, request, timeout, name);
534}
535
536int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name)
537{
538 return avs_dsp_send_rom_msg_timeout(adev, request, timeout: adev->ipc->default_timeout_ms, name);
539}
540
541void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable)
542{
543 const struct avs_spec *const spec = adev->spec;
544 u32 value, mask;
545
546 /*
547 * No particular bit setting order. All of these are required
548 * to have a functional SW <-> FW communication.
549 */
550 value = enable ? AVS_ADSP_ADSPIC_IPC : 0;
551 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value);
552
553 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY;
554 value = enable ? mask : 0;
555 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value);
556}
557
558int avs_ipc_init(struct avs_ipc *ipc, struct device *dev)
559{
560 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL);
561 if (!ipc->rx.data)
562 return -ENOMEM;
563
564 ipc->dev = dev;
565 ipc->ready = false;
566 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS;
567 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work);
568 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work);
569 init_completion(x: &ipc->done_completion);
570 init_completion(x: &ipc->busy_completion);
571 spin_lock_init(&ipc->rx_lock);
572 mutex_init(&ipc->msg_mutex);
573
574 return 0;
575}
576
577void avs_ipc_block(struct avs_ipc *ipc)
578{
579 ipc->ready = false;
580 cancel_work_sync(work: &ipc->recovery_work);
581 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
582 ipc->in_d0ix = false;
583}
584

source code of linux/sound/soc/intel/avs/ipc.c