1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 *
5 */
6
7#include "i915_drv.h"
8#include "i915_irq.h"
9#include "i915_reg.h"
10#include "intel_crtc.h"
11#include "intel_de.h"
12#include "intel_display_types.h"
13#include "intel_dsb.h"
14#include "intel_dsb_buffer.h"
15#include "intel_dsb_regs.h"
16#include "intel_vblank.h"
17#include "intel_vrr.h"
18#include "skl_watermark.h"
19
20#define CACHELINE_BYTES 64
21
22enum dsb_id {
23 INVALID_DSB = -1,
24 DSB1,
25 DSB2,
26 DSB3,
27 MAX_DSB_PER_PIPE
28};
29
30struct intel_dsb {
31 enum dsb_id id;
32
33 struct intel_dsb_buffer dsb_buf;
34 struct intel_crtc *crtc;
35
36 /*
37 * maximum number of dwords the buffer will hold.
38 */
39 unsigned int size;
40
41 /*
42 * free_pos will point the first free dword and
43 * help in calculating tail of command buffer.
44 */
45 unsigned int free_pos;
46
47 /*
48 * ins_start_offset will help to store start dword of the dsb
49 * instuction and help in identifying the batch of auto-increment
50 * register.
51 */
52 unsigned int ins_start_offset;
53
54 int dewake_scanline;
55};
56
57/**
58 * DOC: DSB
59 *
60 * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory
61 * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA
62 * engine that can be programmed to download the DSB from memory.
63 * It allows driver to batch submit display HW programming. This helps to
64 * reduce loading time and CPU activity, thereby making the context switch
65 * faster. DSB Support added from Gen12 Intel graphics based platform.
66 *
67 * DSB's can access only the pipe, plane, and transcoder Data Island Packet
68 * registers.
69 *
70 * DSB HW can support only register writes (both indexed and direct MMIO
71 * writes). There are no registers reads possible with DSB HW engine.
72 */
73
74/* DSB opcodes. */
75#define DSB_OPCODE_SHIFT 24
76#define DSB_OPCODE_NOOP 0x0
77#define DSB_OPCODE_MMIO_WRITE 0x1
78#define DSB_BYTE_EN 0xf
79#define DSB_BYTE_EN_SHIFT 20
80#define DSB_REG_VALUE_MASK 0xfffff
81#define DSB_OPCODE_WAIT_USEC 0x2
82#define DSB_OPCODE_WAIT_SCANLINE 0x3
83#define DSB_OPCODE_WAIT_VBLANKS 0x4
84#define DSB_OPCODE_WAIT_DSL_IN 0x5
85#define DSB_OPCODE_WAIT_DSL_OUT 0x6
86#define DSB_SCANLINE_UPPER_SHIFT 20
87#define DSB_SCANLINE_LOWER_SHIFT 0
88#define DSB_OPCODE_INTERRUPT 0x7
89#define DSB_OPCODE_INDEXED_WRITE 0x9
90/* see DSB_REG_VALUE_MASK */
91#define DSB_OPCODE_POLL 0xA
92/* see DSB_REG_VALUE_MASK */
93
94static bool assert_dsb_has_room(struct intel_dsb *dsb)
95{
96 struct intel_crtc *crtc = dsb->crtc;
97 struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
98
99 /* each instruction is 2 dwords */
100 return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
101 "[CRTC:%d:%s] DSB %d buffer overflow\n",
102 crtc->base.base.id, crtc->base.name, dsb->id);
103}
104
105static void intel_dsb_dump(struct intel_dsb *dsb)
106{
107 struct intel_crtc *crtc = dsb->crtc;
108 struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
109 int i;
110
111 drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n",
112 crtc->base.base.id, crtc->base.name, dsb->id);
113 for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
114 drm_dbg_kms(&i915->drm,
115 " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
116 intel_dsb_buffer_read(&dsb->dsb_buf, i),
117 intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
118 intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
119 intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
120 drm_dbg_kms(&i915->drm, "}\n");
121}
122
123static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
124 enum dsb_id id)
125{
126 return intel_de_read_fw(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY;
127}
128
129static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
130{
131 if (!assert_dsb_has_room(dsb))
132 return;
133
134 /* Every instruction should be 8 byte aligned. */
135 dsb->free_pos = ALIGN(dsb->free_pos, 2);
136
137 dsb->ins_start_offset = dsb->free_pos;
138
139 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val: ldw);
140 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val: udw);
141}
142
143static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
144 u32 opcode, i915_reg_t reg)
145{
146 u32 prev_opcode, prev_reg;
147
148 /*
149 * Nothing emitted yet? Must check before looking
150 * at the actual data since i915_gem_object_create_internal()
151 * does *not* give you zeroed memory!
152 */
153 if (dsb->free_pos == 0)
154 return false;
155
156 prev_opcode = intel_dsb_buffer_read(dsb_buf: &dsb->dsb_buf,
157 idx: dsb->ins_start_offset + 1) & ~DSB_REG_VALUE_MASK;
158 prev_reg = intel_dsb_buffer_read(dsb_buf: &dsb->dsb_buf,
159 idx: dsb->ins_start_offset + 1) & DSB_REG_VALUE_MASK;
160
161 return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
162}
163
164static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg)
165{
166 /* only full byte-enables can be converted to indexed writes */
167 return intel_dsb_prev_ins_is_write(dsb,
168 DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT |
169 DSB_BYTE_EN << DSB_BYTE_EN_SHIFT,
170 reg);
171}
172
173static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
174{
175 return intel_dsb_prev_ins_is_write(dsb,
176 DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT,
177 reg);
178}
179
180/**
181 * intel_dsb_reg_write() - Emit register wriite to the DSB context
182 * @dsb: DSB context
183 * @reg: register address.
184 * @val: value.
185 *
186 * This function is used for writing register-value pair in command
187 * buffer of DSB.
188 */
189void intel_dsb_reg_write(struct intel_dsb *dsb,
190 i915_reg_t reg, u32 val)
191{
192 u32 old_val;
193
194 /*
195 * For example the buffer will look like below for 3 dwords for auto
196 * increment register:
197 * +--------------------------------------------------------+
198 * | size = 3 | offset &| value1 | value2 | value3 | zero |
199 * | | opcode | | | | |
200 * +--------------------------------------------------------+
201 * + + + + + + +
202 * 0 4 8 12 16 20 24
203 * Byte
204 *
205 * As every instruction is 8 byte aligned the index of dsb instruction
206 * will start always from even number while dealing with u32 array. If
207 * we are writing odd no of dwords, Zeros will be added in the end for
208 * padding.
209 */
210 if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) &&
211 !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) {
212 intel_dsb_emit(dsb, ldw: val,
213 udw: (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
214 (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
215 i915_mmio_reg_offset(reg));
216 } else {
217 if (!assert_dsb_has_room(dsb))
218 return;
219
220 /* convert to indexed write? */
221 if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
222 u32 prev_val = intel_dsb_buffer_read(dsb_buf: &dsb->dsb_buf,
223 idx: dsb->ins_start_offset + 0);
224
225 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf,
226 idx: dsb->ins_start_offset + 0, val: 1); /* count */
227 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->ins_start_offset + 1,
228 val: (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
229 i915_mmio_reg_offset(reg));
230 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->ins_start_offset + 2, val: prev_val);
231
232 dsb->free_pos++;
233 }
234
235 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val);
236 /* Update the count */
237 old_val = intel_dsb_buffer_read(dsb_buf: &dsb->dsb_buf, idx: dsb->ins_start_offset);
238 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->ins_start_offset, val: old_val + 1);
239
240 /* if number of data words is odd, then the last dword should be 0.*/
241 if (dsb->free_pos & 0x1)
242 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos, val: 0);
243 }
244}
245
246static u32 intel_dsb_mask_to_byte_en(u32 mask)
247{
248 return (!!(mask & 0xff000000) << 3 |
249 !!(mask & 0x00ff0000) << 2 |
250 !!(mask & 0x0000ff00) << 1 |
251 !!(mask & 0x000000ff) << 0);
252}
253
254/* Note: mask implemented via byte enables! */
255void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
256 i915_reg_t reg, u32 mask, u32 val)
257{
258 intel_dsb_emit(dsb, ldw: val,
259 udw: (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
260 (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) |
261 i915_mmio_reg_offset(reg));
262}
263
264void intel_dsb_noop(struct intel_dsb *dsb, int count)
265{
266 int i;
267
268 for (i = 0; i < count; i++)
269 intel_dsb_emit(dsb, ldw: 0,
270 DSB_OPCODE_NOOP << DSB_OPCODE_SHIFT);
271}
272
273void intel_dsb_nonpost_start(struct intel_dsb *dsb)
274{
275 struct intel_crtc *crtc = dsb->crtc;
276 enum pipe pipe = crtc->pipe;
277
278 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
279 DSB_NON_POSTED, DSB_NON_POSTED);
280 intel_dsb_noop(dsb, count: 4);
281}
282
283void intel_dsb_nonpost_end(struct intel_dsb *dsb)
284{
285 struct intel_crtc *crtc = dsb->crtc;
286 enum pipe pipe = crtc->pipe;
287
288 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
289 DSB_NON_POSTED, val: 0);
290 intel_dsb_noop(dsb, count: 4);
291}
292
293static void intel_dsb_align_tail(struct intel_dsb *dsb)
294{
295 u32 aligned_tail, tail;
296
297 tail = dsb->free_pos * 4;
298 aligned_tail = ALIGN(tail, CACHELINE_BYTES);
299
300 if (aligned_tail > tail)
301 intel_dsb_buffer_memset(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos, val: 0,
302 size: aligned_tail - tail);
303
304 dsb->free_pos = aligned_tail / 4;
305}
306
307void intel_dsb_finish(struct intel_dsb *dsb)
308{
309 struct intel_crtc *crtc = dsb->crtc;
310
311 /*
312 * DSB_FORCE_DEWAKE remains active even after DSB is
313 * disabled, so make sure to clear it (if set during
314 * intel_dsb_commit()).
315 */
316 intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id),
317 DSB_FORCE_DEWAKE, val: 0);
318
319 intel_dsb_align_tail(dsb);
320
321 intel_dsb_buffer_flush_map(dsb_buf: &dsb->dsb_buf);
322}
323
324static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
325{
326 struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
327 const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
328 unsigned int latency = skl_watermark_max_latency(i915, initial_wm_level: 0);
329 int vblank_start;
330
331 if (crtc_state->vrr.enable) {
332 vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
333 } else {
334 vblank_start = adjusted_mode->crtc_vblank_start;
335
336 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
337 vblank_start = DIV_ROUND_UP(vblank_start, 2);
338 }
339
340 return max(0, vblank_start - intel_usecs_to_scanlines(adjusted_mode, latency));
341}
342
343static u32 dsb_chicken(struct intel_crtc *crtc)
344{
345 if (crtc->mode_flags & I915_MODE_FLAG_VRR)
346 return DSB_CTRL_WAIT_SAFE_WINDOW |
347 DSB_CTRL_NO_WAIT_VBLANK |
348 DSB_INST_WAIT_SAFE_WINDOW |
349 DSB_INST_NO_WAIT_VBLANK;
350 else
351 return 0;
352}
353
354static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
355 int dewake_scanline)
356{
357 struct intel_crtc *crtc = dsb->crtc;
358 struct drm_i915_private *dev_priv = to_i915(dev: crtc->base.dev);
359 enum pipe pipe = crtc->pipe;
360 u32 tail;
361
362 tail = dsb->free_pos * 4;
363 if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
364 return;
365
366 if (is_dsb_busy(i915: dev_priv, pipe, id: dsb->id)) {
367 drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
368 crtc->base.base.id, crtc->base.name, dsb->id);
369 return;
370 }
371
372 intel_de_write_fw(i915: dev_priv, DSB_CTRL(pipe, dsb->id),
373 val: ctrl | DSB_ENABLE);
374
375 intel_de_write_fw(i915: dev_priv, DSB_CHICKEN(pipe, dsb->id),
376 val: dsb_chicken(crtc));
377
378 intel_de_write_fw(i915: dev_priv, DSB_HEAD(pipe, dsb->id),
379 val: intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf));
380
381 if (dewake_scanline >= 0) {
382 int diff, hw_dewake_scanline;
383
384 hw_dewake_scanline = intel_crtc_scanline_to_hw(crtc, scanline: dewake_scanline);
385
386 intel_de_write_fw(i915: dev_priv, DSB_PMCTRL(pipe, dsb->id),
387 DSB_ENABLE_DEWAKE |
388 DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
389
390 /*
391 * Force DEwake immediately if we're already past
392 * or close to racing past the target scanline.
393 */
394 diff = dewake_scanline - intel_get_crtc_scanline(crtc);
395 intel_de_write_fw(i915: dev_priv, DSB_PMCTRL_2(pipe, dsb->id),
396 val: (diff >= 0 && diff < 5 ? DSB_FORCE_DEWAKE : 0) |
397 DSB_BLOCK_DEWAKE_EXTENSION);
398 }
399
400 intel_de_write_fw(i915: dev_priv, DSB_TAIL(pipe, dsb->id),
401 val: intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf) + tail);
402}
403
404/**
405 * intel_dsb_commit() - Trigger workload execution of DSB.
406 * @dsb: DSB context
407 * @wait_for_vblank: wait for vblank before executing
408 *
409 * This function is used to do actual write to hardware using DSB.
410 */
411void intel_dsb_commit(struct intel_dsb *dsb,
412 bool wait_for_vblank)
413{
414 _intel_dsb_commit(dsb,
415 ctrl: wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0,
416 dewake_scanline: wait_for_vblank ? dsb->dewake_scanline : -1);
417}
418
419void intel_dsb_wait(struct intel_dsb *dsb)
420{
421 struct intel_crtc *crtc = dsb->crtc;
422 struct drm_i915_private *dev_priv = to_i915(dev: crtc->base.dev);
423 enum pipe pipe = crtc->pipe;
424
425 if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) {
426 u32 offset = intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf);
427
428 intel_de_write_fw(i915: dev_priv, DSB_CTRL(pipe, dsb->id),
429 DSB_ENABLE | DSB_HALT);
430
431 drm_err(&dev_priv->drm,
432 "[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n",
433 crtc->base.base.id, crtc->base.name, dsb->id,
434 intel_de_read_fw(dev_priv, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
435 intel_de_read_fw(dev_priv, DSB_HEAD(pipe, dsb->id)) - offset,
436 intel_de_read_fw(dev_priv, DSB_TAIL(pipe, dsb->id)) - offset);
437
438 intel_dsb_dump(dsb);
439 }
440
441 /* Attempt to reset it */
442 dsb->free_pos = 0;
443 dsb->ins_start_offset = 0;
444 intel_de_write_fw(i915: dev_priv, DSB_CTRL(pipe, dsb->id), val: 0);
445}
446
447/**
448 * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
449 * @crtc_state: the CRTC state
450 * @max_cmds: number of commands we need to fit into command buffer
451 *
452 * This function prepare the command buffer which is used to store dsb
453 * instructions with data.
454 *
455 * Returns:
456 * DSB context, NULL on failure
457 */
458struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
459 unsigned int max_cmds)
460{
461 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
462 struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
463 intel_wakeref_t wakeref;
464 struct intel_dsb *dsb;
465 unsigned int size;
466
467 if (!HAS_DSB(i915))
468 return NULL;
469
470 /* TODO: DSB is broken in Xe KMD, so disabling it until fixed */
471 if (!IS_ENABLED(I915))
472 return NULL;
473
474 dsb = kzalloc(size: sizeof(*dsb), GFP_KERNEL);
475 if (!dsb)
476 goto out;
477
478 wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm);
479
480 /* ~1 qword per instruction, full cachelines */
481 size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
482
483 if (!intel_dsb_buffer_create(crtc, dsb_buf: &dsb->dsb_buf, size))
484 goto out_put_rpm;
485
486 intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref);
487
488 dsb->id = DSB1;
489 dsb->crtc = crtc;
490 dsb->size = size / 4; /* in dwords */
491 dsb->free_pos = 0;
492 dsb->ins_start_offset = 0;
493 dsb->dewake_scanline = intel_dsb_dewake_scanline(crtc_state);
494
495 return dsb;
496
497out_put_rpm:
498 intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref);
499 kfree(objp: dsb);
500out:
501 drm_info_once(&i915->drm,
502 "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
503 crtc->base.base.id, crtc->base.name, DSB1);
504
505 return NULL;
506}
507
508/**
509 * intel_dsb_cleanup() - To cleanup DSB context.
510 * @dsb: DSB context
511 *
512 * This function cleanup the DSB context by unpinning and releasing
513 * the VMA object associated with it.
514 */
515void intel_dsb_cleanup(struct intel_dsb *dsb)
516{
517 intel_dsb_buffer_cleanup(dsb_buf: &dsb->dsb_buf);
518 kfree(objp: dsb);
519}
520

source code of linux/drivers/gpu/drm/i915/display/intel_dsb.c