1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include <drm/drm_blend.h>
7
8	#include "i915_drv.h"
9	#include "i915_fixed.h"
10	#include "i915_reg.h"
11	#include "i9xx_wm.h"
12	#include "intel_atomic.h"
13	#include "intel_atomic_plane.h"
14	#include "intel_bw.h"
15	#include "intel_crtc.h"
16	#include "intel_de.h"
17	#include "intel_display.h"
18	#include "intel_display_power.h"
19	#include "intel_display_types.h"
20	#include "intel_fb.h"
21	#include "intel_pcode.h"
22	#include "intel_wm.h"
23	#include "skl_watermark.h"
24	#include "skl_watermark_regs.h"
25
26	/*It is expected that DSB can do posted writes to every register in
27	* the pipe and planes within 100us. For flip queue use case, the
28	* recommended DSB execution time is 100us + one SAGV block time.
29	*/
30	#define DSB_EXE_TIME 100
31
32	static void skl_sagv_disable(struct drm_i915_private *i915);
33
34	/* Stores plane specific WM parameters */
35	struct skl_wm_params {
36	bool x_tiled, y_tiled;
37	bool rc_surface;
38	bool is_planar;
39	u32 width;
40	u8 cpp;
41	u32 plane_pixel_rate;
42	u32 y_min_scanlines;
43	u32 plane_bytes_per_line;
44	uint_fixed_16_16_t plane_blocks_per_line;
45	uint_fixed_16_16_t y_tile_minimum;
46	u32 linetime_us;
47	u32 dbuf_block_size;
48	};
49
50	u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
51	{
52	u8 enabled_slices = 0;
53	enum dbuf_slice slice;
54
55	for_each_dbuf_slice(i915, slice) {
56	if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
57	enabled_slices |= BIT(slice);
58	}
59
60	return enabled_slices;
61	}
62
63	/*
64	* FIXME: We still don't have the proper code detect if we need to apply the WA,
65	* so assume we'll always need it in order to avoid underruns.
66	*/
67	static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
68	{
69	return DISPLAY_VER(i915) == 9;
70	}
71
72	static bool
73	intel_has_sagv(struct drm_i915_private *i915)
74	{
75	return HAS_SAGV(i915) &&
76	i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
77	}
78
79	static u32
80	intel_sagv_block_time(struct drm_i915_private *i915)
81	{
82	if (DISPLAY_VER(i915) >= 14) {
83	u32 val;
84
85	val = intel_de_read(i915, MTL_LATENCY_SAGV);
86
87	return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
88	} else if (DISPLAY_VER(i915) >= 12) {
89	u32 val = 0;
90	int ret;
91
92	ret = snb_pcode_read(uncore: &i915->uncore,
93	GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
94	val: &val, NULL);
95	if (ret) {
96	drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
97	return 0;
98	}
99
100	return val;
101	} else if (DISPLAY_VER(i915) == 11) {
102	return 10;
103	} else if (HAS_SAGV(i915)) {
104	return 30;
105	} else {
106	return 0;
107	}
108	}
109
110	static void intel_sagv_init(struct drm_i915_private *i915)
111	{
112	if (!HAS_SAGV(i915))
113	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
114
115	/*
116	* Probe to see if we have working SAGV control.
117	* For icl+ this was already determined by intel_bw_init_hw().
118	*/
119	if (DISPLAY_VER(i915) < 11)
120	skl_sagv_disable(i915);
121
122	drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
123
124	i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
125
126	drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
127	str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
128
129	/* avoid overflow when adding with wm0 latency/etc. */
130	if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
131	"Excessive SAGV block time %u, ignoring\n",
132	i915->display.sagv.block_time_us))
133	i915->display.sagv.block_time_us = 0;
134
135	if (!intel_has_sagv(i915))
136	i915->display.sagv.block_time_us = 0;
137	}
138
139	/*
140	* SAGV dynamically adjusts the system agent voltage and clock frequencies
141	* depending on power and performance requirements. The display engine access
142	* to system memory is blocked during the adjustment time. Because of the
143	* blocking time, having this enabled can cause full system hangs and/or pipe
144	* underruns if we don't meet all of the following requirements:
145	*
146	* - <= 1 pipe enabled
147	* - All planes can enable watermarks for latencies >= SAGV engine block time
148	* - We're not using an interlaced display configuration
149	*/
150	static void skl_sagv_enable(struct drm_i915_private *i915)
151	{
152	int ret;
153
154	if (!intel_has_sagv(i915))
155	return;
156
157	if (i915->display.sagv.status == I915_SAGV_ENABLED)
158	return;
159
160	drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
161	ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
162	GEN9_SAGV_ENABLE);
163
164	/* We don't need to wait for SAGV when enabling */
165
166	/*
167	* Some skl systems, pre-release machines in particular,
168	* don't actually have SAGV.
169	*/
170	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
171	drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
172	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
173	return;
174	} else if (ret < 0) {
175	drm_err(&i915->drm, "Failed to enable SAGV\n");
176	return;
177	}
178
179	i915->display.sagv.status = I915_SAGV_ENABLED;
180	}
181
182	static void skl_sagv_disable(struct drm_i915_private *i915)
183	{
184	int ret;
185
186	if (!intel_has_sagv(i915))
187	return;
188
189	if (i915->display.sagv.status == I915_SAGV_DISABLED)
190	return;
191
192	drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
193	/* bspec says to keep retrying for at least 1 ms */
194	ret = skl_pcode_request(uncore: &i915->uncore, GEN9_PCODE_SAGV_CONTROL,
195	GEN9_SAGV_DISABLE,
196	GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
197	timeout_base_ms: 1);
198	/*
199	* Some skl systems, pre-release machines in particular,
200	* don't actually have SAGV.
201	*/
202	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
203	drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
204	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
205	return;
206	} else if (ret < 0) {
207	drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
208	return;
209	}
210
211	i915->display.sagv.status = I915_SAGV_DISABLED;
212	}
213
214	static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
215	{
216	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
217	const struct intel_bw_state *new_bw_state =
218	intel_atomic_get_new_bw_state(state);
219
220	if (!new_bw_state)
221	return;
222
223	if (!intel_can_enable_sagv(i915, bw_state: new_bw_state))
224	skl_sagv_disable(i915);
225	}
226
227	static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
228	{
229	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
230	const struct intel_bw_state *new_bw_state =
231	intel_atomic_get_new_bw_state(state);
232
233	if (!new_bw_state)
234	return;
235
236	if (intel_can_enable_sagv(i915, bw_state: new_bw_state))
237	skl_sagv_enable(i915);
238	}
239
240	static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
241	{
242	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
243	const struct intel_bw_state *old_bw_state =
244	intel_atomic_get_old_bw_state(state);
245	const struct intel_bw_state *new_bw_state =
246	intel_atomic_get_new_bw_state(state);
247	u16 old_mask, new_mask;
248
249	if (!new_bw_state)
250	return;
251
252	old_mask = old_bw_state->qgv_points_mask;
253	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
254
255	if (old_mask == new_mask)
256	return;
257
258	WARN_ON(!new_bw_state->base.changed);
259
260	drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
261	old_mask, new_mask);
262
263	/*
264	* Restrict required qgv points before updating the configuration.
265	* According to BSpec we can't mask and unmask qgv points at the same
266	* time. Also masking should be done before updating the configuration
267	* and unmasking afterwards.
268	*/
269	icl_pcode_restrict_qgv_points(dev_priv: i915, points_mask: new_mask);
270	}
271
272	static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
273	{
274	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
275	const struct intel_bw_state *old_bw_state =
276	intel_atomic_get_old_bw_state(state);
277	const struct intel_bw_state *new_bw_state =
278	intel_atomic_get_new_bw_state(state);
279	u16 old_mask, new_mask;
280
281	if (!new_bw_state)
282	return;
283
284	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
285	new_mask = new_bw_state->qgv_points_mask;
286
287	if (old_mask == new_mask)
288	return;
289
290	WARN_ON(!new_bw_state->base.changed);
291
292	drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
293	old_mask, new_mask);
294
295	/*
296	* Allow required qgv points after updating the configuration.
297	* According to BSpec we can't mask and unmask qgv points at the same
298	* time. Also masking should be done before updating the configuration
299	* and unmasking afterwards.
300	*/
301	icl_pcode_restrict_qgv_points(dev_priv: i915, points_mask: new_mask);
302	}
303
304	void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
305	{
306	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
307
308	/*
309	* Just return if we can't control SAGV or don't have it.
310	* This is different from situation when we have SAGV but just can't
311	* afford it due to DBuf limitation - in case if SAGV is completely
312	* disabled in a BIOS, we are not even allowed to send a PCode request,
313	* as it will throw an error. So have to check it here.
314	*/
315	if (!intel_has_sagv(i915))
316	return;
317
318	if (DISPLAY_VER(i915) >= 11)
319	icl_sagv_pre_plane_update(state);
320	else
321	skl_sagv_pre_plane_update(state);
322	}
323
324	void intel_sagv_post_plane_update(struct intel_atomic_state *state)
325	{
326	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
327
328	/*
329	* Just return if we can't control SAGV or don't have it.
330	* This is different from situation when we have SAGV but just can't
331	* afford it due to DBuf limitation - in case if SAGV is completely
332	* disabled in a BIOS, we are not even allowed to send a PCode request,
333	* as it will throw an error. So have to check it here.
334	*/
335	if (!intel_has_sagv(i915))
336	return;
337
338	if (DISPLAY_VER(i915) >= 11)
339	icl_sagv_post_plane_update(state);
340	else
341	skl_sagv_post_plane_update(state);
342	}
343
344	static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
345	{
346	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
347	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
348	enum plane_id plane_id;
349	int max_level = INT_MAX;
350
351	if (!intel_has_sagv(i915))
352	return false;
353
354	if (!crtc_state->hw.active)
355	return true;
356
357	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
358	return false;
359
360	for_each_plane_id_on_crtc(crtc, plane_id) {
361	const struct skl_plane_wm *wm =
362	&crtc_state->wm.skl.optimal.planes[plane_id];
363	int level;
364
365	/* Skip this plane if it's not enabled */
366	if (!wm->wm[0].enable)
367	continue;
368
369	/* Find the highest enabled wm level for this plane */
370	for (level = i915->display.wm.num_levels - 1;
371	!wm->wm[level].enable; --level)
372	{ }
373
374	/* Highest common enabled wm level for all planes */
375	max_level = min(level, max_level);
376	}
377
378	/* No enabled planes? */
379	if (max_level == INT_MAX)
380	return true;
381
382	for_each_plane_id_on_crtc(crtc, plane_id) {
383	const struct skl_plane_wm *wm =
384	&crtc_state->wm.skl.optimal.planes[plane_id];
385
386	/*
387	* All enabled planes must have enabled a common wm level that
388	* can tolerate memory latencies higher than sagv_block_time_us
389	*/
390	if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
391	return false;
392	}
393
394	return true;
395	}
396
397	static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
398	{
399	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
400	enum plane_id plane_id;
401
402	if (!crtc_state->hw.active)
403	return true;
404
405	for_each_plane_id_on_crtc(crtc, plane_id) {
406	const struct skl_plane_wm *wm =
407	&crtc_state->wm.skl.optimal.planes[plane_id];
408
409	if (wm->wm[0].enable && !wm->sagv.wm0.enable)
410	return false;
411	}
412
413	return true;
414	}
415
416	static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
417	{
418	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
419	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
420
421	if (!i915->display.params.enable_sagv)
422	return false;
423
424	if (DISPLAY_VER(i915) >= 12)
425	return tgl_crtc_can_enable_sagv(crtc_state);
426	else
427	return skl_crtc_can_enable_sagv(crtc_state);
428	}
429
430	bool intel_can_enable_sagv(struct drm_i915_private *i915,
431	const struct intel_bw_state *bw_state)
432	{
433	if (DISPLAY_VER(i915) < 11 &&
434	bw_state->active_pipes && !is_power_of_2(n: bw_state->active_pipes))
435	return false;
436
437	return bw_state->pipe_sagv_reject == 0;
438	}
439
440	static int intel_compute_sagv_mask(struct intel_atomic_state *state)
441	{
442	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
443	int ret;
444	struct intel_crtc *crtc;
445	struct intel_crtc_state *new_crtc_state;
446	struct intel_bw_state *new_bw_state = NULL;
447	const struct intel_bw_state *old_bw_state = NULL;
448	int i;
449
450	for_each_new_intel_crtc_in_state(state, crtc,
451	new_crtc_state, i) {
452	struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
453
454	new_bw_state = intel_atomic_get_bw_state(state);
455	if (IS_ERR(ptr: new_bw_state))
456	return PTR_ERR(ptr: new_bw_state);
457
458	old_bw_state = intel_atomic_get_old_bw_state(state);
459
460	/*
461	* We store use_sagv_wm in the crtc state rather than relying on
462	* that bw state since we have no convenient way to get at the
463	* latter from the plane commit hooks (especially in the legacy
464	* cursor case).
465	*
466	* drm_atomic_check_only() gets upset if we pull more crtcs
467	* into the state, so we have to calculate this based on the
468	* individual intel_crtc_can_enable_sagv() rather than
469	* the overall intel_can_enable_sagv(). Otherwise the
470	* crtcs not included in the commit would not switch to the
471	* SAGV watermarks when we are about to enable SAGV, and that
472	* would lead to underruns. This does mean extra power draw
473	* when only a subset of the crtcs are blocking SAGV as the
474	* other crtcs can't be allowed to use the more optimal
475	* normal (ie. non-SAGV) watermarks.
476	*/
477	pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
478	DISPLAY_VER(i915) >= 12 &&
479	intel_crtc_can_enable_sagv(crtc_state: new_crtc_state);
480
481	if (intel_crtc_can_enable_sagv(crtc_state: new_crtc_state))
482	new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
483	else
484	new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
485	}
486
487	if (!new_bw_state)
488	return 0;
489
490	new_bw_state->active_pipes =
491	intel_calc_active_pipes(state, active_pipes: old_bw_state->active_pipes);
492
493	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
494	ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
495	if (ret)
496	return ret;
497	}
498
499	if (intel_can_enable_sagv(i915, bw_state: new_bw_state) !=
500	intel_can_enable_sagv(i915, bw_state: old_bw_state)) {
501	ret = intel_atomic_serialize_global_state(obj_state: &new_bw_state->base);
502	if (ret)
503	return ret;
504	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
505	ret = intel_atomic_lock_global_state(obj_state: &new_bw_state->base);
506	if (ret)
507	return ret;
508	}
509
510	return 0;
511	}
512
513	static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
514	u16 start, u16 end)
515	{
516	entry->start = start;
517	entry->end = end;
518
519	return end;
520	}
521
522	static int intel_dbuf_slice_size(struct drm_i915_private *i915)
523	{
524	return DISPLAY_INFO(i915)->dbuf.size /
525	hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
526	}
527
528	static void
529	skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
530	struct skl_ddb_entry *ddb)
531	{
532	int slice_size = intel_dbuf_slice_size(i915);
533
534	if (!slice_mask) {
535	ddb->start = 0;
536	ddb->end = 0;
537	return;
538	}
539
540	ddb->start = (ffs(slice_mask) - 1) * slice_size;
541	ddb->end = fls(x: slice_mask) * slice_size;
542
543	WARN_ON(ddb->start >= ddb->end);
544	WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
545	}
546
547	static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
548	{
549	struct skl_ddb_entry ddb;
550
551	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
552	slice_mask = BIT(DBUF_S1);
553	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
554	slice_mask = BIT(DBUF_S3);
555
556	skl_ddb_entry_for_slices(i915, slice_mask, ddb: &ddb);
557
558	return ddb.start;
559	}
560
561	u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
562	const struct skl_ddb_entry *entry)
563	{
564	int slice_size = intel_dbuf_slice_size(i915);
565	enum dbuf_slice start_slice, end_slice;
566	u8 slice_mask = 0;
567
568	if (!skl_ddb_entry_size(entry))
569	return 0;
570
571	start_slice = entry->start / slice_size;
572	end_slice = (entry->end - 1) / slice_size;
573
574	/*
575	* Per plane DDB entry can in a really worst case be on multiple slices
576	* but single entry is anyway contigious.
577	*/
578	while (start_slice <= end_slice) {
579	slice_mask |= BIT(start_slice);
580	start_slice++;
581	}
582
583	return slice_mask;
584	}
585
586	static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
587	{
588	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
589	int hdisplay, vdisplay;
590
591	if (!crtc_state->hw.active)
592	return 0;
593
594	/*
595	* Watermark/ddb requirement highly depends upon width of the
596	* framebuffer, So instead of allocating DDB equally among pipes
597	* distribute DDB based on resolution/width of the display.
598	*/
599	drm_mode_get_hv_timing(mode: pipe_mode, hdisplay: &hdisplay, vdisplay: &vdisplay);
600
601	return hdisplay;
602	}
603
604	static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
605	enum pipe for_pipe,
606	unsigned int *weight_start,
607	unsigned int *weight_end,
608	unsigned int *weight_total)
609	{
610	struct drm_i915_private *i915 =
611	to_i915(dev: dbuf_state->base.state->base.dev);
612	enum pipe pipe;
613
614	*weight_start = 0;
615	*weight_end = 0;
616	*weight_total = 0;
617
618	for_each_pipe(i915, pipe) {
619	int weight = dbuf_state->weight[pipe];
620
621	/*
622	* Do not account pipes using other slice sets
623	* luckily as of current BSpec slice sets do not partially
624	* intersect(pipes share either same one slice or same slice set
625	* i.e no partial intersection), so it is enough to check for
626	* equality for now.
627	*/
628	if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
629	continue;
630
631	*weight_total += weight;
632	if (pipe < for_pipe) {
633	*weight_start += weight;
634	*weight_end += weight;
635	} else if (pipe == for_pipe) {
636	*weight_end += weight;
637	}
638	}
639	}
640
641	static int
642	skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
643	{
644	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
645	unsigned int weight_total, weight_start, weight_end;
646	const struct intel_dbuf_state *old_dbuf_state =
647	intel_atomic_get_old_dbuf_state(state);
648	struct intel_dbuf_state *new_dbuf_state =
649	intel_atomic_get_new_dbuf_state(state);
650	struct intel_crtc_state *crtc_state;
651	struct skl_ddb_entry ddb_slices;
652	enum pipe pipe = crtc->pipe;
653	unsigned int mbus_offset = 0;
654	u32 ddb_range_size;
655	u32 dbuf_slice_mask;
656	u32 start, end;
657	int ret;
658
659	if (new_dbuf_state->weight[pipe] == 0) {
660	skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe], start: 0, end: 0);
661	goto out;
662	}
663
664	dbuf_slice_mask = new_dbuf_state->slices[pipe];
665
666	skl_ddb_entry_for_slices(i915, slice_mask: dbuf_slice_mask, ddb: &ddb_slices);
667	mbus_offset = mbus_ddb_offset(i915, slice_mask: dbuf_slice_mask);
668	ddb_range_size = skl_ddb_entry_size(entry: &ddb_slices);
669
670	intel_crtc_dbuf_weights(dbuf_state: new_dbuf_state, for_pipe: pipe,
671	weight_start: &weight_start, weight_end: &weight_end, weight_total: &weight_total);
672
673	start = ddb_range_size * weight_start / weight_total;
674	end = ddb_range_size * weight_end / weight_total;
675
676	skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe],
677	start: ddb_slices.start - mbus_offset + start,
678	end: ddb_slices.start - mbus_offset + end);
679
680	out:
681	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
682	skl_ddb_entry_equal(e1: &old_dbuf_state->ddb[pipe],
683	e2: &new_dbuf_state->ddb[pipe]))
684	return 0;
685
686	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
687	if (ret)
688	return ret;
689
690	crtc_state = intel_atomic_get_crtc_state(state: &state->base, crtc);
691	if (IS_ERR(ptr: crtc_state))
692	return PTR_ERR(ptr: crtc_state);
693
694	/*
695	* Used for checking overlaps, so we need absolute
696	* offsets instead of MBUS relative offsets.
697	*/
698	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
699	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
700
701	drm_dbg_kms(&i915->drm,
702	"[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
703	crtc->base.base.id, crtc->base.name,
704	old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
705	old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
706	new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
707	old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
708
709	return 0;
710	}
711
712	static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
713	int width, const struct drm_format_info *format,
714	u64 modifier, unsigned int rotation,
715	u32 plane_pixel_rate, struct skl_wm_params *wp,
716	int color_plane);
717
718	static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
719	struct intel_plane *plane,
720	int level,
721	unsigned int latency,
722	const struct skl_wm_params *wp,
723	const struct skl_wm_level *result_prev,
724	struct skl_wm_level *result /* out */);
725
726	static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
727	const struct skl_wm_params *wp)
728	{
729	unsigned int latency = i915->display.wm.skl_latency[level];
730
731	if (latency == 0)
732	return 0;
733
734	/*
735	* WaIncreaseLatencyIPCEnabled: kbl,cfl
736	* Display WA #1141: kbl,cfl
737	*/
738	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
739	skl_watermark_ipc_enabled(i915))
740	latency += 4;
741
742	if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
743	latency += 15;
744
745	return latency;
746	}
747
748	static unsigned int
749	skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
750	int num_active)
751	{
752	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
753	struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
754	struct skl_wm_level wm = {};
755	int ret, min_ddb_alloc = 0;
756	struct skl_wm_params wp;
757	int level;
758
759	ret = skl_compute_wm_params(crtc_state, width: 256,
760	format: drm_format_info(DRM_FORMAT_ARGB8888),
761	DRM_FORMAT_MOD_LINEAR,
762	DRM_MODE_ROTATE_0,
763	plane_pixel_rate: crtc_state->pixel_rate, wp: &wp, color_plane: 0);
764	drm_WARN_ON(&i915->drm, ret);
765
766	for (level = 0; level < i915->display.wm.num_levels; level++) {
767	unsigned int latency = skl_wm_latency(i915, level, wp: &wp);
768
769	skl_compute_plane_wm(crtc_state, plane, level, latency, wp: &wp, result_prev: &wm, result: &wm);
770	if (wm.min_ddb_alloc == U16_MAX)
771	break;
772
773	min_ddb_alloc = wm.min_ddb_alloc;
774	}
775
776	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
777	}
778
779	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
780	{
781	skl_ddb_entry_init(entry,
782	REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
783	REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
784	if (entry->end)
785	entry->end++;
786	}
787
788	static void
789	skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
790	const enum pipe pipe,
791	const enum plane_id plane_id,
792	struct skl_ddb_entry *ddb,
793	struct skl_ddb_entry *ddb_y)
794	{
795	u32 val;
796
797	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
798	if (plane_id == PLANE_CURSOR) {
799	val = intel_de_read(i915, CUR_BUF_CFG(pipe));
800	skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
801	return;
802	}
803
804	val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
805	skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
806
807	if (DISPLAY_VER(i915) >= 11)
808	return;
809
810	val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
811	skl_ddb_entry_init_from_hw(entry: ddb_y, reg: val);
812	}
813
814	static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
815	struct skl_ddb_entry *ddb,
816	struct skl_ddb_entry *ddb_y)
817	{
818	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
819	enum intel_display_power_domain power_domain;
820	enum pipe pipe = crtc->pipe;
821	intel_wakeref_t wakeref;
822	enum plane_id plane_id;
823
824	power_domain = POWER_DOMAIN_PIPE(pipe);
825	wakeref = intel_display_power_get_if_enabled(dev_priv: i915, domain: power_domain);
826	if (!wakeref)
827	return;
828
829	for_each_plane_id_on_crtc(crtc, plane_id)
830	skl_ddb_get_hw_plane_state(i915, pipe,
831	plane_id,
832	ddb: &ddb[plane_id],
833	ddb_y: &ddb_y[plane_id]);
834
835	intel_display_power_put(dev_priv: i915, domain: power_domain, wakeref);
836	}
837
838	struct dbuf_slice_conf_entry {
839	u8 active_pipes;
840	u8 dbuf_mask[I915_MAX_PIPES];
841	bool join_mbus;
842	};
843
844	/*
845	* Table taken from Bspec 12716
846	* Pipes do have some preferred DBuf slice affinity,
847	* plus there are some hardcoded requirements on how
848	* those should be distributed for multipipe scenarios.
849	* For more DBuf slices algorithm can get even more messy
850	* and less readable, so decided to use a table almost
851	* as is from BSpec itself - that way it is at least easier
852	* to compare, change and check.
853	*/
854	static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
855	/* Autogenerated with igt/tools/intel_dbuf_map tool: */
856	{
857	{
858	.active_pipes = BIT(PIPE_A),
859	.dbuf_mask = {
860	[PIPE_A] = BIT(DBUF_S1),
861	},
862	},
863	{
864	.active_pipes = BIT(PIPE_B),
865	.dbuf_mask = {
866	[PIPE_B] = BIT(DBUF_S1),
867	},
868	},
869	{
870	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
871	.dbuf_mask = {
872	[PIPE_A] = BIT(DBUF_S1),
873	[PIPE_B] = BIT(DBUF_S2),
874	},
875	},
876	{
877	.active_pipes = BIT(PIPE_C),
878	.dbuf_mask = {
879	[PIPE_C] = BIT(DBUF_S2),
880	},
881	},
882	{
883	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
884	.dbuf_mask = {
885	[PIPE_A] = BIT(DBUF_S1),
886	[PIPE_C] = BIT(DBUF_S2),
887	},
888	},
889	{
890	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
891	.dbuf_mask = {
892	[PIPE_B] = BIT(DBUF_S1),
893	[PIPE_C] = BIT(DBUF_S2),
894	},
895	},
896	{
897	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
898	.dbuf_mask = {
899	[PIPE_A] = BIT(DBUF_S1),
900	[PIPE_B] = BIT(DBUF_S1),
901	[PIPE_C] = BIT(DBUF_S2),
902	},
903	},
904	{}
905	};
906
907	/*
908	* Table taken from Bspec 49255
909	* Pipes do have some preferred DBuf slice affinity,
910	* plus there are some hardcoded requirements on how
911	* those should be distributed for multipipe scenarios.
912	* For more DBuf slices algorithm can get even more messy
913	* and less readable, so decided to use a table almost
914	* as is from BSpec itself - that way it is at least easier
915	* to compare, change and check.
916	*/
917	static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
918	/* Autogenerated with igt/tools/intel_dbuf_map tool: */
919	{
920	{
921	.active_pipes = BIT(PIPE_A),
922	.dbuf_mask = {
923	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
924	},
925	},
926	{
927	.active_pipes = BIT(PIPE_B),
928	.dbuf_mask = {
929	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
930	},
931	},
932	{
933	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
934	.dbuf_mask = {
935	[PIPE_A] = BIT(DBUF_S2),
936	[PIPE_B] = BIT(DBUF_S1),
937	},
938	},
939	{
940	.active_pipes = BIT(PIPE_C),
941	.dbuf_mask = {
942	[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
943	},
944	},
945	{
946	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
947	.dbuf_mask = {
948	[PIPE_A] = BIT(DBUF_S1),
949	[PIPE_C] = BIT(DBUF_S2),
950	},
951	},
952	{
953	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
954	.dbuf_mask = {
955	[PIPE_B] = BIT(DBUF_S1),
956	[PIPE_C] = BIT(DBUF_S2),
957	},
958	},
959	{
960	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
961	.dbuf_mask = {
962	[PIPE_A] = BIT(DBUF_S1),
963	[PIPE_B] = BIT(DBUF_S1),
964	[PIPE_C] = BIT(DBUF_S2),
965	},
966	},
967	{
968	.active_pipes = BIT(PIPE_D),
969	.dbuf_mask = {
970	[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
971	},
972	},
973	{
974	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
975	.dbuf_mask = {
976	[PIPE_A] = BIT(DBUF_S1),
977	[PIPE_D] = BIT(DBUF_S2),
978	},
979	},
980	{
981	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
982	.dbuf_mask = {
983	[PIPE_B] = BIT(DBUF_S1),
984	[PIPE_D] = BIT(DBUF_S2),
985	},
986	},
987	{
988	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
989	.dbuf_mask = {
990	[PIPE_A] = BIT(DBUF_S1),
991	[PIPE_B] = BIT(DBUF_S1),
992	[PIPE_D] = BIT(DBUF_S2),
993	},
994	},
995	{
996	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
997	.dbuf_mask = {
998	[PIPE_C] = BIT(DBUF_S1),
999	[PIPE_D] = BIT(DBUF_S2),
1000	},
1001	},
1002	{
1003	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1004	.dbuf_mask = {
1005	[PIPE_A] = BIT(DBUF_S1),
1006	[PIPE_C] = BIT(DBUF_S2),
1007	[PIPE_D] = BIT(DBUF_S2),
1008	},
1009	},
1010	{
1011	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1012	.dbuf_mask = {
1013	[PIPE_B] = BIT(DBUF_S1),
1014	[PIPE_C] = BIT(DBUF_S2),
1015	[PIPE_D] = BIT(DBUF_S2),
1016	},
1017	},
1018	{
1019	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1020	.dbuf_mask = {
1021	[PIPE_A] = BIT(DBUF_S1),
1022	[PIPE_B] = BIT(DBUF_S1),
1023	[PIPE_C] = BIT(DBUF_S2),
1024	[PIPE_D] = BIT(DBUF_S2),
1025	},
1026	},
1027	{}
1028	};
1029
1030	static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1031	{
1032	.active_pipes = BIT(PIPE_A),
1033	.dbuf_mask = {
1034	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1035	},
1036	},
1037	{
1038	.active_pipes = BIT(PIPE_B),
1039	.dbuf_mask = {
1040	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1041	},
1042	},
1043	{
1044	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1045	.dbuf_mask = {
1046	[PIPE_A] = BIT(DBUF_S1),
1047	[PIPE_B] = BIT(DBUF_S2),
1048	},
1049	},
1050	{
1051	.active_pipes = BIT(PIPE_C),
1052	.dbuf_mask = {
1053	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1054	},
1055	},
1056	{
1057	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1058	.dbuf_mask = {
1059	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1060	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1061	},
1062	},
1063	{
1064	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1065	.dbuf_mask = {
1066	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1067	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1068	},
1069	},
1070	{
1071	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1072	.dbuf_mask = {
1073	[PIPE_A] = BIT(DBUF_S1),
1074	[PIPE_B] = BIT(DBUF_S2),
1075	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1076	},
1077	},
1078	{
1079	.active_pipes = BIT(PIPE_D),
1080	.dbuf_mask = {
1081	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1082	},
1083	},
1084	{
1085	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1086	.dbuf_mask = {
1087	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1088	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1089	},
1090	},
1091	{
1092	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1093	.dbuf_mask = {
1094	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1095	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1096	},
1097	},
1098	{
1099	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1100	.dbuf_mask = {
1101	[PIPE_A] = BIT(DBUF_S1),
1102	[PIPE_B] = BIT(DBUF_S2),
1103	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1104	},
1105	},
1106	{
1107	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1108	.dbuf_mask = {
1109	[PIPE_C] = BIT(DBUF_S3),
1110	[PIPE_D] = BIT(DBUF_S4),
1111	},
1112	},
1113	{
1114	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1115	.dbuf_mask = {
1116	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1117	[PIPE_C] = BIT(DBUF_S3),
1118	[PIPE_D] = BIT(DBUF_S4),
1119	},
1120	},
1121	{
1122	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1123	.dbuf_mask = {
1124	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1125	[PIPE_C] = BIT(DBUF_S3),
1126	[PIPE_D] = BIT(DBUF_S4),
1127	},
1128	},
1129	{
1130	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1131	.dbuf_mask = {
1132	[PIPE_A] = BIT(DBUF_S1),
1133	[PIPE_B] = BIT(DBUF_S2),
1134	[PIPE_C] = BIT(DBUF_S3),
1135	[PIPE_D] = BIT(DBUF_S4),
1136	},
1137	},
1138	{}
1139	};
1140
1141	static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1142	/*
1143	* Keep the join_mbus cases first so check_mbus_joined()
1144	* will prefer them over the !join_mbus cases.
1145	*/
1146	{
1147	.active_pipes = BIT(PIPE_A),
1148	.dbuf_mask = {
1149	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1150	},
1151	.join_mbus = true,
1152	},
1153	{
1154	.active_pipes = BIT(PIPE_B),
1155	.dbuf_mask = {
1156	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1157	},
1158	.join_mbus = true,
1159	},
1160	{
1161	.active_pipes = BIT(PIPE_A),
1162	.dbuf_mask = {
1163	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1164	},
1165	.join_mbus = false,
1166	},
1167	{
1168	.active_pipes = BIT(PIPE_B),
1169	.dbuf_mask = {
1170	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1171	},
1172	.join_mbus = false,
1173	},
1174	{
1175	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1176	.dbuf_mask = {
1177	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1178	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1179	},
1180	},
1181	{
1182	.active_pipes = BIT(PIPE_C),
1183	.dbuf_mask = {
1184	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1185	},
1186	},
1187	{
1188	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1189	.dbuf_mask = {
1190	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1191	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1192	},
1193	},
1194	{
1195	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1196	.dbuf_mask = {
1197	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1198	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1199	},
1200	},
1201	{
1202	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1203	.dbuf_mask = {
1204	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1205	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1206	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1207	},
1208	},
1209	{
1210	.active_pipes = BIT(PIPE_D),
1211	.dbuf_mask = {
1212	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1213	},
1214	},
1215	{
1216	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1217	.dbuf_mask = {
1218	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1219	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1220	},
1221	},
1222	{
1223	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1224	.dbuf_mask = {
1225	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1226	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1227	},
1228	},
1229	{
1230	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1231	.dbuf_mask = {
1232	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1233	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1234	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1235	},
1236	},
1237	{
1238	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1239	.dbuf_mask = {
1240	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1241	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1242	},
1243	},
1244	{
1245	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1246	.dbuf_mask = {
1247	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1248	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1249	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1250	},
1251	},
1252	{
1253	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1254	.dbuf_mask = {
1255	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1256	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1257	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1258	},
1259	},
1260	{
1261	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1262	.dbuf_mask = {
1263	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1264	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1265	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1266	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1267	},
1268	},
1269	{}
1270
1271	};
1272
1273	static bool check_mbus_joined(u8 active_pipes,
1274	const struct dbuf_slice_conf_entry *dbuf_slices)
1275	{
1276	int i;
1277
1278	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1279	if (dbuf_slices[i].active_pipes == active_pipes)
1280	return dbuf_slices[i].join_mbus;
1281	}
1282	return false;
1283	}
1284
1285	static bool adlp_check_mbus_joined(u8 active_pipes)
1286	{
1287	return check_mbus_joined(active_pipes, dbuf_slices: adlp_allowed_dbufs);
1288	}
1289
1290	static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1291	const struct dbuf_slice_conf_entry *dbuf_slices)
1292	{
1293	int i;
1294
1295	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1296	if (dbuf_slices[i].active_pipes == active_pipes &&
1297	dbuf_slices[i].join_mbus == join_mbus)
1298	return dbuf_slices[i].dbuf_mask[pipe];
1299	}
1300	return 0;
1301	}
1302
1303	/*
1304	* This function finds an entry with same enabled pipe configuration and
1305	* returns correspondent DBuf slice mask as stated in BSpec for particular
1306	* platform.
1307	*/
1308	static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1309	{
1310	/*
1311	* FIXME: For ICL this is still a bit unclear as prev BSpec revision
1312	* required calculating "pipe ratio" in order to determine
1313	* if one or two slices can be used for single pipe configurations
1314	* as additional constraint to the existing table.
1315	* However based on recent info, it should be not "pipe ratio"
1316	* but rather ratio between pixel_rate and cdclk with additional
1317	* constants, so for now we are using only table until this is
1318	* clarified. Also this is the reason why crtc_state param is
1319	* still here - we will need it once those additional constraints
1320	* pop up.
1321	*/
1322	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1323	dbuf_slices: icl_allowed_dbufs);
1324	}
1325
1326	static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1327	{
1328	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1329	dbuf_slices: tgl_allowed_dbufs);
1330	}
1331
1332	static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1333	{
1334	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1335	dbuf_slices: adlp_allowed_dbufs);
1336	}
1337
1338	static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1339	{
1340	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1341	dbuf_slices: dg2_allowed_dbufs);
1342	}
1343
1344	static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1345	{
1346	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
1347	enum pipe pipe = crtc->pipe;
1348
1349	if (IS_DG2(i915))
1350	return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1351	else if (DISPLAY_VER(i915) >= 13)
1352	return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1353	else if (DISPLAY_VER(i915) == 12)
1354	return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1355	else if (DISPLAY_VER(i915) == 11)
1356	return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1357	/*
1358	* For anything else just return one slice yet.
1359	* Should be extended for other platforms.
1360	*/
1361	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1362	}
1363
1364	static bool
1365	use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1366	struct intel_plane *plane)
1367	{
1368	struct drm_i915_private *i915 = to_i915(dev: plane->base.dev);
1369
1370	return DISPLAY_VER(i915) >= 13 &&
1371	crtc_state->uapi.async_flip &&
1372	plane->async_flip;
1373	}
1374
1375	static u64
1376	skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1377	{
1378	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1379	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
1380	enum plane_id plane_id;
1381	u64 data_rate = 0;
1382
1383	for_each_plane_id_on_crtc(crtc, plane_id) {
1384	if (plane_id == PLANE_CURSOR)
1385	continue;
1386
1387	data_rate += crtc_state->rel_data_rate[plane_id];
1388
1389	if (DISPLAY_VER(i915) < 11)
1390	data_rate += crtc_state->rel_data_rate_y[plane_id];
1391	}
1392
1393	return data_rate;
1394	}
1395
1396	static const struct skl_wm_level *
1397	skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1398	enum plane_id plane_id,
1399	int level)
1400	{
1401	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1402
1403	if (level == 0 && pipe_wm->use_sagv_wm)
1404	return &wm->sagv.wm0;
1405
1406	return &wm->wm[level];
1407	}
1408
1409	static const struct skl_wm_level *
1410	skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1411	enum plane_id plane_id)
1412	{
1413	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1414
1415	if (pipe_wm->use_sagv_wm)
1416	return &wm->sagv.trans_wm;
1417
1418	return &wm->trans_wm;
1419	}
1420
1421	/*
1422	* We only disable the watermarks for each plane if
1423	* they exceed the ddb allocation of said plane. This
1424	* is done so that we don't end up touching cursor
1425	* watermarks needlessly when some other plane reduces
1426	* our max possible watermark level.
1427	*
1428	* Bspec has this to say about the PLANE_WM enable bit:
1429	* "All the watermarks at this level for all enabled
1430	* planes must be enabled before the level will be used."
1431	* So this is actually safe to do.
1432	*/
1433	static void
1434	skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1435	{
1436	if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb))
1437	memset(wm, 0, sizeof(*wm));
1438	}
1439
1440	static void
1441	skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1442	const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1443	{
1444	if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb_y) ||
1445	uv_wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1446	memset(wm, 0, sizeof(*wm));
1447	memset(uv_wm, 0, sizeof(*uv_wm));
1448	}
1449	}
1450
1451	static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1452	const struct skl_plane_wm *wm)
1453	{
1454	/*
1455	* Wa_1408961008:icl, ehl
1456	* Wa_14012656716:tgl, adl
1457	* Wa_14017887344:icl
1458	* Wa_14017868169:adl, tgl
1459	* Due to some power saving optimizations, different subsystems
1460	* like PSR, might still use even disabled wm level registers,
1461	* for "reference", so lets keep at least the values sane.
1462	* Considering amount of WA requiring us to do similar things, was
1463	* decided to simply do it for all of the platforms, as those wm
1464	* levels are disabled, this isn't going to do harm anyway.
1465	*/
1466	return level > 0 && !wm->wm[level].enable;
1467	}
1468
1469	struct skl_plane_ddb_iter {
1470	u64 data_rate;
1471	u16 start, size;
1472	};
1473
1474	static void
1475	skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1476	struct skl_ddb_entry *ddb,
1477	const struct skl_wm_level *wm,
1478	u64 data_rate)
1479	{
1480	u16 size, extra = 0;
1481
1482	if (data_rate) {
1483	extra = min_t(u16, iter->size,
1484	DIV64_U64_ROUND_UP(iter->size * data_rate,
1485	iter->data_rate));
1486	iter->size -= extra;
1487	iter->data_rate -= data_rate;
1488	}
1489
1490	/*
1491	* Keep ddb entry of all disabled planes explicitly zeroed
1492	* to avoid skl_ddb_add_affected_planes() adding them to
1493	* the state when other planes change their allocations.
1494	*/
1495	size = wm->min_ddb_alloc + extra;
1496	if (size)
1497	iter->start = skl_ddb_entry_init(entry: ddb, start: iter->start,
1498	end: iter->start + size);
1499	}
1500
1501	static int
1502	skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1503	struct intel_crtc *crtc)
1504	{
1505	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
1506	struct intel_crtc_state *crtc_state =
1507	intel_atomic_get_new_crtc_state(state, crtc);
1508	const struct intel_dbuf_state *dbuf_state =
1509	intel_atomic_get_new_dbuf_state(state);
1510	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1511	int num_active = hweight8(dbuf_state->active_pipes);
1512	struct skl_plane_ddb_iter iter;
1513	enum plane_id plane_id;
1514	u16 cursor_size;
1515	u32 blocks;
1516	int level;
1517
1518	/* Clear the partitioning for disabled planes. */
1519	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1520	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1521
1522	if (!crtc_state->hw.active)
1523	return 0;
1524
1525	iter.start = alloc->start;
1526	iter.size = skl_ddb_entry_size(entry: alloc);
1527	if (iter.size == 0)
1528	return 0;
1529
1530	/* Allocate fixed number of blocks for cursor. */
1531	cursor_size = skl_cursor_allocation(crtc_state, num_active);
1532	iter.size -= cursor_size;
1533	skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1534	start: alloc->end - cursor_size, end: alloc->end);
1535
1536	iter.data_rate = skl_total_relative_data_rate(crtc_state);
1537
1538	/*
1539	* Find the highest watermark level for which we can satisfy the block
1540	* requirement of active planes.
1541	*/
1542	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1543	blocks = 0;
1544	for_each_plane_id_on_crtc(crtc, plane_id) {
1545	const struct skl_plane_wm *wm =
1546	&crtc_state->wm.skl.optimal.planes[plane_id];
1547
1548	if (plane_id == PLANE_CURSOR) {
1549	const struct skl_ddb_entry *ddb =
1550	&crtc_state->wm.skl.plane_ddb[plane_id];
1551
1552	if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1553	drm_WARN_ON(&i915->drm,
1554	wm->wm[level].min_ddb_alloc != U16_MAX);
1555	blocks = U32_MAX;
1556	break;
1557	}
1558	continue;
1559	}
1560
1561	blocks += wm->wm[level].min_ddb_alloc;
1562	blocks += wm->uv_wm[level].min_ddb_alloc;
1563	}
1564
1565	if (blocks <= iter.size) {
1566	iter.size -= blocks;
1567	break;
1568	}
1569	}
1570
1571	if (level < 0) {
1572	drm_dbg_kms(&i915->drm,
1573	"Requested display configuration exceeds system DDB limitations");
1574	drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1575	blocks, iter.size);
1576	return -EINVAL;
1577	}
1578
1579	/* avoid the WARN later when we don't allocate any extra DDB */
1580	if (iter.data_rate == 0)
1581	iter.size = 0;
1582
1583	/*
1584	* Grant each plane the blocks it requires at the highest achievable
1585	* watermark level, plus an extra share of the leftover blocks
1586	* proportional to its relative data rate.
1587	*/
1588	for_each_plane_id_on_crtc(crtc, plane_id) {
1589	struct skl_ddb_entry *ddb =
1590	&crtc_state->wm.skl.plane_ddb[plane_id];
1591	struct skl_ddb_entry *ddb_y =
1592	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1593	const struct skl_plane_wm *wm =
1594	&crtc_state->wm.skl.optimal.planes[plane_id];
1595
1596	if (plane_id == PLANE_CURSOR)
1597	continue;
1598
1599	if (DISPLAY_VER(i915) < 11 &&
1600	crtc_state->nv12_planes & BIT(plane_id)) {
1601	skl_allocate_plane_ddb(iter: &iter, ddb: ddb_y, wm: &wm->wm[level],
1602	data_rate: crtc_state->rel_data_rate_y[plane_id]);
1603	skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->uv_wm[level],
1604	data_rate: crtc_state->rel_data_rate[plane_id]);
1605	} else {
1606	skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->wm[level],
1607	data_rate: crtc_state->rel_data_rate[plane_id]);
1608	}
1609	}
1610	drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1611
1612	/*
1613	* When we calculated watermark values we didn't know how high
1614	* of a level we'd actually be able to hit, so we just marked
1615	* all levels as "enabled." Go back now and disable the ones
1616	* that aren't actually possible.
1617	*/
1618	for (level++; level < i915->display.wm.num_levels; level++) {
1619	for_each_plane_id_on_crtc(crtc, plane_id) {
1620	const struct skl_ddb_entry *ddb =
1621	&crtc_state->wm.skl.plane_ddb[plane_id];
1622	const struct skl_ddb_entry *ddb_y =
1623	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1624	struct skl_plane_wm *wm =
1625	&crtc_state->wm.skl.optimal.planes[plane_id];
1626
1627	if (DISPLAY_VER(i915) < 11 &&
1628	crtc_state->nv12_planes & BIT(plane_id))
1629	skl_check_nv12_wm_level(wm: &wm->wm[level],
1630	uv_wm: &wm->uv_wm[level],
1631	ddb_y, ddb);
1632	else
1633	skl_check_wm_level(wm: &wm->wm[level], ddb);
1634
1635	if (skl_need_wm_copy_wa(i915, level, wm)) {
1636	wm->wm[level].blocks = wm->wm[level - 1].blocks;
1637	wm->wm[level].lines = wm->wm[level - 1].lines;
1638	wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1639	}
1640	}
1641	}
1642
1643	/*
1644	* Go back and disable the transition and SAGV watermarks
1645	* if it turns out we don't have enough DDB blocks for them.
1646	*/
1647	for_each_plane_id_on_crtc(crtc, plane_id) {
1648	const struct skl_ddb_entry *ddb =
1649	&crtc_state->wm.skl.plane_ddb[plane_id];
1650	const struct skl_ddb_entry *ddb_y =
1651	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1652	struct skl_plane_wm *wm =
1653	&crtc_state->wm.skl.optimal.planes[plane_id];
1654
1655	if (DISPLAY_VER(i915) < 11 &&
1656	crtc_state->nv12_planes & BIT(plane_id)) {
1657	skl_check_wm_level(wm: &wm->trans_wm, ddb: ddb_y);
1658	} else {
1659	WARN_ON(skl_ddb_entry_size(ddb_y));
1660
1661	skl_check_wm_level(wm: &wm->trans_wm, ddb);
1662	}
1663
1664	skl_check_wm_level(wm: &wm->sagv.wm0, ddb);
1665	skl_check_wm_level(wm: &wm->sagv.trans_wm, ddb);
1666	}
1667
1668	return 0;
1669	}
1670
1671	/*
1672	* The max latency should be 257 (max the punit can code is 255 and we add 2us
1673	* for the read latency) and cpp should always be <= 8, so that
1674	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
1675	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1676	*/
1677	static uint_fixed_16_16_t
1678	skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1679	u8 cpp, u32 latency, u32 dbuf_block_size)
1680	{
1681	u32 wm_intermediate_val;
1682	uint_fixed_16_16_t ret;
1683
1684	if (latency == 0)
1685	return FP_16_16_MAX;
1686
1687	wm_intermediate_val = latency * pixel_rate * cpp;
1688	ret = div_fixed16(val: wm_intermediate_val, d: 1000 * dbuf_block_size);
1689
1690	if (DISPLAY_VER(i915) >= 10)
1691	ret = add_fixed16_u32(add1: ret, add2: 1);
1692
1693	return ret;
1694	}
1695
1696	static uint_fixed_16_16_t
1697	skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1698	uint_fixed_16_16_t plane_blocks_per_line)
1699	{
1700	u32 wm_intermediate_val;
1701	uint_fixed_16_16_t ret;
1702
1703	if (latency == 0)
1704	return FP_16_16_MAX;
1705
1706	wm_intermediate_val = latency * pixel_rate;
1707	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1708	pipe_htotal * 1000);
1709	ret = mul_u32_fixed16(val: wm_intermediate_val, mul: plane_blocks_per_line);
1710	return ret;
1711	}
1712
1713	static uint_fixed_16_16_t
1714	intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1715	{
1716	struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
1717	u32 pixel_rate;
1718	u32 crtc_htotal;
1719	uint_fixed_16_16_t linetime_us;
1720
1721	if (!crtc_state->hw.active)
1722	return u32_to_fixed16(val: 0);
1723
1724	pixel_rate = crtc_state->pixel_rate;
1725
1726	if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1727	return u32_to_fixed16(val: 0);
1728
1729	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1730	linetime_us = div_fixed16(val: crtc_htotal * 1000, d: pixel_rate);
1731
1732	return linetime_us;
1733	}
1734
1735	static int
1736	skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1737	int width, const struct drm_format_info *format,
1738	u64 modifier, unsigned int rotation,
1739	u32 plane_pixel_rate, struct skl_wm_params *wp,
1740	int color_plane)
1741	{
1742	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1743	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
1744	u32 interm_pbpl;
1745
1746	/* only planar format has two planes */
1747	if (color_plane == 1 &&
1748	!intel_format_info_is_yuv_semiplanar(info: format, modifier)) {
1749	drm_dbg_kms(&i915->drm,
1750	"Non planar format have single plane\n");
1751	return -EINVAL;
1752	}
1753
1754	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1755	wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1756	intel_fb_is_tiled_modifier(modifier);
1757	wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1758	wp->is_planar = intel_format_info_is_yuv_semiplanar(info: format, modifier);
1759
1760	wp->width = width;
1761	if (color_plane == 1 && wp->is_planar)
1762	wp->width /= 2;
1763
1764	wp->cpp = format->cpp[color_plane];
1765	wp->plane_pixel_rate = plane_pixel_rate;
1766
1767	if (DISPLAY_VER(i915) >= 11 &&
1768	modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1769	wp->dbuf_block_size = 256;
1770	else
1771	wp->dbuf_block_size = 512;
1772
1773	if (drm_rotation_90_or_270(rotation)) {
1774	switch (wp->cpp) {
1775	case 1:
1776	wp->y_min_scanlines = 16;
1777	break;
1778	case 2:
1779	wp->y_min_scanlines = 8;
1780	break;
1781	case 4:
1782	wp->y_min_scanlines = 4;
1783	break;
1784	default:
1785	MISSING_CASE(wp->cpp);
1786	return -EINVAL;
1787	}
1788	} else {
1789	wp->y_min_scanlines = 4;
1790	}
1791
1792	if (skl_needs_memory_bw_wa(i915))
1793	wp->y_min_scanlines *= 2;
1794
1795	wp->plane_bytes_per_line = wp->width * wp->cpp;
1796	if (wp->y_tiled) {
1797	interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1798	wp->y_min_scanlines,
1799	wp->dbuf_block_size);
1800
1801	if (DISPLAY_VER(i915) >= 10)
1802	interm_pbpl++;
1803
1804	wp->plane_blocks_per_line = div_fixed16(val: interm_pbpl,
1805	d: wp->y_min_scanlines);
1806	} else {
1807	interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1808	wp->dbuf_block_size);
1809
1810	if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1811	interm_pbpl++;
1812
1813	wp->plane_blocks_per_line = u32_to_fixed16(val: interm_pbpl);
1814	}
1815
1816	wp->y_tile_minimum = mul_u32_fixed16(val: wp->y_min_scanlines,
1817	mul: wp->plane_blocks_per_line);
1818
1819	wp->linetime_us = fixed16_to_u32_round_up(fp: intel_get_linetime_us(crtc_state));
1820
1821	return 0;
1822	}
1823
1824	static int
1825	skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1826	const struct intel_plane_state *plane_state,
1827	struct skl_wm_params *wp, int color_plane)
1828	{
1829	const struct drm_framebuffer *fb = plane_state->hw.fb;
1830	int width;
1831
1832	/*
1833	* Src coordinates are already rotated by 270 degrees for
1834	* the 90/270 degree plane rotation cases (to match the
1835	* GTT mapping), hence no need to account for rotation here.
1836	*/
1837	width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1838
1839	return skl_compute_wm_params(crtc_state, width,
1840	format: fb->format, modifier: fb->modifier,
1841	rotation: plane_state->hw.rotation,
1842	plane_pixel_rate: intel_plane_pixel_rate(crtc_state, plane_state),
1843	wp, color_plane);
1844	}
1845
1846	static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1847	{
1848	if (DISPLAY_VER(i915) >= 10)
1849	return true;
1850
1851	/* The number of lines are ignored for the level 0 watermark. */
1852	return level > 0;
1853	}
1854
1855	static int skl_wm_max_lines(struct drm_i915_private *i915)
1856	{
1857	if (DISPLAY_VER(i915) >= 13)
1858	return 255;
1859	else
1860	return 31;
1861	}
1862
1863	static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1864	struct intel_plane *plane,
1865	int level,
1866	unsigned int latency,
1867	const struct skl_wm_params *wp,
1868	const struct skl_wm_level *result_prev,
1869	struct skl_wm_level *result /* out */)
1870	{
1871	struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
1872	uint_fixed_16_16_t method1, method2;
1873	uint_fixed_16_16_t selected_result;
1874	u32 blocks, lines, min_ddb_alloc = 0;
1875
1876	if (latency == 0 ||
1877	(use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1878	/* reject it */
1879	result->min_ddb_alloc = U16_MAX;
1880	return;
1881	}
1882
1883	method1 = skl_wm_method1(i915, pixel_rate: wp->plane_pixel_rate,
1884	cpp: wp->cpp, latency, dbuf_block_size: wp->dbuf_block_size);
1885	method2 = skl_wm_method2(pixel_rate: wp->plane_pixel_rate,
1886	pipe_htotal: crtc_state->hw.pipe_mode.crtc_htotal,
1887	latency,
1888	plane_blocks_per_line: wp->plane_blocks_per_line);
1889
1890	if (wp->y_tiled) {
1891	selected_result = max_fixed16(max1: method2, max2: wp->y_tile_minimum);
1892	} else {
1893	if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1894	wp->dbuf_block_size < 1) &&
1895	(wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1896	selected_result = method2;
1897	} else if (latency >= wp->linetime_us) {
1898	if (DISPLAY_VER(i915) == 9)
1899	selected_result = min_fixed16(min1: method1, min2: method2);
1900	else
1901	selected_result = method2;
1902	} else {
1903	selected_result = method1;
1904	}
1905	}
1906
1907	blocks = fixed16_to_u32_round_up(fp: selected_result) + 1;
1908	/*
1909	* Lets have blocks at minimum equivalent to plane_blocks_per_line
1910	* as there will be at minimum one line for lines configuration. This
1911	* is a work around for FIFO underruns observed with resolutions like
1912	* 4k 60 Hz in single channel DRAM configurations.
1913	*
1914	* As per the Bspec 49325, if the ddb allocation can hold at least
1915	* one plane_blocks_per_line, we should have selected method2 in
1916	* the above logic. Assuming that modern versions have enough dbuf
1917	* and method2 guarantees blocks equivalent to at least 1 line,
1918	* select the blocks as plane_blocks_per_line.
1919	*
1920	* TODO: Revisit the logic when we have better understanding on DRAM
1921	* channels' impact on the level 0 memory latency and the relevant
1922	* wm calculations.
1923	*/
1924	if (skl_wm_has_lines(i915, level))
1925	blocks = max(blocks,
1926	fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1927	lines = div_round_up_fixed16(val: selected_result,
1928	d: wp->plane_blocks_per_line);
1929
1930	if (DISPLAY_VER(i915) == 9) {
1931	/* Display WA #1125: skl,bxt,kbl */
1932	if (level == 0 && wp->rc_surface)
1933	blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1934
1935	/* Display WA #1126: skl,bxt,kbl */
1936	if (level >= 1 && level <= 7) {
1937	if (wp->y_tiled) {
1938	blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1939	lines += wp->y_min_scanlines;
1940	} else {
1941	blocks++;
1942	}
1943
1944	/*
1945	* Make sure result blocks for higher latency levels are
1946	* at least as high as level below the current level.
1947	* Assumption in DDB algorithm optimization for special
1948	* cases. Also covers Display WA #1125 for RC.
1949	*/
1950	if (result_prev->blocks > blocks)
1951	blocks = result_prev->blocks;
1952	}
1953	}
1954
1955	if (DISPLAY_VER(i915) >= 11) {
1956	if (wp->y_tiled) {
1957	int extra_lines;
1958
1959	if (lines % wp->y_min_scanlines == 0)
1960	extra_lines = wp->y_min_scanlines;
1961	else
1962	extra_lines = wp->y_min_scanlines * 2 -
1963	lines % wp->y_min_scanlines;
1964
1965	min_ddb_alloc = mul_round_up_u32_fixed16(val: lines + extra_lines,
1966	mul: wp->plane_blocks_per_line);
1967	} else {
1968	min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1969	}
1970	}
1971
1972	if (!skl_wm_has_lines(i915, level))
1973	lines = 0;
1974
1975	if (lines > skl_wm_max_lines(i915)) {
1976	/* reject it */
1977	result->min_ddb_alloc = U16_MAX;
1978	return;
1979	}
1980
1981	/*
1982	* If lines is valid, assume we can use this watermark level
1983	* for now. We'll come back and disable it after we calculate the
1984	* DDB allocation if it turns out we don't actually have enough
1985	* blocks to satisfy it.
1986	*/
1987	result->blocks = blocks;
1988	result->lines = lines;
1989	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1990	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1991	result->enable = true;
1992
1993	if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1994	result->can_sagv = latency >= i915->display.sagv.block_time_us;
1995	}
1996
1997	static void
1998	skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1999	struct intel_plane *plane,
2000	const struct skl_wm_params *wm_params,
2001	struct skl_wm_level *levels)
2002	{
2003	struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
2004	struct skl_wm_level *result_prev = &levels[0];
2005	int level;
2006
2007	for (level = 0; level < i915->display.wm.num_levels; level++) {
2008	struct skl_wm_level *result = &levels[level];
2009	unsigned int latency = skl_wm_latency(i915, level, wp: wm_params);
2010
2011	skl_compute_plane_wm(crtc_state, plane, level, latency,
2012	wp: wm_params, result_prev, result);
2013
2014	result_prev = result;
2015	}
2016	}
2017
2018	static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2019	struct intel_plane *plane,
2020	const struct skl_wm_params *wm_params,
2021	struct skl_plane_wm *plane_wm)
2022	{
2023	struct drm_i915_private *i915 = to_i915(dev: crtc_state->uapi.crtc->dev);
2024	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2025	struct skl_wm_level *levels = plane_wm->wm;
2026	unsigned int latency = 0;
2027
2028	if (i915->display.sagv.block_time_us)
2029	latency = i915->display.sagv.block_time_us +
2030	skl_wm_latency(i915, level: 0, wp: wm_params);
2031
2032	skl_compute_plane_wm(crtc_state, plane, level: 0, latency,
2033	wp: wm_params, result_prev: &levels[0],
2034	result: sagv_wm);
2035	}
2036
2037	static void skl_compute_transition_wm(struct drm_i915_private *i915,
2038	struct skl_wm_level *trans_wm,
2039	const struct skl_wm_level *wm0,
2040	const struct skl_wm_params *wp)
2041	{
2042	u16 trans_min, trans_amount, trans_y_tile_min;
2043	u16 wm0_blocks, trans_offset, blocks;
2044
2045	/* Transition WM don't make any sense if ipc is disabled */
2046	if (!skl_watermark_ipc_enabled(i915))
2047	return;
2048
2049	/*
2050	* WaDisableTWM:skl,kbl,cfl,bxt
2051	* Transition WM are not recommended by HW team for GEN9
2052	*/
2053	if (DISPLAY_VER(i915) == 9)
2054	return;
2055
2056	if (DISPLAY_VER(i915) >= 11)
2057	trans_min = 4;
2058	else
2059	trans_min = 14;
2060
2061	/* Display WA #1140: glk,cnl */
2062	if (DISPLAY_VER(i915) == 10)
2063	trans_amount = 0;
2064	else
2065	trans_amount = 10; /* This is configurable amount */
2066
2067	trans_offset = trans_min + trans_amount;
2068
2069	/*
2070	* The spec asks for Selected Result Blocks for wm0 (the real value),
2071	* not Result Blocks (the integer value). Pay attention to the capital
2072	* letters. The value wm_l0->blocks is actually Result Blocks, but
2073	* since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2074	* and since we later will have to get the ceiling of the sum in the
2075	* transition watermarks calculation, we can just pretend Selected
2076	* Result Blocks is Result Blocks minus 1 and it should work for the
2077	* current platforms.
2078	*/
2079	wm0_blocks = wm0->blocks - 1;
2080
2081	if (wp->y_tiled) {
2082	trans_y_tile_min =
2083	(u16)mul_round_up_u32_fixed16(val: 2, mul: wp->y_tile_minimum);
2084	blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2085	} else {
2086	blocks = wm0_blocks + trans_offset;
2087	}
2088	blocks++;
2089
2090	/*
2091	* Just assume we can enable the transition watermark. After
2092	* computing the DDB we'll come back and disable it if that
2093	* assumption turns out to be false.
2094	*/
2095	trans_wm->blocks = blocks;
2096	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2097	trans_wm->enable = true;
2098	}
2099
2100	static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2101	const struct intel_plane_state *plane_state,
2102	struct intel_plane *plane, int color_plane)
2103	{
2104	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2105	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2106	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2107	struct skl_wm_params wm_params;
2108	int ret;
2109
2110	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2111	wp: &wm_params, color_plane);
2112	if (ret)
2113	return ret;
2114
2115	skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->wm);
2116
2117	skl_compute_transition_wm(i915, trans_wm: &wm->trans_wm,
2118	wm0: &wm->wm[0], wp: &wm_params);
2119
2120	if (DISPLAY_VER(i915) >= 12) {
2121	tgl_compute_sagv_wm(crtc_state, plane, wm_params: &wm_params, plane_wm: wm);
2122
2123	skl_compute_transition_wm(i915, trans_wm: &wm->sagv.trans_wm,
2124	wm0: &wm->sagv.wm0, wp: &wm_params);
2125	}
2126
2127	return 0;
2128	}
2129
2130	static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2131	const struct intel_plane_state *plane_state,
2132	struct intel_plane *plane)
2133	{
2134	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2135	struct skl_wm_params wm_params;
2136	int ret;
2137
2138	wm->is_planar = true;
2139
2140	/* uv plane watermarks must also be validated for NV12/Planar */
2141	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2142	wp: &wm_params, color_plane: 1);
2143	if (ret)
2144	return ret;
2145
2146	skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->uv_wm);
2147
2148	return 0;
2149	}
2150
2151	static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2152	const struct intel_plane_state *plane_state)
2153	{
2154	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2155	enum plane_id plane_id = plane->id;
2156	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2157	const struct drm_framebuffer *fb = plane_state->hw.fb;
2158	int ret;
2159
2160	memset(wm, 0, sizeof(*wm));
2161
2162	if (!intel_wm_plane_visible(crtc_state, plane_state))
2163	return 0;
2164
2165	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2166	plane, color_plane: 0);
2167	if (ret)
2168	return ret;
2169
2170	if (fb->format->is_yuv && fb->format->num_planes > 1) {
2171	ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2172	plane);
2173	if (ret)
2174	return ret;
2175	}
2176
2177	return 0;
2178	}
2179
2180	static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2181	const struct intel_plane_state *plane_state)
2182	{
2183	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2184	struct drm_i915_private *i915 = to_i915(dev: plane->base.dev);
2185	enum plane_id plane_id = plane->id;
2186	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2187	int ret;
2188
2189	/* Watermarks calculated in master */
2190	if (plane_state->planar_slave)
2191	return 0;
2192
2193	memset(wm, 0, sizeof(*wm));
2194
2195	if (plane_state->planar_linked_plane) {
2196	const struct drm_framebuffer *fb = plane_state->hw.fb;
2197
2198	drm_WARN_ON(&i915->drm,
2199	!intel_wm_plane_visible(crtc_state, plane_state));
2200	drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2201	fb->format->num_planes == 1);
2202
2203	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2204	plane: plane_state->planar_linked_plane, color_plane: 0);
2205	if (ret)
2206	return ret;
2207
2208	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2209	plane, color_plane: 1);
2210	if (ret)
2211	return ret;
2212	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2213	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2214	plane, color_plane: 0);
2215	if (ret)
2216	return ret;
2217	}
2218
2219	return 0;
2220	}
2221
2222	static bool
2223	skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2224	int wm0_lines, int latency)
2225	{
2226	const struct drm_display_mode *adjusted_mode =
2227	&crtc_state->hw.adjusted_mode;
2228
2229	/* FIXME missing scaler and DSC pre-fill time */
2230	return crtc_state->framestart_delay +
2231	intel_usecs_to_scanlines(adjusted_mode, usecs: latency) +
2232	wm0_lines >
2233	adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2234	}
2235
2236	static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2237	{
2238	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2239	enum plane_id plane_id;
2240	int wm0_lines = 0;
2241
2242	for_each_plane_id_on_crtc(crtc, plane_id) {
2243	const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2244
2245	/* FIXME what about !skl_wm_has_lines() platforms? */
2246	wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2247	}
2248
2249	return wm0_lines;
2250	}
2251
2252	static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2253	int wm0_lines)
2254	{
2255	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2256	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2257	int level;
2258
2259	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2260	int latency;
2261
2262	/* FIXME should we care about the latency w/a's? */
2263	latency = skl_wm_latency(i915, level, NULL);
2264	if (latency == 0)
2265	continue;
2266
2267	/* FIXME is it correct to use 0 latency for wm0 here? */
2268	if (level == 0)
2269	latency = 0;
2270
2271	if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2272	return level;
2273	}
2274
2275	return -EINVAL;
2276	}
2277
2278	static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2279	{
2280	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2281	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2282	int wm0_lines, level;
2283
2284	if (!crtc_state->hw.active)
2285	return 0;
2286
2287	wm0_lines = skl_max_wm0_lines(crtc_state);
2288
2289	level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2290	if (level < 0)
2291	return level;
2292
2293	/*
2294	* PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2295	* based on whether we're limited by the vblank duration.
2296	*/
2297	crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2298
2299	for (level++; level < i915->display.wm.num_levels; level++) {
2300	enum plane_id plane_id;
2301
2302	for_each_plane_id_on_crtc(crtc, plane_id) {
2303	struct skl_plane_wm *wm =
2304	&crtc_state->wm.skl.optimal.planes[plane_id];
2305
2306	/*
2307	* FIXME just clear enable or flag the entire
2308	* thing as bad via min_ddb_alloc=U16_MAX?
2309	*/
2310	wm->wm[level].enable = false;
2311	wm->uv_wm[level].enable = false;
2312	}
2313	}
2314
2315	if (DISPLAY_VER(i915) >= 12 &&
2316	i915->display.sagv.block_time_us &&
2317	skl_is_vblank_too_short(crtc_state, wm0_lines,
2318	latency: i915->display.sagv.block_time_us)) {
2319	enum plane_id plane_id;
2320
2321	for_each_plane_id_on_crtc(crtc, plane_id) {
2322	struct skl_plane_wm *wm =
2323	&crtc_state->wm.skl.optimal.planes[plane_id];
2324
2325	wm->sagv.wm0.enable = false;
2326	wm->sagv.trans_wm.enable = false;
2327	}
2328	}
2329
2330	return 0;
2331	}
2332
2333	static int skl_build_pipe_wm(struct intel_atomic_state *state,
2334	struct intel_crtc *crtc)
2335	{
2336	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2337	struct intel_crtc_state *crtc_state =
2338	intel_atomic_get_new_crtc_state(state, crtc);
2339	const struct intel_plane_state *plane_state;
2340	struct intel_plane *plane;
2341	int ret, i;
2342
2343	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2344	/*
2345	* FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2346	* instead but we don't populate that correctly for NV12 Y
2347	* planes so for now hack this.
2348	*/
2349	if (plane->pipe != crtc->pipe)
2350	continue;
2351
2352	if (DISPLAY_VER(i915) >= 11)
2353	ret = icl_build_plane_wm(crtc_state, plane_state);
2354	else
2355	ret = skl_build_plane_wm(crtc_state, plane_state);
2356	if (ret)
2357	return ret;
2358	}
2359
2360	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2361
2362	return skl_wm_check_vblank(crtc_state);
2363	}
2364
2365	static void skl_ddb_entry_write(struct drm_i915_private *i915,
2366	i915_reg_t reg,
2367	const struct skl_ddb_entry *entry)
2368	{
2369	if (entry->end)
2370	intel_de_write_fw(i915, reg,
2371	PLANE_BUF_END(entry->end - 1) |
2372	PLANE_BUF_START(entry->start));
2373	else
2374	intel_de_write_fw(i915, reg, val: 0);
2375	}
2376
2377	static void skl_write_wm_level(struct drm_i915_private *i915,
2378	i915_reg_t reg,
2379	const struct skl_wm_level *level)
2380	{
2381	u32 val = 0;
2382
2383	if (level->enable)
2384	val |= PLANE_WM_EN;
2385	if (level->ignore_lines)
2386	val |= PLANE_WM_IGNORE_LINES;
2387	val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2388	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2389
2390	intel_de_write_fw(i915, reg, val);
2391	}
2392
2393	void skl_write_plane_wm(struct intel_plane *plane,
2394	const struct intel_crtc_state *crtc_state)
2395	{
2396	struct drm_i915_private *i915 = to_i915(dev: plane->base.dev);
2397	enum plane_id plane_id = plane->id;
2398	enum pipe pipe = plane->pipe;
2399	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2400	const struct skl_ddb_entry *ddb =
2401	&crtc_state->wm.skl.plane_ddb[plane_id];
2402	const struct skl_ddb_entry *ddb_y =
2403	&crtc_state->wm.skl.plane_ddb_y[plane_id];
2404	int level;
2405
2406	for (level = 0; level < i915->display.wm.num_levels; level++)
2407	skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2408	level: skl_plane_wm_level(pipe_wm, plane_id, level));
2409
2410	skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2411	level: skl_plane_trans_wm(pipe_wm, plane_id));
2412
2413	if (HAS_HW_SAGV_WM(i915)) {
2414	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2415
2416	skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2417	level: &wm->sagv.wm0);
2418	skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2419	level: &wm->sagv.trans_wm);
2420	}
2421
2422	skl_ddb_entry_write(i915,
2423	PLANE_BUF_CFG(pipe, plane_id), entry: ddb);
2424
2425	if (DISPLAY_VER(i915) < 11)
2426	skl_ddb_entry_write(i915,
2427	PLANE_NV12_BUF_CFG(pipe, plane_id), entry: ddb_y);
2428	}
2429
2430	void skl_write_cursor_wm(struct intel_plane *plane,
2431	const struct intel_crtc_state *crtc_state)
2432	{
2433	struct drm_i915_private *i915 = to_i915(dev: plane->base.dev);
2434	enum plane_id plane_id = plane->id;
2435	enum pipe pipe = plane->pipe;
2436	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2437	const struct skl_ddb_entry *ddb =
2438	&crtc_state->wm.skl.plane_ddb[plane_id];
2439	int level;
2440
2441	for (level = 0; level < i915->display.wm.num_levels; level++)
2442	skl_write_wm_level(i915, CUR_WM(pipe, level),
2443	level: skl_plane_wm_level(pipe_wm, plane_id, level));
2444
2445	skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2446	level: skl_plane_trans_wm(pipe_wm, plane_id));
2447
2448	if (HAS_HW_SAGV_WM(i915)) {
2449	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2450
2451	skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2452	level: &wm->sagv.wm0);
2453	skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2454	level: &wm->sagv.trans_wm);
2455	}
2456
2457	skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), entry: ddb);
2458	}
2459
2460	static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2461	const struct skl_wm_level *l2)
2462	{
2463	return l1->enable == l2->enable &&
2464	l1->ignore_lines == l2->ignore_lines &&
2465	l1->lines == l2->lines &&
2466	l1->blocks == l2->blocks;
2467	}
2468
2469	static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2470	const struct skl_plane_wm *wm1,
2471	const struct skl_plane_wm *wm2)
2472	{
2473	int level;
2474
2475	for (level = 0; level < i915->display.wm.num_levels; level++) {
2476	/*
2477	* We don't check uv_wm as the hardware doesn't actually
2478	* use it. It only gets used for calculating the required
2479	* ddb allocation.
2480	*/
2481	if (!skl_wm_level_equals(l1: &wm1->wm[level], l2: &wm2->wm[level]))
2482	return false;
2483	}
2484
2485	return skl_wm_level_equals(l1: &wm1->trans_wm, l2: &wm2->trans_wm) &&
2486	skl_wm_level_equals(l1: &wm1->sagv.wm0, l2: &wm2->sagv.wm0) &&
2487	skl_wm_level_equals(l1: &wm1->sagv.trans_wm, l2: &wm2->sagv.trans_wm);
2488	}
2489
2490	static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2491	const struct skl_ddb_entry *b)
2492	{
2493	return a->start < b->end && b->start < a->end;
2494	}
2495
2496	static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2497	const struct skl_ddb_entry *b)
2498	{
2499	if (a->end && b->end) {
2500	a->start = min(a->start, b->start);
2501	a->end = max(a->end, b->end);
2502	} else if (b->end) {
2503	a->start = b->start;
2504	a->end = b->end;
2505	}
2506	}
2507
2508	bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2509	const struct skl_ddb_entry *entries,
2510	int num_entries, int ignore_idx)
2511	{
2512	int i;
2513
2514	for (i = 0; i < num_entries; i++) {
2515	if (i != ignore_idx &&
2516	skl_ddb_entries_overlap(a: ddb, b: &entries[i]))
2517	return true;
2518	}
2519
2520	return false;
2521	}
2522
2523	static int
2524	skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2525	struct intel_crtc_state *new_crtc_state)
2526	{
2527	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2528	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2529	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2530	struct intel_plane *plane;
2531
2532	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2533	struct intel_plane_state *plane_state;
2534	enum plane_id plane_id = plane->id;
2535
2536	if (skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb[plane_id],
2537	e2: &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2538	skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2539	e2: &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2540	continue;
2541
2542	plane_state = intel_atomic_get_plane_state(state, plane);
2543	if (IS_ERR(ptr: plane_state))
2544	return PTR_ERR(ptr: plane_state);
2545
2546	new_crtc_state->update_planes |= BIT(plane_id);
2547	new_crtc_state->async_flip_planes = 0;
2548	new_crtc_state->do_async_flip = false;
2549	}
2550
2551	return 0;
2552	}
2553
2554	static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2555	{
2556	struct drm_i915_private *i915 = to_i915(dev: dbuf_state->base.state->base.dev);
2557	u8 enabled_slices;
2558	enum pipe pipe;
2559
2560	/*
2561	* FIXME: For now we always enable slice S1 as per
2562	* the Bspec display initialization sequence.
2563	*/
2564	enabled_slices = BIT(DBUF_S1);
2565
2566	for_each_pipe(i915, pipe)
2567	enabled_slices |= dbuf_state->slices[pipe];
2568
2569	return enabled_slices;
2570	}
2571
2572	static int
2573	skl_compute_ddb(struct intel_atomic_state *state)
2574	{
2575	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
2576	const struct intel_dbuf_state *old_dbuf_state;
2577	struct intel_dbuf_state *new_dbuf_state = NULL;
2578	const struct intel_crtc_state *old_crtc_state;
2579	struct intel_crtc_state *new_crtc_state;
2580	struct intel_crtc *crtc;
2581	int ret, i;
2582
2583	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2584	new_dbuf_state = intel_atomic_get_dbuf_state(state);
2585	if (IS_ERR(ptr: new_dbuf_state))
2586	return PTR_ERR(ptr: new_dbuf_state);
2587
2588	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2589	break;
2590	}
2591
2592	if (!new_dbuf_state)
2593	return 0;
2594
2595	new_dbuf_state->active_pipes =
2596	intel_calc_active_pipes(state, active_pipes: old_dbuf_state->active_pipes);
2597
2598	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2599	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2600	if (ret)
2601	return ret;
2602	}
2603
2604	if (HAS_MBUS_JOINING(i915))
2605	new_dbuf_state->joined_mbus =
2606	adlp_check_mbus_joined(active_pipes: new_dbuf_state->active_pipes);
2607
2608	for_each_intel_crtc(&i915->drm, crtc) {
2609	enum pipe pipe = crtc->pipe;
2610
2611	new_dbuf_state->slices[pipe] =
2612	skl_compute_dbuf_slices(crtc, active_pipes: new_dbuf_state->active_pipes,
2613	join_mbus: new_dbuf_state->joined_mbus);
2614
2615	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2616	continue;
2617
2618	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2619	if (ret)
2620	return ret;
2621	}
2622
2623	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(dbuf_state: new_dbuf_state);
2624
2625	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2626	old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2627	ret = intel_atomic_serialize_global_state(obj_state: &new_dbuf_state->base);
2628	if (ret)
2629	return ret;
2630
2631	if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2632	/* TODO: Implement vblank synchronized MBUS joining changes */
2633	ret = intel_modeset_all_pipes_late(state, reason: "MBUS joining change");
2634	if (ret)
2635	return ret;
2636	}
2637
2638	drm_dbg_kms(&i915->drm,
2639	"Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2640	old_dbuf_state->enabled_slices,
2641	new_dbuf_state->enabled_slices,
2642	DISPLAY_INFO(i915)->dbuf.slice_mask,
2643	str_yes_no(old_dbuf_state->joined_mbus),
2644	str_yes_no(new_dbuf_state->joined_mbus));
2645	}
2646
2647	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2648	enum pipe pipe = crtc->pipe;
2649
2650	new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state: new_crtc_state);
2651
2652	if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2653	continue;
2654
2655	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2656	if (ret)
2657	return ret;
2658	}
2659
2660	for_each_intel_crtc(&i915->drm, crtc) {
2661	ret = skl_crtc_allocate_ddb(state, crtc);
2662	if (ret)
2663	return ret;
2664	}
2665
2666	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2667	new_crtc_state, i) {
2668	ret = skl_crtc_allocate_plane_ddb(state, crtc);
2669	if (ret)
2670	return ret;
2671
2672	ret = skl_ddb_add_affected_planes(old_crtc_state,
2673	new_crtc_state);
2674	if (ret)
2675	return ret;
2676	}
2677
2678	return 0;
2679	}
2680
2681	static char enast(bool enable)
2682	{
2683	return enable ? '*' : ' ';
2684	}
2685
2686	static void
2687	skl_print_wm_changes(struct intel_atomic_state *state)
2688	{
2689	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
2690	const struct intel_crtc_state *old_crtc_state;
2691	const struct intel_crtc_state *new_crtc_state;
2692	struct intel_plane *plane;
2693	struct intel_crtc *crtc;
2694	int i;
2695
2696	if (!drm_debug_enabled(DRM_UT_KMS))
2697	return;
2698
2699	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2700	new_crtc_state, i) {
2701	const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2702
2703	old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2704	new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2705
2706	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2707	enum plane_id plane_id = plane->id;
2708	const struct skl_ddb_entry *old, *new;
2709
2710	old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2711	new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2712
2713	if (skl_ddb_entry_equal(e1: old, e2: new))
2714	continue;
2715
2716	drm_dbg_kms(&i915->drm,
2717	"[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2718	plane->base.base.id, plane->base.name,
2719	old->start, old->end, new->start, new->end,
2720	skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2721	}
2722
2723	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2724	enum plane_id plane_id = plane->id;
2725	const struct skl_plane_wm *old_wm, *new_wm;
2726
2727	old_wm = &old_pipe_wm->planes[plane_id];
2728	new_wm = &new_pipe_wm->planes[plane_id];
2729
2730	if (skl_plane_wm_equals(i915, wm1: old_wm, wm2: new_wm))
2731	continue;
2732
2733	drm_dbg_kms(&i915->drm,
2734	"[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2735	" -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2736	plane->base.base.id, plane->base.name,
2737	enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2738	enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2739	enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2740	enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2741	enast(old_wm->trans_wm.enable),
2742	enast(old_wm->sagv.wm0.enable),
2743	enast(old_wm->sagv.trans_wm.enable),
2744	enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2745	enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2746	enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2747	enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2748	enast(new_wm->trans_wm.enable),
2749	enast(new_wm->sagv.wm0.enable),
2750	enast(new_wm->sagv.trans_wm.enable));
2751
2752	drm_dbg_kms(&i915->drm,
2753	"[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2754	" -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2755	plane->base.base.id, plane->base.name,
2756	enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2757	enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2758	enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2759	enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2760	enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2761	enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2762	enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2763	enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2764	enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2765	enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2766	enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2767	enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2768	enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2769	enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2770	enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2771	enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2772	enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2773	enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2774	enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2775	enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2776	enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2777	enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2778
2779	drm_dbg_kms(&i915->drm,
2780	"[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2781	" -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2782	plane->base.base.id, plane->base.name,
2783	old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2784	old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2785	old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2786	old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2787	old_wm->trans_wm.blocks,
2788	old_wm->sagv.wm0.blocks,
2789	old_wm->sagv.trans_wm.blocks,
2790	new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2791	new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2792	new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2793	new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2794	new_wm->trans_wm.blocks,
2795	new_wm->sagv.wm0.blocks,
2796	new_wm->sagv.trans_wm.blocks);
2797
2798	drm_dbg_kms(&i915->drm,
2799	"[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2800	" -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2801	plane->base.base.id, plane->base.name,
2802	old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2803	old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2804	old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2805	old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2806	old_wm->trans_wm.min_ddb_alloc,
2807	old_wm->sagv.wm0.min_ddb_alloc,
2808	old_wm->sagv.trans_wm.min_ddb_alloc,
2809	new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2810	new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2811	new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2812	new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2813	new_wm->trans_wm.min_ddb_alloc,
2814	new_wm->sagv.wm0.min_ddb_alloc,
2815	new_wm->sagv.trans_wm.min_ddb_alloc);
2816	}
2817	}
2818	}
2819
2820	static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2821	const struct skl_pipe_wm *old_pipe_wm,
2822	const struct skl_pipe_wm *new_pipe_wm)
2823	{
2824	struct drm_i915_private *i915 = to_i915(dev: plane->base.dev);
2825	int level;
2826
2827	for (level = 0; level < i915->display.wm.num_levels; level++) {
2828	/*
2829	* We don't check uv_wm as the hardware doesn't actually
2830	* use it. It only gets used for calculating the required
2831	* ddb allocation.
2832	*/
2833	if (!skl_wm_level_equals(l1: skl_plane_wm_level(pipe_wm: old_pipe_wm, plane_id: plane->id, level),
2834	l2: skl_plane_wm_level(pipe_wm: new_pipe_wm, plane_id: plane->id, level)))
2835	return false;
2836	}
2837
2838	if (HAS_HW_SAGV_WM(i915)) {
2839	const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2840	const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2841
2842	if (!skl_wm_level_equals(l1: &old_wm->sagv.wm0, l2: &new_wm->sagv.wm0) ||
2843	!skl_wm_level_equals(l1: &old_wm->sagv.trans_wm, l2: &new_wm->sagv.trans_wm))
2844	return false;
2845	}
2846
2847	return skl_wm_level_equals(l1: skl_plane_trans_wm(pipe_wm: old_pipe_wm, plane_id: plane->id),
2848	l2: skl_plane_trans_wm(pipe_wm: new_pipe_wm, plane_id: plane->id));
2849	}
2850
2851	/*
2852	* To make sure the cursor watermark registers are always consistent
2853	* with our computed state the following scenario needs special
2854	* treatment:
2855	*
2856	* 1. enable cursor
2857	* 2. move cursor entirely offscreen
2858	* 3. disable cursor
2859	*
2860	* Step 2. does call .disable_plane() but does not zero the watermarks
2861	* (since we consider an offscreen cursor still active for the purposes
2862	* of watermarks). Step 3. would not normally call .disable_plane()
2863	* because the actual plane visibility isn't changing, and we don't
2864	* deallocate the cursor ddb until the pipe gets disabled. So we must
2865	* force step 3. to call .disable_plane() to update the watermark
2866	* registers properly.
2867	*
2868	* Other planes do not suffer from this issues as their watermarks are
2869	* calculated based on the actual plane visibility. The only time this
2870	* can trigger for the other planes is during the initial readout as the
2871	* default value of the watermarks registers is not zero.
2872	*/
2873	static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2874	struct intel_crtc *crtc)
2875	{
2876	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
2877	const struct intel_crtc_state *old_crtc_state =
2878	intel_atomic_get_old_crtc_state(state, crtc);
2879	struct intel_crtc_state *new_crtc_state =
2880	intel_atomic_get_new_crtc_state(state, crtc);
2881	struct intel_plane *plane;
2882
2883	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2884	struct intel_plane_state *plane_state;
2885	enum plane_id plane_id = plane->id;
2886
2887	/*
2888	* Force a full wm update for every plane on modeset.
2889	* Required because the reset value of the wm registers
2890	* is non-zero, whereas we want all disabled planes to
2891	* have zero watermarks. So if we turn off the relevant
2892	* power well the hardware state will go out of sync
2893	* with the software state.
2894	*/
2895	if (!intel_crtc_needs_modeset(crtc_state: new_crtc_state) &&
2896	skl_plane_selected_wm_equals(plane,
2897	old_pipe_wm: &old_crtc_state->wm.skl.optimal,
2898	new_pipe_wm: &new_crtc_state->wm.skl.optimal))
2899	continue;
2900
2901	plane_state = intel_atomic_get_plane_state(state, plane);
2902	if (IS_ERR(ptr: plane_state))
2903	return PTR_ERR(ptr: plane_state);
2904
2905	new_crtc_state->update_planes |= BIT(plane_id);
2906	new_crtc_state->async_flip_planes = 0;
2907	new_crtc_state->do_async_flip = false;
2908	}
2909
2910	return 0;
2911	}
2912
2913	/*
2914	* If Fixed Refresh Rate:
2915	* Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from
2916	* watermark level1 and up and above. If watermark level 1 is
2917	* invalid program it with all 1's.
2918	* Program PKG_C_LATENCY Added Wake Time = DSB execution time
2919	* If Variable Refresh Rate:
2920	* Program DEEP PKG_C_LATENCY Pkg C with all 1's.
2921	* Program PKG_C_LATENCY Added Wake Time = 0
2922	*/
2923	static void
2924	skl_program_dpkgc_latency(struct drm_i915_private *i915, bool vrr_enabled)
2925	{
2926	u32 max_latency = 0;
2927	u32 clear = 0, val = 0;
2928	u32 added_wake_time = 0;
2929
2930	if (DISPLAY_VER(i915) < 20)
2931	return;
2932
2933	if (vrr_enabled) {
2934	max_latency = LNL_PKG_C_LATENCY_MASK;
2935	added_wake_time = 0;
2936	} else {
2937	max_latency = skl_watermark_max_latency(i915, initial_wm_level: 1);
2938	if (max_latency == 0)
2939	max_latency = LNL_PKG_C_LATENCY_MASK;
2940	added_wake_time = DSB_EXE_TIME +
2941	i915->display.sagv.block_time_us;
2942	}
2943
2944	clear |= LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK;
2945	val |= REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, max_latency);
2946	val |= REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time);
2947
2948	intel_uncore_rmw(uncore: &i915->uncore, LNL_PKG_C_LATENCY, clear, set: val);
2949	}
2950
2951	static int
2952	skl_compute_wm(struct intel_atomic_state *state)
2953	{
2954	struct intel_crtc *crtc;
2955	struct intel_crtc_state __maybe_unused *new_crtc_state;
2956	int ret, i;
2957	bool vrr_enabled = false;
2958
2959	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2960	ret = skl_build_pipe_wm(state, crtc);
2961	if (ret)
2962	return ret;
2963	}
2964
2965	ret = skl_compute_ddb(state);
2966	if (ret)
2967	return ret;
2968
2969	ret = intel_compute_sagv_mask(state);
2970	if (ret)
2971	return ret;
2972
2973	/*
2974	* skl_compute_ddb() will have adjusted the final watermarks
2975	* based on how much ddb is available. Now we can actually
2976	* check if the final watermarks changed.
2977	*/
2978	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2979	ret = skl_wm_add_affected_planes(state, crtc);
2980	if (ret)
2981	return ret;
2982
2983	if (new_crtc_state->vrr.enable)
2984	vrr_enabled = true;
2985	}
2986
2987	skl_program_dpkgc_latency(i915: to_i915(dev: state->base.dev), vrr_enabled);
2988
2989	skl_print_wm_changes(state);
2990
2991	return 0;
2992	}
2993
2994	static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2995	{
2996	level->enable = val & PLANE_WM_EN;
2997	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2998	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2999	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
3000	}
3001
3002	static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
3003	struct skl_pipe_wm *out)
3004	{
3005	struct drm_i915_private *i915 = to_i915(dev: crtc->base.dev);
3006	enum pipe pipe = crtc->pipe;
3007	enum plane_id plane_id;
3008	int level;
3009	u32 val;
3010
3011	for_each_plane_id_on_crtc(crtc, plane_id) {
3012	struct skl_plane_wm *wm = &out->planes[plane_id];
3013
3014	for (level = 0; level < i915->display.wm.num_levels; level++) {
3015	if (plane_id != PLANE_CURSOR)
3016	val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
3017	else
3018	val = intel_de_read(i915, CUR_WM(pipe, level));
3019
3020	skl_wm_level_from_reg_val(val, level: &wm->wm[level]);
3021	}
3022
3023	if (plane_id != PLANE_CURSOR)
3024	val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
3025	else
3026	val = intel_de_read(i915, CUR_WM_TRANS(pipe));
3027
3028	skl_wm_level_from_reg_val(val, level: &wm->trans_wm);
3029
3030	if (HAS_HW_SAGV_WM(i915)) {
3031	if (plane_id != PLANE_CURSOR)
3032	val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
3033	else
3034	val = intel_de_read(i915, CUR_WM_SAGV(pipe));
3035
3036	skl_wm_level_from_reg_val(val, level: &wm->sagv.wm0);
3037
3038	if (plane_id != PLANE_CURSOR)
3039	val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
3040	else
3041	val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
3042
3043	skl_wm_level_from_reg_val(val, level: &wm->sagv.trans_wm);
3044	} else if (DISPLAY_VER(i915) >= 12) {
3045	wm->sagv.wm0 = wm->wm[0];
3046	wm->sagv.trans_wm = wm->trans_wm;
3047	}
3048	}
3049	}
3050
3051	static void skl_wm_get_hw_state(struct drm_i915_private *i915)
3052	{
3053	struct intel_dbuf_state *dbuf_state =
3054	to_intel_dbuf_state(i915->display.dbuf.obj.state);
3055	struct intel_crtc *crtc;
3056
3057	if (HAS_MBUS_JOINING(i915))
3058	dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
3059
3060	for_each_intel_crtc(&i915->drm, crtc) {
3061	struct intel_crtc_state *crtc_state =
3062	to_intel_crtc_state(crtc->base.state);
3063	enum pipe pipe = crtc->pipe;
3064	unsigned int mbus_offset;
3065	enum plane_id plane_id;
3066	u8 slices;
3067
3068	memset(&crtc_state->wm.skl.optimal, 0,
3069	sizeof(crtc_state->wm.skl.optimal));
3070	if (crtc_state->hw.active)
3071	skl_pipe_wm_get_hw_state(crtc, out: &crtc_state->wm.skl.optimal);
3072	crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3073
3074	memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3075
3076	for_each_plane_id_on_crtc(crtc, plane_id) {
3077	struct skl_ddb_entry *ddb =
3078	&crtc_state->wm.skl.plane_ddb[plane_id];
3079	struct skl_ddb_entry *ddb_y =
3080	&crtc_state->wm.skl.plane_ddb_y[plane_id];
3081
3082	if (!crtc_state->hw.active)
3083	continue;
3084
3085	skl_ddb_get_hw_plane_state(i915, pipe: crtc->pipe,
3086	plane_id, ddb, ddb_y);
3087
3088	skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb);
3089	skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb_y);
3090	}
3091
3092	dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3093
3094	/*
3095	* Used for checking overlaps, so we need absolute
3096	* offsets instead of MBUS relative offsets.
3097	*/
3098	slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3099	join_mbus: dbuf_state->joined_mbus);
3100	mbus_offset = mbus_ddb_offset(i915, slice_mask: slices);
3101	crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3102	crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3103
3104	/* The slices actually used by the planes on the pipe */
3105	dbuf_state->slices[pipe] =
3106	skl_ddb_dbuf_slice_mask(i915, entry: &crtc_state->wm.skl.ddb);
3107
3108	drm_dbg_kms(&i915->drm,
3109	"[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3110	crtc->base.base.id, crtc->base.name,
3111	dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3112	dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3113	str_yes_no(dbuf_state->joined_mbus));
3114	}
3115
3116	dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3117	}
3118
3119	static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3120	{
3121	const struct intel_dbuf_state *dbuf_state =
3122	to_intel_dbuf_state(i915->display.dbuf.obj.state);
3123	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3124	struct intel_crtc *crtc;
3125
3126	for_each_intel_crtc(&i915->drm, crtc) {
3127	const struct intel_crtc_state *crtc_state =
3128	to_intel_crtc_state(crtc->base.state);
3129
3130	entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3131	}
3132
3133	for_each_intel_crtc(&i915->drm, crtc) {
3134	const struct intel_crtc_state *crtc_state =
3135	to_intel_crtc_state(crtc->base.state);
3136	u8 slices;
3137
3138	slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3139	join_mbus: dbuf_state->joined_mbus);
3140	if (dbuf_state->slices[crtc->pipe] & ~slices)
3141	return true;
3142
3143	if (skl_ddb_allocation_overlaps(ddb: &crtc_state->wm.skl.ddb, entries,
3144	num_entries: I915_MAX_PIPES, ignore_idx: crtc->pipe))
3145	return true;
3146	}
3147
3148	return false;
3149	}
3150
3151	static void skl_wm_sanitize(struct drm_i915_private *i915)
3152	{
3153	struct intel_crtc *crtc;
3154
3155	/*
3156	* On TGL/RKL (at least) the BIOS likes to assign the planes
3157	* to the wrong DBUF slices. This will cause an infinite loop
3158	* in skl_commit_modeset_enables() as it can't find a way to
3159	* transition between the old bogus DBUF layout to the new
3160	* proper DBUF layout without DBUF allocation overlaps between
3161	* the planes (which cannot be allowed or else the hardware
3162	* may hang). If we detect a bogus DBUF layout just turn off
3163	* all the planes so that skl_commit_modeset_enables() can
3164	* simply ignore them.
3165	*/
3166	if (!skl_dbuf_is_misconfigured(i915))
3167	return;
3168
3169	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3170
3171	for_each_intel_crtc(&i915->drm, crtc) {
3172	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3173	const struct intel_plane_state *plane_state =
3174	to_intel_plane_state(plane->base.state);
3175	struct intel_crtc_state *crtc_state =
3176	to_intel_crtc_state(crtc->base.state);
3177
3178	if (plane_state->uapi.visible)
3179	intel_plane_disable_noatomic(crtc, plane);
3180
3181	drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3182
3183	memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3184	}
3185	}
3186
3187	static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3188	{
3189	skl_wm_get_hw_state(i915);
3190	skl_wm_sanitize(i915);
3191	}
3192
3193	void intel_wm_state_verify(struct intel_atomic_state *state,
3194	struct intel_crtc *crtc)
3195	{
3196	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3197	const struct intel_crtc_state *new_crtc_state =
3198	intel_atomic_get_new_crtc_state(state, crtc);
3199	struct skl_hw_state {
3200	struct skl_ddb_entry ddb[I915_MAX_PLANES];
3201	struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3202	struct skl_pipe_wm wm;
3203	} *hw;
3204	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3205	struct intel_plane *plane;
3206	u8 hw_enabled_slices;
3207	int level;
3208
3209	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3210	return;
3211
3212	hw = kzalloc(size: sizeof(*hw), GFP_KERNEL);
3213	if (!hw)
3214	return;
3215
3216	skl_pipe_wm_get_hw_state(crtc, out: &hw->wm);
3217
3218	skl_pipe_ddb_get_hw_state(crtc, ddb: hw->ddb, ddb_y: hw->ddb_y);
3219
3220	hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3221
3222	if (DISPLAY_VER(i915) >= 11 &&
3223	hw_enabled_slices != i915->display.dbuf.enabled_slices)
3224	drm_err(&i915->drm,
3225	"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3226	i915->display.dbuf.enabled_slices,
3227	hw_enabled_slices);
3228
3229	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3230	const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3231	const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3232
3233	/* Watermarks */
3234	for (level = 0; level < i915->display.wm.num_levels; level++) {
3235	hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3236	sw_wm_level = skl_plane_wm_level(pipe_wm: sw_wm, plane_id: plane->id, level);
3237
3238	if (skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level))
3239	continue;
3240
3241	drm_err(&i915->drm,
3242	"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3243	plane->base.base.id, plane->base.name, level,
3244	sw_wm_level->enable,
3245	sw_wm_level->blocks,
3246	sw_wm_level->lines,
3247	hw_wm_level->enable,
3248	hw_wm_level->blocks,
3249	hw_wm_level->lines);
3250	}
3251
3252	hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3253	sw_wm_level = skl_plane_trans_wm(pipe_wm: sw_wm, plane_id: plane->id);
3254
3255	if (!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3256	drm_err(&i915->drm,
3257	"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3258	plane->base.base.id, plane->base.name,
3259	sw_wm_level->enable,
3260	sw_wm_level->blocks,
3261	sw_wm_level->lines,
3262	hw_wm_level->enable,
3263	hw_wm_level->blocks,
3264	hw_wm_level->lines);
3265	}
3266
3267	hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3268	sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3269
3270	if (HAS_HW_SAGV_WM(i915) &&
3271	!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3272	drm_err(&i915->drm,
3273	"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3274	plane->base.base.id, plane->base.name,
3275	sw_wm_level->enable,
3276	sw_wm_level->blocks,
3277	sw_wm_level->lines,
3278	hw_wm_level->enable,
3279	hw_wm_level->blocks,
3280	hw_wm_level->lines);
3281	}
3282
3283	hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3284	sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3285
3286	if (HAS_HW_SAGV_WM(i915) &&
3287	!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3288	drm_err(&i915->drm,
3289	"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3290	plane->base.base.id, plane->base.name,
3291	sw_wm_level->enable,
3292	sw_wm_level->blocks,
3293	sw_wm_level->lines,
3294	hw_wm_level->enable,
3295	hw_wm_level->blocks,
3296	hw_wm_level->lines);
3297	}
3298
3299	/* DDB */
3300	hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3301	sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3302
3303	if (!skl_ddb_entry_equal(e1: hw_ddb_entry, e2: sw_ddb_entry)) {
3304	drm_err(&i915->drm,
3305	"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3306	plane->base.base.id, plane->base.name,
3307	sw_ddb_entry->start, sw_ddb_entry->end,
3308	hw_ddb_entry->start, hw_ddb_entry->end);
3309	}
3310	}
3311
3312	kfree(objp: hw);
3313	}
3314
3315	bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3316	{
3317	return i915->display.wm.ipc_enabled;
3318	}
3319
3320	void skl_watermark_ipc_update(struct drm_i915_private *i915)
3321	{
3322	if (!HAS_IPC(i915))
3323	return;
3324
3325	intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3326	set: skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3327	}
3328
3329	static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3330	{
3331	/* Display WA #0477 WaDisableIPC: skl */
3332	if (IS_SKYLAKE(i915))
3333	return false;
3334
3335	/* Display WA #1141: SKL:all KBL:all CFL */
3336	if (IS_KABYLAKE(i915) ||
3337	IS_COFFEELAKE(i915) ||
3338	IS_COMETLAKE(i915))
3339	return i915->dram_info.symmetric_memory;
3340
3341	return true;
3342	}
3343
3344	void skl_watermark_ipc_init(struct drm_i915_private *i915)
3345	{
3346	if (!HAS_IPC(i915))
3347	return;
3348
3349	i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3350
3351	skl_watermark_ipc_update(i915);
3352	}
3353
3354	static void
3355	adjust_wm_latency(struct drm_i915_private *i915,
3356	u16 wm[], int num_levels, int read_latency)
3357	{
3358	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3359	int i, level;
3360
3361	/*
3362	* If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3363	* need to be disabled. We make sure to sanitize the values out
3364	* of the punit to satisfy this requirement.
3365	*/
3366	for (level = 1; level < num_levels; level++) {
3367	if (wm[level] == 0) {
3368	for (i = level + 1; i < num_levels; i++)
3369	wm[i] = 0;
3370
3371	num_levels = level;
3372	break;
3373	}
3374	}
3375
3376	/*
3377	* WaWmMemoryReadLatency
3378	*
3379	* punit doesn't take into account the read latency so we need
3380	* to add proper adjustement to each valid level we retrieve
3381	* from the punit when level 0 response data is 0us.
3382	*/
3383	if (wm[0] == 0) {
3384	for (level = 0; level < num_levels; level++)
3385	wm[level] += read_latency;
3386	}
3387
3388	/*
3389	* WA Level-0 adjustment for 16GB DIMMs: SKL+
3390	* If we could not get dimm info enable this WA to prevent from
3391	* any underrun. If not able to get Dimm info assume 16GB dimm
3392	* to avoid any underrun.
3393	*/
3394	if (wm_lv_0_adjust_needed)
3395	wm[0] += 1;
3396	}
3397
3398	static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3399	{
3400	int num_levels = i915->display.wm.num_levels;
3401	u32 val;
3402
3403	val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3404	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3405	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3406
3407	val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3408	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3409	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3410
3411	val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3412	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3413	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3414
3415	adjust_wm_latency(i915, wm, num_levels, read_latency: 6);
3416	}
3417
3418	static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3419	{
3420	int num_levels = i915->display.wm.num_levels;
3421	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3422	int mult = IS_DG2(i915) ? 2 : 1;
3423	u32 val;
3424	int ret;
3425
3426	/* read the first set of memory latencies[0:3] */
3427	val = 0; /* data0 to be programmed to 0 for first set */
3428	ret = snb_pcode_read(uncore: &i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3429	if (ret) {
3430	drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3431	return;
3432	}
3433
3434	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3435	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3436	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3437	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3438
3439	/* read the second set of memory latencies[4:7] */
3440	val = 1; /* data0 to be programmed to 1 for second set */
3441	ret = snb_pcode_read(uncore: &i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3442	if (ret) {
3443	drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3444	return;
3445	}
3446
3447	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3448	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3449	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3450	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3451
3452	adjust_wm_latency(i915, wm, num_levels, read_latency);
3453	}
3454
3455	static void skl_setup_wm_latency(struct drm_i915_private *i915)
3456	{
3457	if (HAS_HW_SAGV_WM(i915))
3458	i915->display.wm.num_levels = 6;
3459	else
3460	i915->display.wm.num_levels = 8;
3461
3462	if (DISPLAY_VER(i915) >= 14)
3463	mtl_read_wm_latency(i915, wm: i915->display.wm.skl_latency);
3464	else
3465	skl_read_wm_latency(i915, wm: i915->display.wm.skl_latency);
3466
3467	intel_print_wm_latency(i915, name: "Gen9 Plane", wm: i915->display.wm.skl_latency);
3468	}
3469
3470	static const struct intel_wm_funcs skl_wm_funcs = {
3471	.compute_global_watermarks = skl_compute_wm,
3472	.get_hw_state = skl_wm_get_hw_state_and_sanitize,
3473	};
3474
3475	void skl_wm_init(struct drm_i915_private *i915)
3476	{
3477	intel_sagv_init(i915);
3478
3479	skl_setup_wm_latency(i915);
3480
3481	i915->display.funcs.wm = &skl_wm_funcs;
3482	}
3483
3484	static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3485	{
3486	struct intel_dbuf_state *dbuf_state;
3487
3488	dbuf_state = kmemdup(p: obj->state, size: sizeof(*dbuf_state), GFP_KERNEL);
3489	if (!dbuf_state)
3490	return NULL;
3491
3492	return &dbuf_state->base;
3493	}
3494
3495	static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3496	struct intel_global_state *state)
3497	{
3498	kfree(objp: state);
3499	}
3500
3501	static const struct intel_global_state_funcs intel_dbuf_funcs = {
3502	.atomic_duplicate_state = intel_dbuf_duplicate_state,
3503	.atomic_destroy_state = intel_dbuf_destroy_state,
3504	};
3505
3506	struct intel_dbuf_state *
3507	intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3508	{
3509	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3510	struct intel_global_state *dbuf_state;
3511
3512	dbuf_state = intel_atomic_get_global_obj_state(state, obj: &i915->display.dbuf.obj);
3513	if (IS_ERR(ptr: dbuf_state))
3514	return ERR_CAST(ptr: dbuf_state);
3515
3516	return to_intel_dbuf_state(dbuf_state);
3517	}
3518
3519	int intel_dbuf_init(struct drm_i915_private *i915)
3520	{
3521	struct intel_dbuf_state *dbuf_state;
3522
3523	dbuf_state = kzalloc(size: sizeof(*dbuf_state), GFP_KERNEL);
3524	if (!dbuf_state)
3525	return -ENOMEM;
3526
3527	intel_atomic_global_obj_init(dev_priv: i915, obj: &i915->display.dbuf.obj,
3528	state: &dbuf_state->base, funcs: &intel_dbuf_funcs);
3529
3530	return 0;
3531	}
3532
3533	/*
3534	* Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3535	* update the request state of all DBUS slices.
3536	*/
3537	static void update_mbus_pre_enable(struct intel_atomic_state *state)
3538	{
3539	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3540	u32 mbus_ctl, dbuf_min_tracker_val;
3541	enum dbuf_slice slice;
3542	const struct intel_dbuf_state *dbuf_state =
3543	intel_atomic_get_new_dbuf_state(state);
3544
3545	if (!HAS_MBUS_JOINING(i915))
3546	return;
3547
3548	/*
3549	* TODO: Implement vblank synchronized MBUS joining changes.
3550	* Must be properly coordinated with dbuf reprogramming.
3551	*/
3552	if (dbuf_state->joined_mbus) {
3553	mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3554	MBUS_JOIN_PIPE_SELECT_NONE;
3555	dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3556	} else {
3557	mbus_ctl = MBUS_HASHING_MODE_2x2 |
3558	MBUS_JOIN_PIPE_SELECT_NONE;
3559	dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3560	}
3561
3562	intel_de_rmw(i915, MBUS_CTL,
3563	MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3564	MBUS_JOIN_PIPE_SELECT_MASK, set: mbus_ctl);
3565
3566	for_each_dbuf_slice(i915, slice)
3567	intel_de_rmw(i915, DBUF_CTL_S(slice),
3568	DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3569	set: dbuf_min_tracker_val);
3570	}
3571
3572	void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3573	{
3574	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3575	const struct intel_dbuf_state *new_dbuf_state =
3576	intel_atomic_get_new_dbuf_state(state);
3577	const struct intel_dbuf_state *old_dbuf_state =
3578	intel_atomic_get_old_dbuf_state(state);
3579
3580	if (!new_dbuf_state ||
3581	(new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3582	new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3583	return;
3584
3585	WARN_ON(!new_dbuf_state->base.changed);
3586
3587	update_mbus_pre_enable(state);
3588	gen9_dbuf_slices_update(dev_priv: i915,
3589	req_slices: old_dbuf_state->enabled_slices |
3590	new_dbuf_state->enabled_slices);
3591	}
3592
3593	void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3594	{
3595	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3596	const struct intel_dbuf_state *new_dbuf_state =
3597	intel_atomic_get_new_dbuf_state(state);
3598	const struct intel_dbuf_state *old_dbuf_state =
3599	intel_atomic_get_old_dbuf_state(state);
3600
3601	if (!new_dbuf_state ||
3602	(new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3603	new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3604	return;
3605
3606	WARN_ON(!new_dbuf_state->base.changed);
3607
3608	gen9_dbuf_slices_update(dev_priv: i915,
3609	req_slices: new_dbuf_state->enabled_slices);
3610	}
3611
3612	static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3613	{
3614	switch (pipe) {
3615	case PIPE_A:
3616	return !(active_pipes & BIT(PIPE_D));
3617	case PIPE_D:
3618	return !(active_pipes & BIT(PIPE_A));
3619	case PIPE_B:
3620	return !(active_pipes & BIT(PIPE_C));
3621	case PIPE_C:
3622	return !(active_pipes & BIT(PIPE_B));
3623	default: /* to suppress compiler warning */
3624	MISSING_CASE(pipe);
3625	break;
3626	}
3627
3628	return false;
3629	}
3630
3631	void intel_mbus_dbox_update(struct intel_atomic_state *state)
3632	{
3633	struct drm_i915_private *i915 = to_i915(dev: state->base.dev);
3634	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3635	const struct intel_crtc_state *new_crtc_state;
3636	const struct intel_crtc *crtc;
3637	u32 val = 0;
3638	int i;
3639
3640	if (DISPLAY_VER(i915) < 11)
3641	return;
3642
3643	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3644	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3645	if (!new_dbuf_state ||
3646	(new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3647	new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3648	return;
3649
3650	if (DISPLAY_VER(i915) >= 14)
3651	val |= MBUS_DBOX_I_CREDIT(2);
3652
3653	if (DISPLAY_VER(i915) >= 12) {
3654	val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3655	val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3656	val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3657	}
3658
3659	if (DISPLAY_VER(i915) >= 14)
3660	val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3661	MBUS_DBOX_A_CREDIT(8);
3662	else if (IS_ALDERLAKE_P(i915))
3663	/* Wa_22010947358:adl-p */
3664	val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3665	MBUS_DBOX_A_CREDIT(4);
3666	else
3667	val |= MBUS_DBOX_A_CREDIT(2);
3668
3669	if (DISPLAY_VER(i915) >= 14) {
3670	val |= MBUS_DBOX_B_CREDIT(0xA);
3671	} else if (IS_ALDERLAKE_P(i915)) {
3672	val |= MBUS_DBOX_BW_CREDIT(2);
3673	val |= MBUS_DBOX_B_CREDIT(8);
3674	} else if (DISPLAY_VER(i915) >= 12) {
3675	val |= MBUS_DBOX_BW_CREDIT(2);
3676	val |= MBUS_DBOX_B_CREDIT(12);
3677	} else {
3678	val |= MBUS_DBOX_BW_CREDIT(1);
3679	val |= MBUS_DBOX_B_CREDIT(8);
3680	}
3681
3682	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3683	u32 pipe_val = val;
3684
3685	if (!new_crtc_state->hw.active)
3686	continue;
3687
3688	if (DISPLAY_VER(i915) >= 14) {
3689	if (xelpdp_is_only_pipe_per_dbuf_bank(pipe: crtc->pipe,
3690	active_pipes: new_dbuf_state->active_pipes))
3691	pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3692	else
3693	pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3694	}
3695
3696	intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), val: pipe_val);
3697	}
3698	}
3699
3700	static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3701	{
3702	struct drm_i915_private *i915 = m->private;
3703
3704	seq_printf(m, fmt: "Isochronous Priority Control: %s\n",
3705	str_yes_no(v: skl_watermark_ipc_enabled(i915)));
3706	return 0;
3707	}
3708
3709	static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3710	{
3711	struct drm_i915_private *i915 = inode->i_private;
3712
3713	return single_open(file, skl_watermark_ipc_status_show, i915);
3714	}
3715
3716	static ssize_t skl_watermark_ipc_status_write(struct file *file,
3717	const char __user *ubuf,
3718	size_t len, loff_t *offp)
3719	{
3720	struct seq_file *m = file->private_data;
3721	struct drm_i915_private *i915 = m->private;
3722	intel_wakeref_t wakeref;
3723	bool enable;
3724	int ret;
3725
3726	ret = kstrtobool_from_user(s: ubuf, count: len, res: &enable);
3727	if (ret < 0)
3728	return ret;
3729
3730	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3731	if (!skl_watermark_ipc_enabled(i915) && enable)
3732	drm_info(&i915->drm,
3733	"Enabling IPC: WM will be proper only after next commit\n");
3734	i915->display.wm.ipc_enabled = enable;
3735	skl_watermark_ipc_update(i915);
3736	}
3737
3738	return len;
3739	}
3740
3741	static const struct file_operations skl_watermark_ipc_status_fops = {
3742	.owner = THIS_MODULE,
3743	.open = skl_watermark_ipc_status_open,
3744	.read = seq_read,
3745	.llseek = seq_lseek,
3746	.release = single_release,
3747	.write = skl_watermark_ipc_status_write
3748	};
3749
3750	static int intel_sagv_status_show(struct seq_file *m, void *unused)
3751	{
3752	struct drm_i915_private *i915 = m->private;
3753	static const char * const sagv_status[] = {
3754	[I915_SAGV_UNKNOWN] = "unknown",
3755	[I915_SAGV_DISABLED] = "disabled",
3756	[I915_SAGV_ENABLED] = "enabled",
3757	[I915_SAGV_NOT_CONTROLLED] = "not controlled",
3758	};
3759
3760	seq_printf(m, fmt: "SAGV available: %s\n", str_yes_no(v: intel_has_sagv(i915)));
3761	seq_printf(m, fmt: "SAGV modparam: %s\n",
3762	str_enabled_disabled(v: i915->display.params.enable_sagv));
3763	seq_printf(m, fmt: "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3764	seq_printf(m, fmt: "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3765
3766	return 0;
3767	}
3768
3769	DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3770
3771	void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3772	{
3773	struct drm_minor *minor = i915->drm.primary;
3774
3775	if (HAS_IPC(i915))
3776	debugfs_create_file(name: "i915_ipc_status", mode: 0644, parent: minor->debugfs_root, data: i915,
3777	fops: &skl_watermark_ipc_status_fops);
3778
3779	if (HAS_SAGV(i915))
3780	debugfs_create_file(name: "i915_sagv_status", mode: 0444, parent: minor->debugfs_root, data: i915,
3781	fops: &intel_sagv_status_fops);
3782	}
3783
3784	unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level)
3785	{
3786	int level;
3787
3788	for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) {
3789	unsigned int latency = skl_wm_latency(i915, level, NULL);
3790
3791	if (latency)
3792	return latency;
3793	}
3794
3795	return 0;
3796	}
3797