1	/*
2	* Copyright (C) 2014 Red Hat
3	* Copyright (C) 2014 Intel Corp.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	* Authors:
24	* Rob Clark <robdclark@gmail.com>
25	* Daniel Vetter <daniel.vetter@ffwll.ch>
26	*/
27
28	#include <linux/dma-fence.h>
29	#include <linux/ktime.h>
30
31	#include <drm/drm_atomic.h>
32	#include <drm/drm_atomic_helper.h>
33	#include <drm/drm_atomic_uapi.h>
34	#include <drm/drm_blend.h>
35	#include <drm/drm_bridge.h>
36	#include <drm/drm_damage_helper.h>
37	#include <drm/drm_device.h>
38	#include <drm/drm_drv.h>
39	#include <drm/drm_framebuffer.h>
40	#include <drm/drm_gem_atomic_helper.h>
41	#include <drm/drm_print.h>
42	#include <drm/drm_self_refresh_helper.h>
43	#include <drm/drm_vblank.h>
44	#include <drm/drm_writeback.h>
45
46	#include "drm_crtc_helper_internal.h"
47	#include "drm_crtc_internal.h"
48
49	/**
50	* DOC: overview
51	*
52	* This helper library provides implementations of check and commit functions on
53	* top of the CRTC modeset helper callbacks and the plane helper callbacks. It
54	* also provides convenience implementations for the atomic state handling
55	* callbacks for drivers which don't need to subclass the drm core structures to
56	* add their own additional internal state.
57	*
58	* This library also provides default implementations for the check callback in
59	* drm_atomic_helper_check() and for the commit callback with
60	* drm_atomic_helper_commit(). But the individual stages and callbacks are
61	* exposed to allow drivers to mix and match and e.g. use the plane helpers only
62	* together with a driver private modeset implementation.
63	*
64	* This library also provides implementations for all the legacy driver
65	* interfaces on top of the atomic interface. See drm_atomic_helper_set_config(),
66	* drm_atomic_helper_disable_plane(), and the various functions to implement
67	* set_property callbacks. New drivers must not implement these functions
68	* themselves but must use the provided helpers.
69	*
70	* The atomic helper uses the same function table structures as all other
71	* modesetting helpers. See the documentation for &struct drm_crtc_helper_funcs,
72	* struct &drm_encoder_helper_funcs and &struct drm_connector_helper_funcs. It
73	* also shares the &struct drm_plane_helper_funcs function table with the plane
74	* helpers.
75	*/
76	static void
77	drm_atomic_helper_plane_changed(struct drm_atomic_state *state,
78	struct drm_plane_state *old_plane_state,
79	struct drm_plane_state *plane_state,
80	struct drm_plane *plane)
81	{
82	struct drm_crtc_state *crtc_state;
83
84	if (old_plane_state->crtc) {
85	crtc_state = drm_atomic_get_new_crtc_state(state,
86	crtc: old_plane_state->crtc);
87
88	if (WARN_ON(!crtc_state))
89	return;
90
91	crtc_state->planes_changed = true;
92	}
93
94	if (plane_state->crtc) {
95	crtc_state = drm_atomic_get_new_crtc_state(state, crtc: plane_state->crtc);
96
97	if (WARN_ON(!crtc_state))
98	return;
99
100	crtc_state->planes_changed = true;
101	}
102	}
103
104	static int handle_conflicting_encoders(struct drm_atomic_state *state,
105	bool disable_conflicting_encoders)
106	{
107	struct drm_connector_state *new_conn_state;
108	struct drm_connector *connector;
109	struct drm_connector_list_iter conn_iter;
110	struct drm_encoder *encoder;
111	unsigned int encoder_mask = 0;
112	int i, ret = 0;
113
114	/*
115	* First loop, find all newly assigned encoders from the connectors
116	* part of the state. If the same encoder is assigned to multiple
117	* connectors bail out.
118	*/
119	for_each_new_connector_in_state(state, connector, new_conn_state, i) {
120	const struct drm_connector_helper_funcs *funcs = connector->helper_private;
121	struct drm_encoder *new_encoder;
122
123	if (!new_conn_state->crtc)
124	continue;
125
126	if (funcs->atomic_best_encoder)
127	new_encoder = funcs->atomic_best_encoder(connector,
128	state);
129	else if (funcs->best_encoder)
130	new_encoder = funcs->best_encoder(connector);
131	else
132	new_encoder = drm_connector_get_single_encoder(connector);
133
134	if (new_encoder) {
135	if (encoder_mask & drm_encoder_mask(encoder: new_encoder)) {
136	drm_dbg_atomic(connector->dev,
137	"[ENCODER:%d:%s] on [CONNECTOR:%d:%s] already assigned\n",
138	new_encoder->base.id, new_encoder->name,
139	connector->base.id, connector->name);
140
141	return -EINVAL;
142	}
143
144	encoder_mask |= drm_encoder_mask(encoder: new_encoder);
145	}
146	}
147
148	if (!encoder_mask)
149	return 0;
150
151	/*
152	* Second loop, iterate over all connectors not part of the state.
153	*
154	* If a conflicting encoder is found and disable_conflicting_encoders
155	* is not set, an error is returned. Userspace can provide a solution
156	* through the atomic ioctl.
157	*
158	* If the flag is set conflicting connectors are removed from the CRTC
159	* and the CRTC is disabled if no encoder is left. This preserves
160	* compatibility with the legacy set_config behavior.
161	*/
162	drm_connector_list_iter_begin(dev: state->dev, iter: &conn_iter);
163	drm_for_each_connector_iter(connector, &conn_iter) {
164	struct drm_crtc_state *crtc_state;
165
166	if (drm_atomic_get_new_connector_state(state, connector))
167	continue;
168
169	encoder = connector->state->best_encoder;
170	if (!encoder || !(encoder_mask & drm_encoder_mask(encoder)))
171	continue;
172
173	if (!disable_conflicting_encoders) {
174	drm_dbg_atomic(connector->dev,
175	"[ENCODER:%d:%s] in use on [CRTC:%d:%s] by [CONNECTOR:%d:%s]\n",
176	encoder->base.id, encoder->name,
177	connector->state->crtc->base.id,
178	connector->state->crtc->name,
179	connector->base.id, connector->name);
180	ret = -EINVAL;
181	goto out;
182	}
183
184	new_conn_state = drm_atomic_get_connector_state(state, connector);
185	if (IS_ERR(ptr: new_conn_state)) {
186	ret = PTR_ERR(ptr: new_conn_state);
187	goto out;
188	}
189
190	drm_dbg_atomic(connector->dev,
191	"[ENCODER:%d:%s] in use on [CRTC:%d:%s], disabling [CONNECTOR:%d:%s]\n",
192	encoder->base.id, encoder->name,
193	new_conn_state->crtc->base.id, new_conn_state->crtc->name,
194	connector->base.id, connector->name);
195
196	crtc_state = drm_atomic_get_new_crtc_state(state, crtc: new_conn_state->crtc);
197
198	ret = drm_atomic_set_crtc_for_connector(conn_state: new_conn_state, NULL);
199	if (ret)
200	goto out;
201
202	if (!crtc_state->connector_mask) {
203	ret = drm_atomic_set_mode_prop_for_crtc(state: crtc_state,
204	NULL);
205	if (ret < 0)
206	goto out;
207
208	crtc_state->active = false;
209	}
210	}
211	out:
212	drm_connector_list_iter_end(iter: &conn_iter);
213
214	return ret;
215	}
216
217	static void
218	set_best_encoder(struct drm_atomic_state *state,
219	struct drm_connector_state *conn_state,
220	struct drm_encoder *encoder)
221	{
222	struct drm_crtc_state *crtc_state;
223	struct drm_crtc *crtc;
224
225	if (conn_state->best_encoder) {
226	/* Unset the encoder_mask in the old crtc state. */
227	crtc = conn_state->connector->state->crtc;
228
229	/* A NULL crtc is an error here because we should have
230	* duplicated a NULL best_encoder when crtc was NULL.
231	* As an exception restoring duplicated atomic state
232	* during resume is allowed, so don't warn when
233	* best_encoder is equal to encoder we intend to set.
234	*/
235	WARN_ON(!crtc && encoder != conn_state->best_encoder);
236	if (crtc) {
237	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
238
239	crtc_state->encoder_mask &=
240	~drm_encoder_mask(encoder: conn_state->best_encoder);
241	}
242	}
243
244	if (encoder) {
245	crtc = conn_state->crtc;
246	WARN_ON(!crtc);
247	if (crtc) {
248	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
249
250	crtc_state->encoder_mask |=
251	drm_encoder_mask(encoder);
252	}
253	}
254
255	conn_state->best_encoder = encoder;
256	}
257
258	static void
259	steal_encoder(struct drm_atomic_state *state,
260	struct drm_encoder *encoder)
261	{
262	struct drm_crtc_state *crtc_state;
263	struct drm_connector *connector;
264	struct drm_connector_state *old_connector_state, *new_connector_state;
265	int i;
266
267	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
268	struct drm_crtc *encoder_crtc;
269
270	if (new_connector_state->best_encoder != encoder)
271	continue;
272
273	encoder_crtc = old_connector_state->crtc;
274
275	drm_dbg_atomic(encoder->dev,
276	"[ENCODER:%d:%s] in use on [CRTC:%d:%s], stealing it\n",
277	encoder->base.id, encoder->name,
278	encoder_crtc->base.id, encoder_crtc->name);
279
280	set_best_encoder(state, conn_state: new_connector_state, NULL);
281
282	crtc_state = drm_atomic_get_new_crtc_state(state, crtc: encoder_crtc);
283	crtc_state->connectors_changed = true;
284
285	return;
286	}
287	}
288
289	static int
290	update_connector_routing(struct drm_atomic_state *state,
291	struct drm_connector *connector,
292	struct drm_connector_state *old_connector_state,
293	struct drm_connector_state *new_connector_state,
294	bool added_by_user)
295	{
296	const struct drm_connector_helper_funcs *funcs;
297	struct drm_encoder *new_encoder;
298	struct drm_crtc_state *crtc_state;
299
300	drm_dbg_atomic(connector->dev, "Updating routing for [CONNECTOR:%d:%s]\n",
301	connector->base.id, connector->name);
302
303	if (old_connector_state->crtc != new_connector_state->crtc) {
304	if (old_connector_state->crtc) {
305	crtc_state = drm_atomic_get_new_crtc_state(state, crtc: old_connector_state->crtc);
306	crtc_state->connectors_changed = true;
307	}
308
309	if (new_connector_state->crtc) {
310	crtc_state = drm_atomic_get_new_crtc_state(state, crtc: new_connector_state->crtc);
311	crtc_state->connectors_changed = true;
312	}
313	}
314
315	if (!new_connector_state->crtc) {
316	drm_dbg_atomic(connector->dev, "Disabling [CONNECTOR:%d:%s]\n",
317	connector->base.id, connector->name);
318
319	set_best_encoder(state, conn_state: new_connector_state, NULL);
320
321	return 0;
322	}
323
324	crtc_state = drm_atomic_get_new_crtc_state(state,
325	crtc: new_connector_state->crtc);
326	/*
327	* For compatibility with legacy users, we want to make sure that
328	* we allow DPMS On->Off modesets on unregistered connectors. Modesets
329	* which would result in anything else must be considered invalid, to
330	* avoid turning on new displays on dead connectors.
331	*
332	* Since the connector can be unregistered at any point during an
333	* atomic check or commit, this is racy. But that's OK: all we care
334	* about is ensuring that userspace can't do anything but shut off the
335	* display on a connector that was destroyed after it's been notified,
336	* not before.
337	*
338	* Additionally, we also want to ignore connector registration when
339	* we're trying to restore an atomic state during system resume since
340	* there's a chance the connector may have been destroyed during the
341	* process, but it's better to ignore that then cause
342	* drm_atomic_helper_resume() to fail.
343	*
344	* Last, we want to ignore connector registration when the connector
345	* was not pulled in the atomic state by user-space (ie, was pulled
346	* in by the driver, e.g. when updating a DP-MST stream).
347	*/
348	if (!state->duplicated && drm_connector_is_unregistered(connector) &&
349	added_by_user && crtc_state->active) {
350	drm_dbg_atomic(connector->dev,
351	"[CONNECTOR:%d:%s] is not registered\n",
352	connector->base.id, connector->name);
353	return -EINVAL;
354	}
355
356	funcs = connector->helper_private;
357
358	if (funcs->atomic_best_encoder)
359	new_encoder = funcs->atomic_best_encoder(connector, state);
360	else if (funcs->best_encoder)
361	new_encoder = funcs->best_encoder(connector);
362	else
363	new_encoder = drm_connector_get_single_encoder(connector);
364
365	if (!new_encoder) {
366	drm_dbg_atomic(connector->dev,
367	"No suitable encoder found for [CONNECTOR:%d:%s]\n",
368	connector->base.id, connector->name);
369	return -EINVAL;
370	}
371
372	if (!drm_encoder_crtc_ok(encoder: new_encoder, crtc: new_connector_state->crtc)) {
373	drm_dbg_atomic(connector->dev,
374	"[ENCODER:%d:%s] incompatible with [CRTC:%d:%s]\n",
375	new_encoder->base.id,
376	new_encoder->name,
377	new_connector_state->crtc->base.id,
378	new_connector_state->crtc->name);
379	return -EINVAL;
380	}
381
382	if (new_encoder == new_connector_state->best_encoder) {
383	set_best_encoder(state, conn_state: new_connector_state, encoder: new_encoder);
384
385	drm_dbg_atomic(connector->dev,
386	"[CONNECTOR:%d:%s] keeps [ENCODER:%d:%s], now on [CRTC:%d:%s]\n",
387	connector->base.id,
388	connector->name,
389	new_encoder->base.id,
390	new_encoder->name,
391	new_connector_state->crtc->base.id,
392	new_connector_state->crtc->name);
393
394	return 0;
395	}
396
397	steal_encoder(state, encoder: new_encoder);
398
399	set_best_encoder(state, conn_state: new_connector_state, encoder: new_encoder);
400
401	crtc_state->connectors_changed = true;
402
403	drm_dbg_atomic(connector->dev,
404	"[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d:%s]\n",
405	connector->base.id,
406	connector->name,
407	new_encoder->base.id,
408	new_encoder->name,
409	new_connector_state->crtc->base.id,
410	new_connector_state->crtc->name);
411
412	return 0;
413	}
414
415	static int
416	mode_fixup(struct drm_atomic_state *state)
417	{
418	struct drm_crtc *crtc;
419	struct drm_crtc_state *new_crtc_state;
420	struct drm_connector *connector;
421	struct drm_connector_state *new_conn_state;
422	int i;
423	int ret;
424
425	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
426	if (!new_crtc_state->mode_changed &&
427	!new_crtc_state->connectors_changed)
428	continue;
429
430	drm_mode_copy(dst: &new_crtc_state->adjusted_mode, src: &new_crtc_state->mode);
431	}
432
433	for_each_new_connector_in_state(state, connector, new_conn_state, i) {
434	const struct drm_encoder_helper_funcs *funcs;
435	struct drm_encoder *encoder;
436	struct drm_bridge *bridge;
437
438	WARN_ON(!!new_conn_state->best_encoder != !!new_conn_state->crtc);
439
440	if (!new_conn_state->crtc || !new_conn_state->best_encoder)
441	continue;
442
443	new_crtc_state =
444	drm_atomic_get_new_crtc_state(state, crtc: new_conn_state->crtc);
445
446	/*
447	* Each encoder has at most one connector (since we always steal
448	* it away), so we won't call ->mode_fixup twice.
449	*/
450	encoder = new_conn_state->best_encoder;
451	funcs = encoder->helper_private;
452
453	bridge = drm_bridge_chain_get_first_bridge(encoder);
454	ret = drm_atomic_bridge_chain_check(bridge,
455	crtc_state: new_crtc_state,
456	conn_state: new_conn_state);
457	if (ret) {
458	drm_dbg_atomic(encoder->dev, "Bridge atomic check failed\n");
459	return ret;
460	}
461
462	if (funcs && funcs->atomic_check) {
463	ret = funcs->atomic_check(encoder, new_crtc_state,
464	new_conn_state);
465	if (ret) {
466	drm_dbg_atomic(encoder->dev,
467	"[ENCODER:%d:%s] check failed\n",
468	encoder->base.id, encoder->name);
469	return ret;
470	}
471	} else if (funcs && funcs->mode_fixup) {
472	ret = funcs->mode_fixup(encoder, &new_crtc_state->mode,
473	&new_crtc_state->adjusted_mode);
474	if (!ret) {
475	drm_dbg_atomic(encoder->dev,
476	"[ENCODER:%d:%s] fixup failed\n",
477	encoder->base.id, encoder->name);
478	return -EINVAL;
479	}
480	}
481	}
482
483	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
484	const struct drm_crtc_helper_funcs *funcs;
485
486	if (!new_crtc_state->enable)
487	continue;
488
489	if (!new_crtc_state->mode_changed &&
490	!new_crtc_state->connectors_changed)
491	continue;
492
493	funcs = crtc->helper_private;
494	if (!funcs || !funcs->mode_fixup)
495	continue;
496
497	ret = funcs->mode_fixup(crtc, &new_crtc_state->mode,
498	&new_crtc_state->adjusted_mode);
499	if (!ret) {
500	drm_dbg_atomic(crtc->dev, "[CRTC:%d:%s] fixup failed\n",
501	crtc->base.id, crtc->name);
502	return -EINVAL;
503	}
504	}
505
506	return 0;
507	}
508
509	static enum drm_mode_status mode_valid_path(struct drm_connector *connector,
510	struct drm_encoder *encoder,
511	struct drm_crtc *crtc,
512	const struct drm_display_mode *mode)
513	{
514	struct drm_bridge *bridge;
515	enum drm_mode_status ret;
516
517	ret = drm_encoder_mode_valid(encoder, mode);
518	if (ret != MODE_OK) {
519	drm_dbg_atomic(encoder->dev,
520	"[ENCODER:%d:%s] mode_valid() failed\n",
521	encoder->base.id, encoder->name);
522	return ret;
523	}
524
525	bridge = drm_bridge_chain_get_first_bridge(encoder);
526	ret = drm_bridge_chain_mode_valid(bridge, info: &connector->display_info,
527	mode);
528	if (ret != MODE_OK) {
529	drm_dbg_atomic(encoder->dev, "[BRIDGE] mode_valid() failed\n");
530	return ret;
531	}
532
533	ret = drm_crtc_mode_valid(crtc, mode);
534	if (ret != MODE_OK) {
535	drm_dbg_atomic(encoder->dev, "[CRTC:%d:%s] mode_valid() failed\n",
536	crtc->base.id, crtc->name);
537	return ret;
538	}
539
540	return ret;
541	}
542
543	static int
544	mode_valid(struct drm_atomic_state *state)
545	{
546	struct drm_connector_state *conn_state;
547	struct drm_connector *connector;
548	int i;
549
550	for_each_new_connector_in_state(state, connector, conn_state, i) {
551	struct drm_encoder *encoder = conn_state->best_encoder;
552	struct drm_crtc *crtc = conn_state->crtc;
553	struct drm_crtc_state *crtc_state;
554	enum drm_mode_status mode_status;
555	const struct drm_display_mode *mode;
556
557	if (!crtc || !encoder)
558	continue;
559
560	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
561	if (!crtc_state)
562	continue;
563	if (!crtc_state->mode_changed && !crtc_state->connectors_changed)
564	continue;
565
566	mode = &crtc_state->mode;
567
568	mode_status = mode_valid_path(connector, encoder, crtc, mode);
569	if (mode_status != MODE_OK)
570	return -EINVAL;
571	}
572
573	return 0;
574	}
575
576	/**
577	* drm_atomic_helper_check_modeset - validate state object for modeset changes
578	* @dev: DRM device
579	* @state: the driver state object
580	*
581	* Check the state object to see if the requested state is physically possible.
582	* This does all the CRTC and connector related computations for an atomic
583	* update and adds any additional connectors needed for full modesets. It calls
584	* the various per-object callbacks in the follow order:
585	*
586	* 1. &drm_connector_helper_funcs.atomic_best_encoder for determining the new encoder.
587	* 2. &drm_connector_helper_funcs.atomic_check to validate the connector state.
588	* 3. If it's determined a modeset is needed then all connectors on the affected
589	* CRTC are added and &drm_connector_helper_funcs.atomic_check is run on them.
590	* 4. &drm_encoder_helper_funcs.mode_valid, &drm_bridge_funcs.mode_valid and
591	* &drm_crtc_helper_funcs.mode_valid are called on the affected components.
592	* 5. &drm_bridge_funcs.mode_fixup is called on all encoder bridges.
593	* 6. &drm_encoder_helper_funcs.atomic_check is called to validate any encoder state.
594	* This function is only called when the encoder will be part of a configured CRTC,
595	* it must not be used for implementing connector property validation.
596	* If this function is NULL, &drm_atomic_encoder_helper_funcs.mode_fixup is called
597	* instead.
598	* 7. &drm_crtc_helper_funcs.mode_fixup is called last, to fix up the mode with CRTC constraints.
599	*
600	* &drm_crtc_state.mode_changed is set when the input mode is changed.
601	* &drm_crtc_state.connectors_changed is set when a connector is added or
602	* removed from the CRTC. &drm_crtc_state.active_changed is set when
603	* &drm_crtc_state.active changes, which is used for DPMS.
604	* &drm_crtc_state.no_vblank is set from the result of drm_dev_has_vblank().
605	* See also: drm_atomic_crtc_needs_modeset()
606	*
607	* IMPORTANT:
608	*
609	* Drivers which set &drm_crtc_state.mode_changed (e.g. in their
610	* &drm_plane_helper_funcs.atomic_check hooks if a plane update can't be done
611	* without a full modeset) _must_ call this function after that change. It is
612	* permitted to call this function multiple times for the same update, e.g.
613	* when the &drm_crtc_helper_funcs.atomic_check functions depend upon the
614	* adjusted dotclock for fifo space allocation and watermark computation.
615	*
616	* RETURNS:
617	* Zero for success or -errno
618	*/
619	int
620	drm_atomic_helper_check_modeset(struct drm_device *dev,
621	struct drm_atomic_state *state)
622	{
623	struct drm_crtc *crtc;
624	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
625	struct drm_connector *connector;
626	struct drm_connector_state *old_connector_state, *new_connector_state;
627	int i, ret;
628	unsigned int connectors_mask = 0, user_connectors_mask = 0;
629
630	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i)
631	user_connectors_mask |= BIT(i);
632
633	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
634	bool has_connectors =
635	!!new_crtc_state->connector_mask;
636
637	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
638
639	if (!drm_mode_equal(mode1: &old_crtc_state->mode, mode2: &new_crtc_state->mode)) {
640	drm_dbg_atomic(dev, "[CRTC:%d:%s] mode changed\n",
641	crtc->base.id, crtc->name);
642	new_crtc_state->mode_changed = true;
643	}
644
645	if (old_crtc_state->enable != new_crtc_state->enable) {
646	drm_dbg_atomic(dev, "[CRTC:%d:%s] enable changed\n",
647	crtc->base.id, crtc->name);
648
649	/*
650	* For clarity this assignment is done here, but
651	* enable == 0 is only true when there are no
652	* connectors and a NULL mode.
653	*
654	* The other way around is true as well. enable != 0
655	* implies that connectors are attached and a mode is set.
656	*/
657	new_crtc_state->mode_changed = true;
658	new_crtc_state->connectors_changed = true;
659	}
660
661	if (old_crtc_state->active != new_crtc_state->active) {
662	drm_dbg_atomic(dev, "[CRTC:%d:%s] active changed\n",
663	crtc->base.id, crtc->name);
664	new_crtc_state->active_changed = true;
665	}
666
667	if (new_crtc_state->enable != has_connectors) {
668	drm_dbg_atomic(dev, "[CRTC:%d:%s] enabled/connectors mismatch\n",
669	crtc->base.id, crtc->name);
670
671	return -EINVAL;
672	}
673
674	if (drm_dev_has_vblank(dev))
675	new_crtc_state->no_vblank = false;
676	else
677	new_crtc_state->no_vblank = true;
678	}
679
680	ret = handle_conflicting_encoders(state, disable_conflicting_encoders: false);
681	if (ret)
682	return ret;
683
684	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
685	const struct drm_connector_helper_funcs *funcs = connector->helper_private;
686
687	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
688
689	/*
690	* This only sets crtc->connectors_changed for routing changes,
691	* drivers must set crtc->connectors_changed themselves when
692	* connector properties need to be updated.
693	*/
694	ret = update_connector_routing(state, connector,
695	old_connector_state,
696	new_connector_state,
697	BIT(i) & user_connectors_mask);
698	if (ret)
699	return ret;
700	if (old_connector_state->crtc) {
701	new_crtc_state = drm_atomic_get_new_crtc_state(state,
702	crtc: old_connector_state->crtc);
703	if (old_connector_state->link_status !=
704	new_connector_state->link_status)
705	new_crtc_state->connectors_changed = true;
706
707	if (old_connector_state->max_requested_bpc !=
708	new_connector_state->max_requested_bpc)
709	new_crtc_state->connectors_changed = true;
710	}
711
712	if (funcs->atomic_check)
713	ret = funcs->atomic_check(connector, state);
714	if (ret) {
715	drm_dbg_atomic(dev,
716	"[CONNECTOR:%d:%s] driver check failed\n",
717	connector->base.id, connector->name);
718	return ret;
719	}
720
721	connectors_mask |= BIT(i);
722	}
723
724	/*
725	* After all the routing has been prepared we need to add in any
726	* connector which is itself unchanged, but whose CRTC changes its
727	* configuration. This must be done before calling mode_fixup in case a
728	* crtc only changed its mode but has the same set of connectors.
729	*/
730	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
731	if (!drm_atomic_crtc_needs_modeset(state: new_crtc_state))
732	continue;
733
734	drm_dbg_atomic(dev,
735	"[CRTC:%d:%s] needs all connectors, enable: %c, active: %c\n",
736	crtc->base.id, crtc->name,
737	new_crtc_state->enable ? 'y' : 'n',
738	new_crtc_state->active ? 'y' : 'n');
739
740	ret = drm_atomic_add_affected_connectors(state, crtc);
741	if (ret != 0)
742	return ret;
743
744	ret = drm_atomic_add_affected_planes(state, crtc);
745	if (ret != 0)
746	return ret;
747	}
748
749	/*
750	* Iterate over all connectors again, to make sure atomic_check()
751	* has been called on them when a modeset is forced.
752	*/
753	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
754	const struct drm_connector_helper_funcs *funcs = connector->helper_private;
755
756	if (connectors_mask & BIT(i))
757	continue;
758
759	if (funcs->atomic_check)
760	ret = funcs->atomic_check(connector, state);
761	if (ret) {
762	drm_dbg_atomic(dev,
763	"[CONNECTOR:%d:%s] driver check failed\n",
764	connector->base.id, connector->name);
765	return ret;
766	}
767	}
768
769	/*
770	* Iterate over all connectors again, and add all affected bridges to
771	* the state.
772	*/
773	for_each_oldnew_connector_in_state(state, connector,
774	old_connector_state,
775	new_connector_state, i) {
776	struct drm_encoder *encoder;
777
778	encoder = old_connector_state->best_encoder;
779	ret = drm_atomic_add_encoder_bridges(state, encoder);
780	if (ret)
781	return ret;
782
783	encoder = new_connector_state->best_encoder;
784	ret = drm_atomic_add_encoder_bridges(state, encoder);
785	if (ret)
786	return ret;
787	}
788
789	ret = mode_valid(state);
790	if (ret)
791	return ret;
792
793	return mode_fixup(state);
794	}
795	EXPORT_SYMBOL(drm_atomic_helper_check_modeset);
796
797	/**
798	* drm_atomic_helper_check_wb_connector_state() - Check writeback connector state
799	* @connector: corresponding connector
800	* @state: the driver state object
801	*
802	* Checks if the writeback connector state is valid, and returns an error if it
803	* isn't.
804	*
805	* RETURNS:
806	* Zero for success or -errno
807	*/
808	int
809	drm_atomic_helper_check_wb_connector_state(struct drm_connector *connector,
810	struct drm_atomic_state *state)
811	{
812	struct drm_connector_state *conn_state =
813	drm_atomic_get_new_connector_state(state, connector);
814	struct drm_writeback_job *wb_job = conn_state->writeback_job;
815	struct drm_property_blob *pixel_format_blob;
816	struct drm_framebuffer *fb;
817	size_t i, nformats;
818	u32 *formats;
819
820	if (!wb_job || !wb_job->fb)
821	return 0;
822
823	pixel_format_blob = wb_job->connector->pixel_formats_blob_ptr;
824	nformats = pixel_format_blob->length / sizeof(u32);
825	formats = pixel_format_blob->data;
826	fb = wb_job->fb;
827
828	for (i = 0; i < nformats; i++)
829	if (fb->format->format == formats[i])
830	return 0;
831
832	drm_dbg_kms(connector->dev, "Invalid pixel format %p4cc\n", &fb->format->format);
833
834	return -EINVAL;
835	}
836	EXPORT_SYMBOL(drm_atomic_helper_check_wb_connector_state);
837
838	/**
839	* drm_atomic_helper_check_plane_state() - Check plane state for validity
840	* @plane_state: plane state to check
841	* @crtc_state: CRTC state to check
842	* @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
843	* @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
844	* @can_position: is it legal to position the plane such that it
845	* doesn't cover the entire CRTC? This will generally
846	* only be false for primary planes.
847	* @can_update_disabled: can the plane be updated while the CRTC
848	* is disabled?
849	*
850	* Checks that a desired plane update is valid, and updates various
851	* bits of derived state (clipped coordinates etc.). Drivers that provide
852	* their own plane handling rather than helper-provided implementations may
853	* still wish to call this function to avoid duplication of error checking
854	* code.
855	*
856	* RETURNS:
857	* Zero if update appears valid, error code on failure
858	*/
859	int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
860	const struct drm_crtc_state *crtc_state,
861	int min_scale,
862	int max_scale,
863	bool can_position,
864	bool can_update_disabled)
865	{
866	struct drm_framebuffer *fb = plane_state->fb;
867	struct drm_rect *src = &plane_state->src;
868	struct drm_rect *dst = &plane_state->dst;
869	unsigned int rotation = plane_state->rotation;
870	struct drm_rect clip = {};
871	int hscale, vscale;
872
873	WARN_ON(plane_state->crtc && plane_state->crtc != crtc_state->crtc);
874
875	*src = drm_plane_state_src(state: plane_state);
876	*dst = drm_plane_state_dest(state: plane_state);
877
878	if (!fb) {
879	plane_state->visible = false;
880	return 0;
881	}
882
883	/* crtc should only be NULL when disabling (i.e., !fb) */
884	if (WARN_ON(!plane_state->crtc)) {
885	plane_state->visible = false;
886	return 0;
887	}
888
889	if (!crtc_state->enable && !can_update_disabled) {
890	drm_dbg_kms(plane_state->plane->dev,
891	"Cannot update plane of a disabled CRTC.\n");
892	return -EINVAL;
893	}
894
895	drm_rect_rotate(r: src, width: fb->width << 16, height: fb->height << 16, rotation);
896
897	/* Check scaling */
898	hscale = drm_rect_calc_hscale(src, dst, min_hscale: min_scale, max_hscale: max_scale);
899	vscale = drm_rect_calc_vscale(src, dst, min_vscale: min_scale, max_vscale: max_scale);
900	if (hscale < 0 || vscale < 0) {
901	drm_dbg_kms(plane_state->plane->dev,
902	"Invalid scaling of plane\n");
903	drm_rect_debug_print(prefix: "src: ", r: &plane_state->src, fixed_point: true);
904	drm_rect_debug_print(prefix: "dst: ", r: &plane_state->dst, fixed_point: false);
905	return -ERANGE;
906	}
907
908	if (crtc_state->enable)
909	drm_mode_get_hv_timing(mode: &crtc_state->mode, hdisplay: &clip.x2, vdisplay: &clip.y2);
910
911	plane_state->visible = drm_rect_clip_scaled(src, dst, clip: &clip);
912
913	drm_rect_rotate_inv(r: src, width: fb->width << 16, height: fb->height << 16, rotation);
914
915	if (!plane_state->visible)
916	/*
917	* Plane isn't visible; some drivers can handle this
918	* so we just return success here. Drivers that can't
919	* (including those that use the primary plane helper's
920	* update function) will return an error from their
921	* update_plane handler.
922	*/
923	return 0;
924
925	if (!can_position && !drm_rect_equals(r1: dst, r2: &clip)) {
926	drm_dbg_kms(plane_state->plane->dev,
927	"Plane must cover entire CRTC\n");
928	drm_rect_debug_print(prefix: "dst: ", r: dst, fixed_point: false);
929	drm_rect_debug_print(prefix: "clip: ", r: &clip, fixed_point: false);
930	return -EINVAL;
931	}
932
933	return 0;
934	}
935	EXPORT_SYMBOL(drm_atomic_helper_check_plane_state);
936
937	/**
938	* drm_atomic_helper_check_crtc_primary_plane() - Check CRTC state for primary plane
939	* @crtc_state: CRTC state to check
940	*
941	* Checks that a CRTC has at least one primary plane attached to it, which is
942	* a requirement on some hardware. Note that this only involves the CRTC side
943	* of the test. To test if the primary plane is visible or if it can be updated
944	* without the CRTC being enabled, use drm_atomic_helper_check_plane_state() in
945	* the plane's atomic check.
946	*
947	* RETURNS:
948	* 0 if a primary plane is attached to the CRTC, or an error code otherwise
949	*/
950	int drm_atomic_helper_check_crtc_primary_plane(struct drm_crtc_state *crtc_state)
951	{
952	struct drm_crtc *crtc = crtc_state->crtc;
953	struct drm_device *dev = crtc->dev;
954	struct drm_plane *plane;
955
956	/* needs at least one primary plane to be enabled */
957	drm_for_each_plane_mask(plane, dev, crtc_state->plane_mask) {
958	if (plane->type == DRM_PLANE_TYPE_PRIMARY)
959	return 0;
960	}
961
962	drm_dbg_atomic(dev, "[CRTC:%d:%s] primary plane missing\n", crtc->base.id, crtc->name);
963
964	return -EINVAL;
965	}
966	EXPORT_SYMBOL(drm_atomic_helper_check_crtc_primary_plane);
967
968	/**
969	* drm_atomic_helper_check_planes - validate state object for planes changes
970	* @dev: DRM device
971	* @state: the driver state object
972	*
973	* Check the state object to see if the requested state is physically possible.
974	* This does all the plane update related checks using by calling into the
975	* &drm_crtc_helper_funcs.atomic_check and &drm_plane_helper_funcs.atomic_check
976	* hooks provided by the driver.
977	*
978	* It also sets &drm_crtc_state.planes_changed to indicate that a CRTC has
979	* updated planes.
980	*
981	* RETURNS:
982	* Zero for success or -errno
983	*/
984	int
985	drm_atomic_helper_check_planes(struct drm_device *dev,
986	struct drm_atomic_state *state)
987	{
988	struct drm_crtc *crtc;
989	struct drm_crtc_state *new_crtc_state;
990	struct drm_plane *plane;
991	struct drm_plane_state *new_plane_state, *old_plane_state;
992	int i, ret = 0;
993
994	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
995	const struct drm_plane_helper_funcs *funcs;
996
997	WARN_ON(!drm_modeset_is_locked(&plane->mutex));
998
999	funcs = plane->helper_private;
1000
1001	drm_atomic_helper_plane_changed(state, old_plane_state, plane_state: new_plane_state, plane);
1002
1003	drm_atomic_helper_check_plane_damage(state, plane_state: new_plane_state);
1004
1005	if (!funcs || !funcs->atomic_check)
1006	continue;
1007
1008	ret = funcs->atomic_check(plane, state);
1009	if (ret) {
1010	drm_dbg_atomic(plane->dev,
1011	"[PLANE:%d:%s] atomic driver check failed\n",
1012	plane->base.id, plane->name);
1013	return ret;
1014	}
1015	}
1016
1017	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
1018	const struct drm_crtc_helper_funcs *funcs;
1019
1020	funcs = crtc->helper_private;
1021
1022	if (!funcs || !funcs->atomic_check)
1023	continue;
1024
1025	ret = funcs->atomic_check(crtc, state);
1026	if (ret) {
1027	drm_dbg_atomic(crtc->dev,
1028	"[CRTC:%d:%s] atomic driver check failed\n",
1029	crtc->base.id, crtc->name);
1030	return ret;
1031	}
1032	}
1033
1034	return ret;
1035	}
1036	EXPORT_SYMBOL(drm_atomic_helper_check_planes);
1037
1038	/**
1039	* drm_atomic_helper_check - validate state object
1040	* @dev: DRM device
1041	* @state: the driver state object
1042	*
1043	* Check the state object to see if the requested state is physically possible.
1044	* Only CRTCs and planes have check callbacks, so for any additional (global)
1045	* checking that a driver needs it can simply wrap that around this function.
1046	* Drivers without such needs can directly use this as their
1047	* &drm_mode_config_funcs.atomic_check callback.
1048	*
1049	* This just wraps the two parts of the state checking for planes and modeset
1050	* state in the default order: First it calls drm_atomic_helper_check_modeset()
1051	* and then drm_atomic_helper_check_planes(). The assumption is that the
1052	* @drm_plane_helper_funcs.atomic_check and @drm_crtc_helper_funcs.atomic_check
1053	* functions depend upon an updated adjusted_mode.clock to e.g. properly compute
1054	* watermarks.
1055	*
1056	* Note that zpos normalization will add all enable planes to the state which
1057	* might not desired for some drivers.
1058	* For example enable/disable of a cursor plane which have fixed zpos value
1059	* would trigger all other enabled planes to be forced to the state change.
1060	*
1061	* RETURNS:
1062	* Zero for success or -errno
1063	*/
1064	int drm_atomic_helper_check(struct drm_device *dev,
1065	struct drm_atomic_state *state)
1066	{
1067	int ret;
1068
1069	ret = drm_atomic_helper_check_modeset(dev, state);
1070	if (ret)
1071	return ret;
1072
1073	if (dev->mode_config.normalize_zpos) {
1074	ret = drm_atomic_normalize_zpos(dev, state);
1075	if (ret)
1076	return ret;
1077	}
1078
1079	ret = drm_atomic_helper_check_planes(dev, state);
1080	if (ret)
1081	return ret;
1082
1083	if (state->legacy_cursor_update)
1084	state->async_update = !drm_atomic_helper_async_check(dev, state);
1085
1086	drm_self_refresh_helper_alter_state(state);
1087
1088	return ret;
1089	}
1090	EXPORT_SYMBOL(drm_atomic_helper_check);
1091
1092	static bool
1093	crtc_needs_disable(struct drm_crtc_state *old_state,
1094	struct drm_crtc_state *new_state)
1095	{
1096	/*
1097	* No new_state means the CRTC is off, so the only criteria is whether
1098	* it's currently active or in self refresh mode.
1099	*/
1100	if (!new_state)
1101	return drm_atomic_crtc_effectively_active(state: old_state);
1102
1103	/*
1104	* We need to disable bridge(s) and CRTC if we're transitioning out of
1105	* self-refresh and changing CRTCs at the same time, because the
1106	* bridge tracks self-refresh status via CRTC state.
1107	*/
1108	if (old_state->self_refresh_active &&
1109	old_state->crtc != new_state->crtc)
1110	return true;
1111
1112	/*
1113	* We also need to run through the crtc_funcs->disable() function if
1114	* the CRTC is currently on, if it's transitioning to self refresh
1115	* mode, or if it's in self refresh mode and needs to be fully
1116	* disabled.
1117	*/
1118	return old_state->active ||
1119	(old_state->self_refresh_active && !new_state->active) ||
1120	new_state->self_refresh_active;
1121	}
1122
1123	static void
1124	disable_outputs(struct drm_device *dev, struct drm_atomic_state *old_state)
1125	{
1126	struct drm_connector *connector;
1127	struct drm_connector_state *old_conn_state, *new_conn_state;
1128	struct drm_crtc *crtc;
1129	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
1130	int i;
1131
1132	for_each_oldnew_connector_in_state(old_state, connector, old_conn_state, new_conn_state, i) {
1133	const struct drm_encoder_helper_funcs *funcs;
1134	struct drm_encoder *encoder;
1135	struct drm_bridge *bridge;
1136
1137	/*
1138	* Shut down everything that's in the changeset and currently
1139	* still on. So need to check the old, saved state.
1140	*/
1141	if (!old_conn_state->crtc)
1142	continue;
1143
1144	old_crtc_state = drm_atomic_get_old_crtc_state(state: old_state, crtc: old_conn_state->crtc);
1145
1146	if (new_conn_state->crtc)
1147	new_crtc_state = drm_atomic_get_new_crtc_state(
1148	state: old_state,
1149	crtc: new_conn_state->crtc);
1150	else
1151	new_crtc_state = NULL;
1152
1153	if (!crtc_needs_disable(old_state: old_crtc_state, new_state: new_crtc_state) ||
1154	!drm_atomic_crtc_needs_modeset(state: old_conn_state->crtc->state))
1155	continue;
1156
1157	encoder = old_conn_state->best_encoder;
1158
1159	/* We shouldn't get this far if we didn't previously have
1160	* an encoder.. but WARN_ON() rather than explode.
1161	*/
1162	if (WARN_ON(!encoder))
1163	continue;
1164
1165	funcs = encoder->helper_private;
1166
1167	drm_dbg_atomic(dev, "disabling [ENCODER:%d:%s]\n",
1168	encoder->base.id, encoder->name);
1169
1170	/*
1171	* Each encoder has at most one connector (since we always steal
1172	* it away), so we won't call disable hooks twice.
1173	*/
1174	bridge = drm_bridge_chain_get_first_bridge(encoder);
1175	drm_atomic_bridge_chain_disable(bridge, state: old_state);
1176
1177	/* Right function depends upon target state. */
1178	if (funcs) {
1179	if (funcs->atomic_disable)
1180	funcs->atomic_disable(encoder, old_state);
1181	else if (new_conn_state->crtc && funcs->prepare)
1182	funcs->prepare(encoder);
1183	else if (funcs->disable)
1184	funcs->disable(encoder);
1185	else if (funcs->dpms)
1186	funcs->dpms(encoder, DRM_MODE_DPMS_OFF);
1187	}
1188
1189	drm_atomic_bridge_chain_post_disable(bridge, state: old_state);
1190	}
1191
1192	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
1193	const struct drm_crtc_helper_funcs *funcs;
1194	int ret;
1195
1196	/* Shut down everything that needs a full modeset. */
1197	if (!drm_atomic_crtc_needs_modeset(state: new_crtc_state))
1198	continue;
1199
1200	if (!crtc_needs_disable(old_state: old_crtc_state, new_state: new_crtc_state))
1201	continue;
1202
1203	funcs = crtc->helper_private;
1204
1205	drm_dbg_atomic(dev, "disabling [CRTC:%d:%s]\n",
1206	crtc->base.id, crtc->name);
1207
1208
1209	/* Right function depends upon target state. */
1210	if (new_crtc_state->enable && funcs->prepare)
1211	funcs->prepare(crtc);
1212	else if (funcs->atomic_disable)
1213	funcs->atomic_disable(crtc, old_state);
1214	else if (funcs->disable)
1215	funcs->disable(crtc);
1216	else if (funcs->dpms)
1217	funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
1218
1219	if (!drm_dev_has_vblank(dev))
1220	continue;
1221
1222	ret = drm_crtc_vblank_get(crtc);
1223	/*
1224	* Self-refresh is not a true "disable"; ensure vblank remains
1225	* enabled.
1226	*/
1227	if (new_crtc_state->self_refresh_active)
1228	WARN_ONCE(ret != 0,
1229	"driver disabled vblank in self-refresh\n");
1230	else
1231	WARN_ONCE(ret != -EINVAL,
1232	"driver forgot to call drm_crtc_vblank_off()\n");
1233	if (ret == 0)
1234	drm_crtc_vblank_put(crtc);
1235	}
1236	}
1237
1238	/**
1239	* drm_atomic_helper_update_legacy_modeset_state - update legacy modeset state
1240	* @dev: DRM device
1241	* @old_state: atomic state object with old state structures
1242	*
1243	* This function updates all the various legacy modeset state pointers in
1244	* connectors, encoders and CRTCs.
1245	*
1246	* Drivers can use this for building their own atomic commit if they don't have
1247	* a pure helper-based modeset implementation.
1248	*
1249	* Since these updates are not synchronized with lockings, only code paths
1250	* called from &drm_mode_config_helper_funcs.atomic_commit_tail can look at the
1251	* legacy state filled out by this helper. Defacto this means this helper and
1252	* the legacy state pointers are only really useful for transitioning an
1253	* existing driver to the atomic world.
1254	*/
1255	void
1256	drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev,
1257	struct drm_atomic_state *old_state)
1258	{
1259	struct drm_connector *connector;
1260	struct drm_connector_state *old_conn_state, *new_conn_state;
1261	struct drm_crtc *crtc;
1262	struct drm_crtc_state *new_crtc_state;
1263	int i;
1264
1265	/* clear out existing links and update dpms */
1266	for_each_oldnew_connector_in_state(old_state, connector, old_conn_state, new_conn_state, i) {
1267	if (connector->encoder) {
1268	WARN_ON(!connector->encoder->crtc);
1269
1270	connector->encoder->crtc = NULL;
1271	connector->encoder = NULL;
1272	}
1273
1274	crtc = new_conn_state->crtc;
1275	if ((!crtc && old_conn_state->crtc) ||
1276	(crtc && drm_atomic_crtc_needs_modeset(state: crtc->state))) {
1277	int mode = DRM_MODE_DPMS_OFF;
1278
1279	if (crtc && crtc->state->active)
1280	mode = DRM_MODE_DPMS_ON;
1281
1282	connector->dpms = mode;
1283	}
1284	}
1285
1286	/* set new links */
1287	for_each_new_connector_in_state(old_state, connector, new_conn_state, i) {
1288	if (!new_conn_state->crtc)
1289	continue;
1290
1291	if (WARN_ON(!new_conn_state->best_encoder))
1292	continue;
1293
1294	connector->encoder = new_conn_state->best_encoder;
1295	connector->encoder->crtc = new_conn_state->crtc;
1296	}
1297
1298	/* set legacy state in the crtc structure */
1299	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
1300	struct drm_plane *primary = crtc->primary;
1301	struct drm_plane_state *new_plane_state;
1302
1303	crtc->mode = new_crtc_state->mode;
1304	crtc->enabled = new_crtc_state->enable;
1305
1306	new_plane_state =
1307	drm_atomic_get_new_plane_state(state: old_state, plane: primary);
1308
1309	if (new_plane_state && new_plane_state->crtc == crtc) {
1310	crtc->x = new_plane_state->src_x >> 16;
1311	crtc->y = new_plane_state->src_y >> 16;
1312	}
1313	}
1314	}
1315	EXPORT_SYMBOL(drm_atomic_helper_update_legacy_modeset_state);
1316
1317	/**
1318	* drm_atomic_helper_calc_timestamping_constants - update vblank timestamping constants
1319	* @state: atomic state object
1320	*
1321	* Updates the timestamping constants used for precise vblank timestamps
1322	* by calling drm_calc_timestamping_constants() for all enabled crtcs in @state.
1323	*/
1324	void drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_state *state)
1325	{
1326	struct drm_crtc_state *new_crtc_state;
1327	struct drm_crtc *crtc;
1328	int i;
1329
1330	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
1331	if (new_crtc_state->enable)
1332	drm_calc_timestamping_constants(crtc,
1333	mode: &new_crtc_state->adjusted_mode);
1334	}
1335	}
1336	EXPORT_SYMBOL(drm_atomic_helper_calc_timestamping_constants);
1337
1338	static void
1339	crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *old_state)
1340	{
1341	struct drm_crtc *crtc;
1342	struct drm_crtc_state *new_crtc_state;
1343	struct drm_connector *connector;
1344	struct drm_connector_state *new_conn_state;
1345	int i;
1346
1347	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
1348	const struct drm_crtc_helper_funcs *funcs;
1349
1350	if (!new_crtc_state->mode_changed)
1351	continue;
1352
1353	funcs = crtc->helper_private;
1354
1355	if (new_crtc_state->enable && funcs->mode_set_nofb) {
1356	drm_dbg_atomic(dev, "modeset on [CRTC:%d:%s]\n",
1357	crtc->base.id, crtc->name);
1358
1359	funcs->mode_set_nofb(crtc);
1360	}
1361	}
1362
1363	for_each_new_connector_in_state(old_state, connector, new_conn_state, i) {
1364	const struct drm_encoder_helper_funcs *funcs;
1365	struct drm_encoder *encoder;
1366	struct drm_display_mode *mode, *adjusted_mode;
1367	struct drm_bridge *bridge;
1368
1369	if (!new_conn_state->best_encoder)
1370	continue;
1371
1372	encoder = new_conn_state->best_encoder;
1373	funcs = encoder->helper_private;
1374	new_crtc_state = new_conn_state->crtc->state;
1375	mode = &new_crtc_state->mode;
1376	adjusted_mode = &new_crtc_state->adjusted_mode;
1377
1378	if (!new_crtc_state->mode_changed)
1379	continue;
1380
1381	drm_dbg_atomic(dev, "modeset on [ENCODER:%d:%s]\n",
1382	encoder->base.id, encoder->name);
1383
1384	/*
1385	* Each encoder has at most one connector (since we always steal
1386	* it away), so we won't call mode_set hooks twice.
1387	*/
1388	if (funcs && funcs->atomic_mode_set) {
1389	funcs->atomic_mode_set(encoder, new_crtc_state,
1390	new_conn_state);
1391	} else if (funcs && funcs->mode_set) {
1392	funcs->mode_set(encoder, mode, adjusted_mode);
1393	}
1394
1395	bridge = drm_bridge_chain_get_first_bridge(encoder);
1396	drm_bridge_chain_mode_set(bridge, mode, adjusted_mode);
1397	}
1398	}
1399
1400	/**
1401	* drm_atomic_helper_commit_modeset_disables - modeset commit to disable outputs
1402	* @dev: DRM device
1403	* @old_state: atomic state object with old state structures
1404	*
1405	* This function shuts down all the outputs that need to be shut down and
1406	* prepares them (if required) with the new mode.
1407	*
1408	* For compatibility with legacy CRTC helpers this should be called before
1409	* drm_atomic_helper_commit_planes(), which is what the default commit function
1410	* does. But drivers with different needs can group the modeset commits together
1411	* and do the plane commits at the end. This is useful for drivers doing runtime
1412	* PM since planes updates then only happen when the CRTC is actually enabled.
1413	*/
1414	void drm_atomic_helper_commit_modeset_disables(struct drm_device *dev,
1415	struct drm_atomic_state *old_state)
1416	{
1417	disable_outputs(dev, old_state);
1418
1419	drm_atomic_helper_update_legacy_modeset_state(dev, old_state);
1420	drm_atomic_helper_calc_timestamping_constants(old_state);
1421
1422	crtc_set_mode(dev, old_state);
1423	}
1424	EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_disables);
1425
1426	static void drm_atomic_helper_commit_writebacks(struct drm_device *dev,
1427	struct drm_atomic_state *old_state)
1428	{
1429	struct drm_connector *connector;
1430	struct drm_connector_state *new_conn_state;
1431	int i;
1432
1433	for_each_new_connector_in_state(old_state, connector, new_conn_state, i) {
1434	const struct drm_connector_helper_funcs *funcs;
1435
1436	funcs = connector->helper_private;
1437	if (!funcs->atomic_commit)
1438	continue;
1439
1440	if (new_conn_state->writeback_job && new_conn_state->writeback_job->fb) {
1441	WARN_ON(connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK);
1442	funcs->atomic_commit(connector, old_state);
1443	}
1444	}
1445	}
1446
1447	/**
1448	* drm_atomic_helper_commit_modeset_enables - modeset commit to enable outputs
1449	* @dev: DRM device
1450	* @old_state: atomic state object with old state structures
1451	*
1452	* This function enables all the outputs with the new configuration which had to
1453	* be turned off for the update.
1454	*
1455	* For compatibility with legacy CRTC helpers this should be called after
1456	* drm_atomic_helper_commit_planes(), which is what the default commit function
1457	* does. But drivers with different needs can group the modeset commits together
1458	* and do the plane commits at the end. This is useful for drivers doing runtime
1459	* PM since planes updates then only happen when the CRTC is actually enabled.
1460	*/
1461	void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
1462	struct drm_atomic_state *old_state)
1463	{
1464	struct drm_crtc *crtc;
1465	struct drm_crtc_state *old_crtc_state;
1466	struct drm_crtc_state *new_crtc_state;
1467	struct drm_connector *connector;
1468	struct drm_connector_state *new_conn_state;
1469	int i;
1470
1471	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
1472	const struct drm_crtc_helper_funcs *funcs;
1473
1474	/* Need to filter out CRTCs where only planes change. */
1475	if (!drm_atomic_crtc_needs_modeset(state: new_crtc_state))
1476	continue;
1477
1478	if (!new_crtc_state->active)
1479	continue;
1480
1481	funcs = crtc->helper_private;
1482
1483	if (new_crtc_state->enable) {
1484	drm_dbg_atomic(dev, "enabling [CRTC:%d:%s]\n",
1485	crtc->base.id, crtc->name);
1486	if (funcs->atomic_enable)
1487	funcs->atomic_enable(crtc, old_state);
1488	else if (funcs->commit)
1489	funcs->commit(crtc);
1490	}
1491	}
1492
1493	for_each_new_connector_in_state(old_state, connector, new_conn_state, i) {
1494	const struct drm_encoder_helper_funcs *funcs;
1495	struct drm_encoder *encoder;
1496	struct drm_bridge *bridge;
1497
1498	if (!new_conn_state->best_encoder)
1499	continue;
1500
1501	if (!new_conn_state->crtc->state->active ||
1502	!drm_atomic_crtc_needs_modeset(state: new_conn_state->crtc->state))
1503	continue;
1504
1505	encoder = new_conn_state->best_encoder;
1506	funcs = encoder->helper_private;
1507
1508	drm_dbg_atomic(dev, "enabling [ENCODER:%d:%s]\n",
1509	encoder->base.id, encoder->name);
1510
1511	/*
1512	* Each encoder has at most one connector (since we always steal
1513	* it away), so we won't call enable hooks twice.
1514	*/
1515	bridge = drm_bridge_chain_get_first_bridge(encoder);
1516	drm_atomic_bridge_chain_pre_enable(bridge, state: old_state);
1517
1518	if (funcs) {
1519	if (funcs->atomic_enable)
1520	funcs->atomic_enable(encoder, old_state);
1521	else if (funcs->enable)
1522	funcs->enable(encoder);
1523	else if (funcs->commit)
1524	funcs->commit(encoder);
1525	}
1526
1527	drm_atomic_bridge_chain_enable(bridge, state: old_state);
1528	}
1529
1530	drm_atomic_helper_commit_writebacks(dev, old_state);
1531	}
1532	EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_enables);
1533
1534	/*
1535	* For atomic updates which touch just a single CRTC, calculate the time of the
1536	* next vblank, and inform all the fences of the deadline.
1537	*/
1538	static void set_fence_deadline(struct drm_device *dev,
1539	struct drm_atomic_state *state)
1540	{
1541	struct drm_crtc *crtc;
1542	struct drm_crtc_state *new_crtc_state;
1543	struct drm_plane *plane;
1544	struct drm_plane_state *new_plane_state;
1545	ktime_t vbltime = 0;
1546	int i;
1547
1548	for_each_new_crtc_in_state (state, crtc, new_crtc_state, i) {
1549	ktime_t v;
1550
1551	if (drm_atomic_crtc_needs_modeset(state: new_crtc_state))
1552	continue;
1553
1554	if (!new_crtc_state->active)
1555	continue;
1556
1557	if (drm_crtc_next_vblank_start(crtc, vblanktime: &v))
1558	continue;
1559
1560	if (!vbltime || ktime_before(cmp1: v, cmp2: vbltime))
1561	vbltime = v;
1562	}
1563
1564	/* If no CRTCs updated, then nothing to do: */
1565	if (!vbltime)
1566	return;
1567
1568	for_each_new_plane_in_state (state, plane, new_plane_state, i) {
1569	if (!new_plane_state->fence)
1570	continue;
1571	dma_fence_set_deadline(fence: new_plane_state->fence, deadline: vbltime);
1572	}
1573	}
1574
1575	/**
1576	* drm_atomic_helper_wait_for_fences - wait for fences stashed in plane state
1577	* @dev: DRM device
1578	* @state: atomic state object with old state structures
1579	* @pre_swap: If true, do an interruptible wait, and @state is the new state.
1580	* Otherwise @state is the old state.
1581	*
1582	* For implicit sync, driver should fish the exclusive fence out from the
1583	* incoming fb's and stash it in the drm_plane_state. This is called after
1584	* drm_atomic_helper_swap_state() so it uses the current plane state (and
1585	* just uses the atomic state to find the changed planes)
1586	*
1587	* Note that @pre_swap is needed since the point where we block for fences moves
1588	* around depending upon whether an atomic commit is blocking or
1589	* non-blocking. For non-blocking commit all waiting needs to happen after
1590	* drm_atomic_helper_swap_state() is called, but for blocking commits we want
1591	* to wait **before** we do anything that can't be easily rolled back. That is
1592	* before we call drm_atomic_helper_swap_state().
1593	*
1594	* Returns zero if success or < 0 if dma_fence_wait() fails.
1595	*/
1596	int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
1597	struct drm_atomic_state *state,
1598	bool pre_swap)
1599	{
1600	struct drm_plane *plane;
1601	struct drm_plane_state *new_plane_state;
1602	int i, ret;
1603
1604	set_fence_deadline(dev, state);
1605
1606	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
1607	if (!new_plane_state->fence)
1608	continue;
1609
1610	WARN_ON(!new_plane_state->fb);
1611
1612	/*
1613	* If waiting for fences pre-swap (ie: nonblock), userspace can
1614	* still interrupt the operation. Instead of blocking until the
1615	* timer expires, make the wait interruptible.
1616	*/
1617	ret = dma_fence_wait(fence: new_plane_state->fence, intr: pre_swap);
1618	if (ret)
1619	return ret;
1620
1621	dma_fence_put(fence: new_plane_state->fence);
1622	new_plane_state->fence = NULL;
1623	}
1624
1625	return 0;
1626	}
1627	EXPORT_SYMBOL(drm_atomic_helper_wait_for_fences);
1628
1629	/**
1630	* drm_atomic_helper_wait_for_vblanks - wait for vblank on CRTCs
1631	* @dev: DRM device
1632	* @old_state: atomic state object with old state structures
1633	*
1634	* Helper to, after atomic commit, wait for vblanks on all affected
1635	* CRTCs (ie. before cleaning up old framebuffers using
1636	* drm_atomic_helper_cleanup_planes()). It will only wait on CRTCs where the
1637	* framebuffers have actually changed to optimize for the legacy cursor and
1638	* plane update use-case.
1639	*
1640	* Drivers using the nonblocking commit tracking support initialized by calling
1641	* drm_atomic_helper_setup_commit() should look at
1642	* drm_atomic_helper_wait_for_flip_done() as an alternative.
1643	*/
1644	void
1645	drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
1646	struct drm_atomic_state *old_state)
1647	{
1648	struct drm_crtc *crtc;
1649	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
1650	int i, ret;
1651	unsigned int crtc_mask = 0;
1652
1653	/*
1654	* Legacy cursor ioctls are completely unsynced, and userspace
1655	* relies on that (by doing tons of cursor updates).
1656	*/
1657	if (old_state->legacy_cursor_update)
1658	return;
1659
1660	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
1661	if (!new_crtc_state->active)
1662	continue;
1663
1664	ret = drm_crtc_vblank_get(crtc);
1665	if (ret != 0)
1666	continue;
1667
1668	crtc_mask |= drm_crtc_mask(crtc);
1669	old_state->crtcs[i].last_vblank_count = drm_crtc_vblank_count(crtc);
1670	}
1671
1672	for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i) {
1673	if (!(crtc_mask & drm_crtc_mask(crtc)))
1674	continue;
1675
1676	ret = wait_event_timeout(dev->vblank[i].queue,
1677	old_state->crtcs[i].last_vblank_count !=
1678	drm_crtc_vblank_count(crtc),
1679	msecs_to_jiffies(100));
1680
1681	WARN(!ret, "[CRTC:%d:%s] vblank wait timed out\n",
1682	crtc->base.id, crtc->name);
1683
1684	drm_crtc_vblank_put(crtc);
1685	}
1686	}
1687	EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
1688
1689	/**
1690	* drm_atomic_helper_wait_for_flip_done - wait for all page flips to be done
1691	* @dev: DRM device
1692	* @old_state: atomic state object with old state structures
1693	*
1694	* Helper to, after atomic commit, wait for page flips on all affected
1695	* crtcs (ie. before cleaning up old framebuffers using
1696	* drm_atomic_helper_cleanup_planes()). Compared to
1697	* drm_atomic_helper_wait_for_vblanks() this waits for the completion on all
1698	* CRTCs, assuming that cursors-only updates are signalling their completion
1699	* immediately (or using a different path).
1700	*
1701	* This requires that drivers use the nonblocking commit tracking support
1702	* initialized using drm_atomic_helper_setup_commit().
1703	*/
1704	void drm_atomic_helper_wait_for_flip_done(struct drm_device *dev,
1705	struct drm_atomic_state *old_state)
1706	{
1707	struct drm_crtc *crtc;
1708	int i;
1709
1710	for (i = 0; i < dev->mode_config.num_crtc; i++) {
1711	struct drm_crtc_commit *commit = old_state->crtcs[i].commit;
1712	int ret;
1713
1714	crtc = old_state->crtcs[i].ptr;
1715
1716	if (!crtc || !commit)
1717	continue;
1718
1719	ret = wait_for_completion_timeout(x: &commit->flip_done, timeout: 10 * HZ);
1720	if (ret == 0)
1721	drm_err(dev, "[CRTC:%d:%s] flip_done timed out\n",
1722	crtc->base.id, crtc->name);
1723	}
1724
1725	if (old_state->fake_commit)
1726	complete_all(&old_state->fake_commit->flip_done);
1727	}
1728	EXPORT_SYMBOL(drm_atomic_helper_wait_for_flip_done);
1729
1730	/**
1731	* drm_atomic_helper_commit_tail - commit atomic update to hardware
1732	* @old_state: atomic state object with old state structures
1733	*
1734	* This is the default implementation for the
1735	* &drm_mode_config_helper_funcs.atomic_commit_tail hook, for drivers
1736	* that do not support runtime_pm or do not need the CRTC to be
1737	* enabled to perform a commit. Otherwise, see
1738	* drm_atomic_helper_commit_tail_rpm().
1739	*
1740	* Note that the default ordering of how the various stages are called is to
1741	* match the legacy modeset helper library closest.
1742	*/
1743	void drm_atomic_helper_commit_tail(struct drm_atomic_state *old_state)
1744	{
1745	struct drm_device *dev = old_state->dev;
1746
1747	drm_atomic_helper_commit_modeset_disables(dev, old_state);
1748
1749	drm_atomic_helper_commit_planes(dev, state: old_state, flags: 0);
1750
1751	drm_atomic_helper_commit_modeset_enables(dev, old_state);
1752
1753	drm_atomic_helper_fake_vblank(state: old_state);
1754
1755	drm_atomic_helper_commit_hw_done(state: old_state);
1756
1757	drm_atomic_helper_wait_for_vblanks(dev, old_state);
1758
1759	drm_atomic_helper_cleanup_planes(dev, old_state);
1760	}
1761	EXPORT_SYMBOL(drm_atomic_helper_commit_tail);
1762
1763	/**
1764	* drm_atomic_helper_commit_tail_rpm - commit atomic update to hardware
1765	* @old_state: new modeset state to be committed
1766	*
1767	* This is an alternative implementation for the
1768	* &drm_mode_config_helper_funcs.atomic_commit_tail hook, for drivers
1769	* that support runtime_pm or need the CRTC to be enabled to perform a
1770	* commit. Otherwise, one should use the default implementation
1771	* drm_atomic_helper_commit_tail().
1772	*/
1773	void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_state *old_state)
1774	{
1775	struct drm_device *dev = old_state->dev;
1776
1777	drm_atomic_helper_commit_modeset_disables(dev, old_state);
1778
1779	drm_atomic_helper_commit_modeset_enables(dev, old_state);
1780
1781	drm_atomic_helper_commit_planes(dev, state: old_state,
1782	DRM_PLANE_COMMIT_ACTIVE_ONLY);
1783
1784	drm_atomic_helper_fake_vblank(state: old_state);
1785
1786	drm_atomic_helper_commit_hw_done(state: old_state);
1787
1788	drm_atomic_helper_wait_for_vblanks(dev, old_state);
1789
1790	drm_atomic_helper_cleanup_planes(dev, old_state);
1791	}
1792	EXPORT_SYMBOL(drm_atomic_helper_commit_tail_rpm);
1793
1794	static void commit_tail(struct drm_atomic_state *old_state)
1795	{
1796	struct drm_device *dev = old_state->dev;
1797	const struct drm_mode_config_helper_funcs *funcs;
1798	struct drm_crtc_state *new_crtc_state;
1799	struct drm_crtc *crtc;
1800	ktime_t start;
1801	s64 commit_time_ms;
1802	unsigned int i, new_self_refresh_mask = 0;
1803
1804	funcs = dev->mode_config.helper_private;
1805
1806	/*
1807	* We're measuring the _entire_ commit, so the time will vary depending
1808	* on how many fences and objects are involved. For the purposes of self
1809	* refresh, this is desirable since it'll give us an idea of how
1810	* congested things are. This will inform our decision on how often we
1811	* should enter self refresh after idle.
1812	*
1813	* These times will be averaged out in the self refresh helpers to avoid
1814	* overreacting over one outlier frame
1815	*/
1816	start = ktime_get();
1817
1818	drm_atomic_helper_wait_for_fences(dev, old_state, false);
1819
1820	drm_atomic_helper_wait_for_dependencies(state: old_state);
1821
1822	/*
1823	* We cannot safely access new_crtc_state after
1824	* drm_atomic_helper_commit_hw_done() so figure out which crtc's have
1825	* self-refresh active beforehand:
1826	*/
1827	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i)
1828	if (new_crtc_state->self_refresh_active)
1829	new_self_refresh_mask |= BIT(i);
1830
1831	if (funcs && funcs->atomic_commit_tail)
1832	funcs->atomic_commit_tail(old_state);
1833	else
1834	drm_atomic_helper_commit_tail(old_state);
1835
1836	commit_time_ms = ktime_ms_delta(later: ktime_get(), earlier: start);
1837	if (commit_time_ms > 0)
1838	drm_self_refresh_helper_update_avg_times(state: old_state,
1839	commit_time_ms: (unsigned long)commit_time_ms,
1840	new_self_refresh_mask);
1841
1842	drm_atomic_helper_commit_cleanup_done(state: old_state);
1843
1844	drm_atomic_state_put(state: old_state);
1845	}
1846
1847	static void commit_work(struct work_struct *work)
1848	{
1849	struct drm_atomic_state *state = container_of(work,
1850	struct drm_atomic_state,
1851	commit_work);
1852	commit_tail(old_state: state);
1853	}
1854
1855	/**
1856	* drm_atomic_helper_async_check - check if state can be committed asynchronously
1857	* @dev: DRM device
1858	* @state: the driver state object
1859	*
1860	* This helper will check if it is possible to commit the state asynchronously.
1861	* Async commits are not supposed to swap the states like normal sync commits
1862	* but just do in-place changes on the current state.
1863	*
1864	* It will return 0 if the commit can happen in an asynchronous fashion or error
1865	* if not. Note that error just mean it can't be committed asynchronously, if it
1866	* fails the commit should be treated like a normal synchronous commit.
1867	*/
1868	int drm_atomic_helper_async_check(struct drm_device *dev,
1869	struct drm_atomic_state *state)
1870	{
1871	struct drm_crtc *crtc;
1872	struct drm_crtc_state *crtc_state;
1873	struct drm_plane *plane = NULL;
1874	struct drm_plane_state *old_plane_state = NULL;
1875	struct drm_plane_state *new_plane_state = NULL;
1876	const struct drm_plane_helper_funcs *funcs;
1877	int i, ret, n_planes = 0;
1878
1879	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
1880	if (drm_atomic_crtc_needs_modeset(state: crtc_state))
1881	return -EINVAL;
1882	}
1883
1884	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
1885	n_planes++;
1886
1887	/* FIXME: we support only single plane updates for now */
1888	if (n_planes != 1) {
1889	drm_dbg_atomic(dev,
1890	"only single plane async updates are supported\n");
1891	return -EINVAL;
1892	}
1893
1894	if (!new_plane_state->crtc ||
1895	old_plane_state->crtc != new_plane_state->crtc) {
1896	drm_dbg_atomic(dev,
1897	"[PLANE:%d:%s] async update cannot change CRTC\n",
1898	plane->base.id, plane->name);
1899	return -EINVAL;
1900	}
1901
1902	funcs = plane->helper_private;
1903	if (!funcs->atomic_async_update) {
1904	drm_dbg_atomic(dev,
1905	"[PLANE:%d:%s] driver does not support async updates\n",
1906	plane->base.id, plane->name);
1907	return -EINVAL;
1908	}
1909
1910	if (new_plane_state->fence) {
1911	drm_dbg_atomic(dev,
1912	"[PLANE:%d:%s] missing fence for async update\n",
1913	plane->base.id, plane->name);
1914	return -EINVAL;
1915	}
1916
1917	/*
1918	* Don't do an async update if there is an outstanding commit modifying
1919	* the plane. This prevents our async update's changes from getting
1920	* overridden by a previous synchronous update's state.
1921	*/
1922	if (old_plane_state->commit &&
1923	!try_wait_for_completion(x: &old_plane_state->commit->hw_done)) {
1924	drm_dbg_atomic(dev,
1925	"[PLANE:%d:%s] inflight previous commit preventing async commit\n",
1926	plane->base.id, plane->name);
1927	return -EBUSY;
1928	}
1929
1930	ret = funcs->atomic_async_check(plane, state);
1931	if (ret != 0)
1932	drm_dbg_atomic(dev,
1933	"[PLANE:%d:%s] driver async check failed\n",
1934	plane->base.id, plane->name);
1935	return ret;
1936	}
1937	EXPORT_SYMBOL(drm_atomic_helper_async_check);
1938
1939	/**
1940	* drm_atomic_helper_async_commit - commit state asynchronously
1941	* @dev: DRM device
1942	* @state: the driver state object
1943	*
1944	* This function commits a state asynchronously, i.e., not vblank
1945	* synchronized. It should be used on a state only when
1946	* drm_atomic_async_check() succeeds. Async commits are not supposed to swap
1947	* the states like normal sync commits, but just do in-place changes on the
1948	* current state.
1949	*
1950	* TODO: Implement full swap instead of doing in-place changes.
1951	*/
1952	void drm_atomic_helper_async_commit(struct drm_device *dev,
1953	struct drm_atomic_state *state)
1954	{
1955	struct drm_plane *plane;
1956	struct drm_plane_state *plane_state;
1957	const struct drm_plane_helper_funcs *funcs;
1958	int i;
1959
1960	for_each_new_plane_in_state(state, plane, plane_state, i) {
1961	struct drm_framebuffer *new_fb = plane_state->fb;
1962	struct drm_framebuffer *old_fb = plane->state->fb;
1963
1964	funcs = plane->helper_private;
1965	funcs->atomic_async_update(plane, state);
1966
1967	/*
1968	* ->atomic_async_update() is supposed to update the
1969	* plane->state in-place, make sure at least common
1970	* properties have been properly updated.
1971	*/
1972	WARN_ON_ONCE(plane->state->fb != new_fb);
1973	WARN_ON_ONCE(plane->state->crtc_x != plane_state->crtc_x);
1974	WARN_ON_ONCE(plane->state->crtc_y != plane_state->crtc_y);
1975	WARN_ON_ONCE(plane->state->src_x != plane_state->src_x);
1976	WARN_ON_ONCE(plane->state->src_y != plane_state->src_y);
1977
1978	/*
1979	* Make sure the FBs have been swapped so that cleanups in the
1980	* new_state performs a cleanup in the old FB.
1981	*/
1982	WARN_ON_ONCE(plane_state->fb != old_fb);
1983	}
1984	}
1985	EXPORT_SYMBOL(drm_atomic_helper_async_commit);
1986
1987	/**
1988	* drm_atomic_helper_commit - commit validated state object
1989	* @dev: DRM device
1990	* @state: the driver state object
1991	* @nonblock: whether nonblocking behavior is requested.
1992	*
1993	* This function commits a with drm_atomic_helper_check() pre-validated state
1994	* object. This can still fail when e.g. the framebuffer reservation fails. This
1995	* function implements nonblocking commits, using
1996	* drm_atomic_helper_setup_commit() and related functions.
1997	*
1998	* Committing the actual hardware state is done through the
1999	* &drm_mode_config_helper_funcs.atomic_commit_tail callback, or its default
2000	* implementation drm_atomic_helper_commit_tail().
2001	*
2002	* RETURNS:
2003	* Zero for success or -errno.
2004	*/
2005	int drm_atomic_helper_commit(struct drm_device *dev,
2006	struct drm_atomic_state *state,
2007	bool nonblock)
2008	{
2009	int ret;
2010
2011	if (state->async_update) {
2012	ret = drm_atomic_helper_prepare_planes(dev, state);
2013	if (ret)
2014	return ret;
2015
2016	drm_atomic_helper_async_commit(dev, state);
2017	drm_atomic_helper_unprepare_planes(dev, state);
2018
2019	return 0;
2020	}
2021
2022	ret = drm_atomic_helper_setup_commit(state, nonblock);
2023	if (ret)
2024	return ret;
2025
2026	INIT_WORK(&state->commit_work, commit_work);
2027
2028	ret = drm_atomic_helper_prepare_planes(dev, state);
2029	if (ret)
2030	return ret;
2031
2032	if (!nonblock) {
2033	ret = drm_atomic_helper_wait_for_fences(dev, state, true);
2034	if (ret)
2035	goto err;
2036	}
2037
2038	/*
2039	* This is the point of no return - everything below never fails except
2040	* when the hw goes bonghits. Which means we can commit the new state on
2041	* the software side now.
2042	*/
2043
2044	ret = drm_atomic_helper_swap_state(state, stall: true);
2045	if (ret)
2046	goto err;
2047
2048	/*
2049	* Everything below can be run asynchronously without the need to grab
2050	* any modeset locks at all under one condition: It must be guaranteed
2051	* that the asynchronous work has either been cancelled (if the driver
2052	* supports it, which at least requires that the framebuffers get
2053	* cleaned up with drm_atomic_helper_cleanup_planes()) or completed
2054	* before the new state gets committed on the software side with
2055	* drm_atomic_helper_swap_state().
2056	*
2057	* This scheme allows new atomic state updates to be prepared and
2058	* checked in parallel to the asynchronous completion of the previous
2059	* update. Which is important since compositors need to figure out the
2060	* composition of the next frame right after having submitted the
2061	* current layout.
2062	*
2063	* NOTE: Commit work has multiple phases, first hardware commit, then
2064	* cleanup. We want them to overlap, hence need system_unbound_wq to
2065	* make sure work items don't artificially stall on each another.
2066	*/
2067
2068	drm_atomic_state_get(state);
2069	if (nonblock)
2070	queue_work(wq: system_unbound_wq, work: &state->commit_work);
2071	else
2072	commit_tail(old_state: state);
2073
2074	return 0;
2075
2076	err:
2077	drm_atomic_helper_unprepare_planes(dev, state);
2078	return ret;
2079	}
2080	EXPORT_SYMBOL(drm_atomic_helper_commit);
2081
2082	/**
2083	* DOC: implementing nonblocking commit
2084	*
2085	* Nonblocking atomic commits should use struct &drm_crtc_commit to sequence
2086	* different operations against each another. Locks, especially struct
2087	* &drm_modeset_lock, should not be held in worker threads or any other
2088	* asynchronous context used to commit the hardware state.
2089	*
2090	* drm_atomic_helper_commit() implements the recommended sequence for
2091	* nonblocking commits, using drm_atomic_helper_setup_commit() internally:
2092	*
2093	* 1. Run drm_atomic_helper_prepare_planes(). Since this can fail and we
2094	* need to propagate out of memory/VRAM errors to userspace, it must be called
2095	* synchronously.
2096	*
2097	* 2. Synchronize with any outstanding nonblocking commit worker threads which
2098	* might be affected by the new state update. This is handled by
2099	* drm_atomic_helper_setup_commit().
2100	*
2101	* Asynchronous workers need to have sufficient parallelism to be able to run
2102	* different atomic commits on different CRTCs in parallel. The simplest way to
2103	* achieve this is by running them on the &system_unbound_wq work queue. Note
2104	* that drivers are not required to split up atomic commits and run an
2105	* individual commit in parallel - userspace is supposed to do that if it cares.
2106	* But it might be beneficial to do that for modesets, since those necessarily
2107	* must be done as one global operation, and enabling or disabling a CRTC can
2108	* take a long time. But even that is not required.
2109	*
2110	* IMPORTANT: A &drm_atomic_state update for multiple CRTCs is sequenced
2111	* against all CRTCs therein. Therefore for atomic state updates which only flip
2112	* planes the driver must not get the struct &drm_crtc_state of unrelated CRTCs
2113	* in its atomic check code: This would prevent committing of atomic updates to
2114	* multiple CRTCs in parallel. In general, adding additional state structures
2115	* should be avoided as much as possible, because this reduces parallelism in
2116	* (nonblocking) commits, both due to locking and due to commit sequencing
2117	* requirements.
2118	*
2119	* 3. The software state is updated synchronously with
2120	* drm_atomic_helper_swap_state(). Doing this under the protection of all modeset
2121	* locks means concurrent callers never see inconsistent state. Note that commit
2122	* workers do not hold any locks; their access is only coordinated through
2123	* ordering. If workers would access state only through the pointers in the
2124	* free-standing state objects (currently not the case for any driver) then even
2125	* multiple pending commits could be in-flight at the same time.
2126	*
2127	* 4. Schedule a work item to do all subsequent steps, using the split-out
2128	* commit helpers: a) pre-plane commit b) plane commit c) post-plane commit and
2129	* then cleaning up the framebuffers after the old framebuffer is no longer
2130	* being displayed. The scheduled work should synchronize against other workers
2131	* using the &drm_crtc_commit infrastructure as needed. See
2132	* drm_atomic_helper_setup_commit() for more details.
2133	*/
2134
2135	static int stall_checks(struct drm_crtc *crtc, bool nonblock)
2136	{
2137	struct drm_crtc_commit *commit, *stall_commit = NULL;
2138	bool completed = true;
2139	int i;
2140	long ret = 0;
2141
2142	spin_lock(lock: &crtc->commit_lock);
2143	i = 0;
2144	list_for_each_entry(commit, &crtc->commit_list, commit_entry) {
2145	if (i == 0) {
2146	completed = try_wait_for_completion(x: &commit->flip_done);
2147	/*
2148	* Userspace is not allowed to get ahead of the previous
2149	* commit with nonblocking ones.
2150	*/
2151	if (!completed && nonblock) {
2152	spin_unlock(lock: &crtc->commit_lock);
2153	drm_dbg_atomic(crtc->dev,
2154	"[CRTC:%d:%s] busy with a previous commit\n",
2155	crtc->base.id, crtc->name);
2156
2157	return -EBUSY;
2158	}
2159	} else if (i == 1) {
2160	stall_commit = drm_crtc_commit_get(commit);
2161	break;
2162	}
2163
2164	i++;
2165	}
2166	spin_unlock(lock: &crtc->commit_lock);
2167
2168	if (!stall_commit)
2169	return 0;
2170
2171	/* We don't want to let commits get ahead of cleanup work too much,
2172	* stalling on 2nd previous commit means triple-buffer won't ever stall.
2173	*/
2174	ret = wait_for_completion_interruptible_timeout(x: &stall_commit->cleanup_done,
2175	timeout: 10*HZ);
2176	if (ret == 0)
2177	drm_err(crtc->dev, "[CRTC:%d:%s] cleanup_done timed out\n",
2178	crtc->base.id, crtc->name);
2179
2180	drm_crtc_commit_put(commit: stall_commit);
2181
2182	return ret < 0 ? ret : 0;
2183	}
2184
2185	static void release_crtc_commit(struct completion *completion)
2186	{
2187	struct drm_crtc_commit *commit = container_of(completion,
2188	typeof(*commit),
2189	flip_done);
2190
2191	drm_crtc_commit_put(commit);
2192	}
2193
2194	static void init_commit(struct drm_crtc_commit *commit, struct drm_crtc *crtc)
2195	{
2196	init_completion(x: &commit->flip_done);
2197	init_completion(x: &commit->hw_done);
2198	init_completion(x: &commit->cleanup_done);
2199	INIT_LIST_HEAD(list: &commit->commit_entry);
2200	kref_init(kref: &commit->ref);
2201	commit->crtc = crtc;
2202	}
2203
2204	static struct drm_crtc_commit *
2205	crtc_or_fake_commit(struct drm_atomic_state *state, struct drm_crtc *crtc)
2206	{
2207	if (crtc) {
2208	struct drm_crtc_state *new_crtc_state;
2209
2210	new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
2211
2212	return new_crtc_state->commit;
2213	}
2214
2215	if (!state->fake_commit) {
2216	state->fake_commit = kzalloc(size: sizeof(*state->fake_commit), GFP_KERNEL);
2217	if (!state->fake_commit)
2218	return NULL;
2219
2220	init_commit(commit: state->fake_commit, NULL);
2221	}
2222
2223	return state->fake_commit;
2224	}
2225
2226	/**
2227	* drm_atomic_helper_setup_commit - setup possibly nonblocking commit
2228	* @state: new modeset state to be committed
2229	* @nonblock: whether nonblocking behavior is requested.
2230	*
2231	* This function prepares @state to be used by the atomic helper's support for
2232	* nonblocking commits. Drivers using the nonblocking commit infrastructure
2233	* should always call this function from their
2234	* &drm_mode_config_funcs.atomic_commit hook.
2235	*
2236	* Drivers that need to extend the commit setup to private objects can use the
2237	* &drm_mode_config_helper_funcs.atomic_commit_setup hook.
2238	*
2239	* To be able to use this support drivers need to use a few more helper
2240	* functions. drm_atomic_helper_wait_for_dependencies() must be called before
2241	* actually committing the hardware state, and for nonblocking commits this call
2242	* must be placed in the async worker. See also drm_atomic_helper_swap_state()
2243	* and its stall parameter, for when a driver's commit hooks look at the
2244	* &drm_crtc.state, &drm_plane.state or &drm_connector.state pointer directly.
2245	*
2246	* Completion of the hardware commit step must be signalled using
2247	* drm_atomic_helper_commit_hw_done(). After this step the driver is not allowed
2248	* to read or change any permanent software or hardware modeset state. The only
2249	* exception is state protected by other means than &drm_modeset_lock locks.
2250	* Only the free standing @state with pointers to the old state structures can
2251	* be inspected, e.g. to clean up old buffers using
2252	* drm_atomic_helper_cleanup_planes().
2253	*
2254	* At the very end, before cleaning up @state drivers must call
2255	* drm_atomic_helper_commit_cleanup_done().
2256	*
2257	* This is all implemented by in drm_atomic_helper_commit(), giving drivers a
2258	* complete and easy-to-use default implementation of the atomic_commit() hook.
2259	*
2260	* The tracking of asynchronously executed and still pending commits is done
2261	* using the core structure &drm_crtc_commit.
2262	*
2263	* By default there's no need to clean up resources allocated by this function
2264	* explicitly: drm_atomic_state_default_clear() will take care of that
2265	* automatically.
2266	*
2267	* Returns:
2268	*
2269	* 0 on success. -EBUSY when userspace schedules nonblocking commits too fast,
2270	* -ENOMEM on allocation failures and -EINTR when a signal is pending.
2271	*/
2272	int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
2273	bool nonblock)
2274	{
2275	struct drm_crtc *crtc;
2276	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
2277	struct drm_connector *conn;
2278	struct drm_connector_state *old_conn_state, *new_conn_state;
2279	struct drm_plane *plane;
2280	struct drm_plane_state *old_plane_state, *new_plane_state;
2281	struct drm_crtc_commit *commit;
2282	const struct drm_mode_config_helper_funcs *funcs;
2283	int i, ret;
2284
2285	funcs = state->dev->mode_config.helper_private;
2286
2287	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2288	commit = kzalloc(size: sizeof(*commit), GFP_KERNEL);
2289	if (!commit)
2290	return -ENOMEM;
2291
2292	init_commit(commit, crtc);
2293
2294	new_crtc_state->commit = commit;
2295
2296	ret = stall_checks(crtc, nonblock);
2297	if (ret)
2298	return ret;
2299
2300	/*
2301	* Drivers only send out events when at least either current or
2302	* new CRTC state is active. Complete right away if everything
2303	* stays off.
2304	*/
2305	if (!old_crtc_state->active && !new_crtc_state->active) {
2306	complete_all(&commit->flip_done);
2307	continue;
2308	}
2309
2310	/* Legacy cursor updates are fully unsynced. */
2311	if (state->legacy_cursor_update) {
2312	complete_all(&commit->flip_done);
2313	continue;
2314	}
2315
2316	if (!new_crtc_state->event) {
2317	commit->event = kzalloc(size: sizeof(*commit->event),
2318	GFP_KERNEL);
2319	if (!commit->event)
2320	return -ENOMEM;
2321
2322	new_crtc_state->event = commit->event;
2323	}
2324
2325	new_crtc_state->event->base.completion = &commit->flip_done;
2326	new_crtc_state->event->base.completion_release = release_crtc_commit;
2327	drm_crtc_commit_get(commit);
2328
2329	commit->abort_completion = true;
2330
2331	state->crtcs[i].commit = commit;
2332	drm_crtc_commit_get(commit);
2333	}
2334
2335	for_each_oldnew_connector_in_state(state, conn, old_conn_state, new_conn_state, i) {
2336	/*
2337	* Userspace is not allowed to get ahead of the previous
2338	* commit with nonblocking ones.
2339	*/
2340	if (nonblock && old_conn_state->commit &&
2341	!try_wait_for_completion(x: &old_conn_state->commit->flip_done)) {
2342	drm_dbg_atomic(conn->dev,
2343	"[CONNECTOR:%d:%s] busy with a previous commit\n",
2344	conn->base.id, conn->name);
2345
2346	return -EBUSY;
2347	}
2348
2349	/* Always track connectors explicitly for e.g. link retraining. */
2350	commit = crtc_or_fake_commit(state, crtc: new_conn_state->crtc ?: old_conn_state->crtc);
2351	if (!commit)
2352	return -ENOMEM;
2353
2354	new_conn_state->commit = drm_crtc_commit_get(commit);
2355	}
2356
2357	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
2358	/*
2359	* Userspace is not allowed to get ahead of the previous
2360	* commit with nonblocking ones.
2361	*/
2362	if (nonblock && old_plane_state->commit &&
2363	!try_wait_for_completion(x: &old_plane_state->commit->flip_done)) {
2364	drm_dbg_atomic(plane->dev,
2365	"[PLANE:%d:%s] busy with a previous commit\n",
2366	plane->base.id, plane->name);
2367
2368	return -EBUSY;
2369	}
2370
2371	/* Always track planes explicitly for async pageflip support. */
2372	commit = crtc_or_fake_commit(state, crtc: new_plane_state->crtc ?: old_plane_state->crtc);
2373	if (!commit)
2374	return -ENOMEM;
2375
2376	new_plane_state->commit = drm_crtc_commit_get(commit);
2377	}
2378
2379	if (funcs && funcs->atomic_commit_setup)
2380	return funcs->atomic_commit_setup(state);
2381
2382	return 0;
2383	}
2384	EXPORT_SYMBOL(drm_atomic_helper_setup_commit);
2385
2386	/**
2387	* drm_atomic_helper_wait_for_dependencies - wait for required preceding commits
2388	* @old_state: atomic state object with old state structures
2389	*
2390	* This function waits for all preceding commits that touch the same CRTC as
2391	* @old_state to both be committed to the hardware (as signalled by
2392	* drm_atomic_helper_commit_hw_done()) and executed by the hardware (as signalled
2393	* by calling drm_crtc_send_vblank_event() on the &drm_crtc_state.event).
2394	*
2395	* This is part of the atomic helper support for nonblocking commits, see
2396	* drm_atomic_helper_setup_commit() for an overview.
2397	*/
2398	void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *old_state)
2399	{
2400	struct drm_crtc *crtc;
2401	struct drm_crtc_state *old_crtc_state;
2402	struct drm_plane *plane;
2403	struct drm_plane_state *old_plane_state;
2404	struct drm_connector *conn;
2405	struct drm_connector_state *old_conn_state;
2406	int i;
2407	long ret;
2408
2409	for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i) {
2410	ret = drm_crtc_commit_wait(commit: old_crtc_state->commit);
2411	if (ret)
2412	drm_err(crtc->dev,
2413	"[CRTC:%d:%s] commit wait timed out\n",
2414	crtc->base.id, crtc->name);
2415	}
2416
2417	for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
2418	ret = drm_crtc_commit_wait(commit: old_conn_state->commit);
2419	if (ret)
2420	drm_err(conn->dev,
2421	"[CONNECTOR:%d:%s] commit wait timed out\n",
2422	conn->base.id, conn->name);
2423	}
2424
2425	for_each_old_plane_in_state(old_state, plane, old_plane_state, i) {
2426	ret = drm_crtc_commit_wait(commit: old_plane_state->commit);
2427	if (ret)
2428	drm_err(plane->dev,
2429	"[PLANE:%d:%s] commit wait timed out\n",
2430	plane->base.id, plane->name);
2431	}
2432	}
2433	EXPORT_SYMBOL(drm_atomic_helper_wait_for_dependencies);
2434
2435	/**
2436	* drm_atomic_helper_fake_vblank - fake VBLANK events if needed
2437	* @old_state: atomic state object with old state structures
2438	*
2439	* This function walks all CRTCs and fakes VBLANK events on those with
2440	* &drm_crtc_state.no_vblank set to true and &drm_crtc_state.event != NULL.
2441	* The primary use of this function is writeback connectors working in oneshot
2442	* mode and faking VBLANK events. In this case they only fake the VBLANK event
2443	* when a job is queued, and any change to the pipeline that does not touch the
2444	* connector is leading to timeouts when calling
2445	* drm_atomic_helper_wait_for_vblanks() or
2446	* drm_atomic_helper_wait_for_flip_done(). In addition to writeback
2447	* connectors, this function can also fake VBLANK events for CRTCs without
2448	* VBLANK interrupt.
2449	*
2450	* This is part of the atomic helper support for nonblocking commits, see
2451	* drm_atomic_helper_setup_commit() for an overview.
2452	*/
2453	void drm_atomic_helper_fake_vblank(struct drm_atomic_state *old_state)
2454	{
2455	struct drm_crtc_state *new_crtc_state;
2456	struct drm_crtc *crtc;
2457	int i;
2458
2459	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
2460	unsigned long flags;
2461
2462	if (!new_crtc_state->no_vblank)
2463	continue;
2464
2465	spin_lock_irqsave(&old_state->dev->event_lock, flags);
2466	if (new_crtc_state->event) {
2467	drm_crtc_send_vblank_event(crtc,
2468	e: new_crtc_state->event);
2469	new_crtc_state->event = NULL;
2470	}
2471	spin_unlock_irqrestore(lock: &old_state->dev->event_lock, flags);
2472	}
2473	}
2474	EXPORT_SYMBOL(drm_atomic_helper_fake_vblank);
2475
2476	/**
2477	* drm_atomic_helper_commit_hw_done - setup possible nonblocking commit
2478	* @old_state: atomic state object with old state structures
2479	*
2480	* This function is used to signal completion of the hardware commit step. After
2481	* this step the driver is not allowed to read or change any permanent software
2482	* or hardware modeset state. The only exception is state protected by other
2483	* means than &drm_modeset_lock locks.
2484	*
2485	* Drivers should try to postpone any expensive or delayed cleanup work after
2486	* this function is called.
2487	*
2488	* This is part of the atomic helper support for nonblocking commits, see
2489	* drm_atomic_helper_setup_commit() for an overview.
2490	*/
2491	void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *old_state)
2492	{
2493	struct drm_crtc *crtc;
2494	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
2495	struct drm_crtc_commit *commit;
2496	int i;
2497
2498	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
2499	commit = new_crtc_state->commit;
2500	if (!commit)
2501	continue;
2502
2503	/*
2504	* copy new_crtc_state->commit to old_crtc_state->commit,
2505	* it's unsafe to touch new_crtc_state after hw_done,
2506	* but we still need to do so in cleanup_done().
2507	*/
2508	if (old_crtc_state->commit)
2509	drm_crtc_commit_put(commit: old_crtc_state->commit);
2510
2511	old_crtc_state->commit = drm_crtc_commit_get(commit);
2512
2513	/* backend must have consumed any event by now */
2514	WARN_ON(new_crtc_state->event);
2515	complete_all(&commit->hw_done);
2516	}
2517
2518	if (old_state->fake_commit) {
2519	complete_all(&old_state->fake_commit->hw_done);
2520	complete_all(&old_state->fake_commit->flip_done);
2521	}
2522	}
2523	EXPORT_SYMBOL(drm_atomic_helper_commit_hw_done);
2524
2525	/**
2526	* drm_atomic_helper_commit_cleanup_done - signal completion of commit
2527	* @old_state: atomic state object with old state structures
2528	*
2529	* This signals completion of the atomic update @old_state, including any
2530	* cleanup work. If used, it must be called right before calling
2531	* drm_atomic_state_put().
2532	*
2533	* This is part of the atomic helper support for nonblocking commits, see
2534	* drm_atomic_helper_setup_commit() for an overview.
2535	*/
2536	void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *old_state)
2537	{
2538	struct drm_crtc *crtc;
2539	struct drm_crtc_state *old_crtc_state;
2540	struct drm_crtc_commit *commit;
2541	int i;
2542
2543	for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i) {
2544	commit = old_crtc_state->commit;
2545	if (WARN_ON(!commit))
2546	continue;
2547
2548	complete_all(&commit->cleanup_done);
2549	WARN_ON(!try_wait_for_completion(&commit->hw_done));
2550
2551	spin_lock(lock: &crtc->commit_lock);
2552	list_del(entry: &commit->commit_entry);
2553	spin_unlock(lock: &crtc->commit_lock);
2554	}
2555
2556	if (old_state->fake_commit) {
2557	complete_all(&old_state->fake_commit->cleanup_done);
2558	WARN_ON(!try_wait_for_completion(&old_state->fake_commit->hw_done));
2559	}
2560	}
2561	EXPORT_SYMBOL(drm_atomic_helper_commit_cleanup_done);
2562
2563	/**
2564	* drm_atomic_helper_prepare_planes - prepare plane resources before commit
2565	* @dev: DRM device
2566	* @state: atomic state object with new state structures
2567	*
2568	* This function prepares plane state, specifically framebuffers, for the new
2569	* configuration, by calling &drm_plane_helper_funcs.prepare_fb. If any failure
2570	* is encountered this function will call &drm_plane_helper_funcs.cleanup_fb on
2571	* any already successfully prepared framebuffer.
2572	*
2573	* Returns:
2574	* 0 on success, negative error code on failure.
2575	*/
2576	int drm_atomic_helper_prepare_planes(struct drm_device *dev,
2577	struct drm_atomic_state *state)
2578	{
2579	struct drm_connector *connector;
2580	struct drm_connector_state *new_conn_state;
2581	struct drm_plane *plane;
2582	struct drm_plane_state *new_plane_state;
2583	int ret, i, j;
2584
2585	for_each_new_connector_in_state(state, connector, new_conn_state, i) {
2586	if (!new_conn_state->writeback_job)
2587	continue;
2588
2589	ret = drm_writeback_prepare_job(job: new_conn_state->writeback_job);
2590	if (ret < 0)
2591	return ret;
2592	}
2593
2594	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2595	const struct drm_plane_helper_funcs *funcs;
2596
2597	funcs = plane->helper_private;
2598
2599	if (funcs->prepare_fb) {
2600	ret = funcs->prepare_fb(plane, new_plane_state);
2601	if (ret)
2602	goto fail_prepare_fb;
2603	} else {
2604	WARN_ON_ONCE(funcs->cleanup_fb);
2605
2606	if (!drm_core_check_feature(dev, feature: DRIVER_GEM))
2607	continue;
2608
2609	ret = drm_gem_plane_helper_prepare_fb(plane, state: new_plane_state);
2610	if (ret)
2611	goto fail_prepare_fb;
2612	}
2613	}
2614
2615	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2616	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2617
2618	if (funcs->begin_fb_access) {
2619	ret = funcs->begin_fb_access(plane, new_plane_state);
2620	if (ret)
2621	goto fail_begin_fb_access;
2622	}
2623	}
2624
2625	return 0;
2626
2627	fail_begin_fb_access:
2628	for_each_new_plane_in_state(state, plane, new_plane_state, j) {
2629	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2630
2631	if (j >= i)
2632	continue;
2633
2634	if (funcs->end_fb_access)
2635	funcs->end_fb_access(plane, new_plane_state);
2636	}
2637	i = j; /* set i to upper limit to cleanup all planes */
2638	fail_prepare_fb:
2639	for_each_new_plane_in_state(state, plane, new_plane_state, j) {
2640	const struct drm_plane_helper_funcs *funcs;
2641
2642	if (j >= i)
2643	continue;
2644
2645	funcs = plane->helper_private;
2646
2647	if (funcs->cleanup_fb)
2648	funcs->cleanup_fb(plane, new_plane_state);
2649	}
2650
2651	return ret;
2652	}
2653	EXPORT_SYMBOL(drm_atomic_helper_prepare_planes);
2654
2655	/**
2656	* drm_atomic_helper_unprepare_planes - release plane resources on aborts
2657	* @dev: DRM device
2658	* @state: atomic state object with old state structures
2659	*
2660	* This function cleans up plane state, specifically framebuffers, from the
2661	* atomic state. It undoes the effects of drm_atomic_helper_prepare_planes()
2662	* when aborting an atomic commit. For cleaning up after a successful commit
2663	* use drm_atomic_helper_cleanup_planes().
2664	*/
2665	void drm_atomic_helper_unprepare_planes(struct drm_device *dev,
2666	struct drm_atomic_state *state)
2667	{
2668	struct drm_plane *plane;
2669	struct drm_plane_state *new_plane_state;
2670	int i;
2671
2672	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2673	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2674
2675	if (funcs->end_fb_access)
2676	funcs->end_fb_access(plane, new_plane_state);
2677	}
2678
2679	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2680	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2681
2682	if (funcs->cleanup_fb)
2683	funcs->cleanup_fb(plane, new_plane_state);
2684	}
2685	}
2686	EXPORT_SYMBOL(drm_atomic_helper_unprepare_planes);
2687
2688	static bool plane_crtc_active(const struct drm_plane_state *state)
2689	{
2690	return state->crtc && state->crtc->state->active;
2691	}
2692
2693	/**
2694	* drm_atomic_helper_commit_planes - commit plane state
2695	* @dev: DRM device
2696	* @old_state: atomic state object with old state structures
2697	* @flags: flags for committing plane state
2698	*
2699	* This function commits the new plane state using the plane and atomic helper
2700	* functions for planes and CRTCs. It assumes that the atomic state has already
2701	* been pushed into the relevant object state pointers, since this step can no
2702	* longer fail.
2703	*
2704	* It still requires the global state object @old_state to know which planes and
2705	* crtcs need to be updated though.
2706	*
2707	* Note that this function does all plane updates across all CRTCs in one step.
2708	* If the hardware can't support this approach look at
2709	* drm_atomic_helper_commit_planes_on_crtc() instead.
2710	*
2711	* Plane parameters can be updated by applications while the associated CRTC is
2712	* disabled. The DRM/KMS core will store the parameters in the plane state,
2713	* which will be available to the driver when the CRTC is turned on. As a result
2714	* most drivers don't need to be immediately notified of plane updates for a
2715	* disabled CRTC.
2716	*
2717	* Unless otherwise needed, drivers are advised to set the ACTIVE_ONLY flag in
2718	* @flags in order not to receive plane update notifications related to a
2719	* disabled CRTC. This avoids the need to manually ignore plane updates in
2720	* driver code when the driver and/or hardware can't or just don't need to deal
2721	* with updates on disabled CRTCs, for example when supporting runtime PM.
2722	*
2723	* Drivers may set the NO_DISABLE_AFTER_MODESET flag in @flags if the relevant
2724	* display controllers require to disable a CRTC's planes when the CRTC is
2725	* disabled. This function would skip the &drm_plane_helper_funcs.atomic_disable
2726	* call for a plane if the CRTC of the old plane state needs a modesetting
2727	* operation. Of course, the drivers need to disable the planes in their CRTC
2728	* disable callbacks since no one else would do that.
2729	*
2730	* The drm_atomic_helper_commit() default implementation doesn't set the
2731	* ACTIVE_ONLY flag to most closely match the behaviour of the legacy helpers.
2732	* This should not be copied blindly by drivers.
2733	*/
2734	void drm_atomic_helper_commit_planes(struct drm_device *dev,
2735	struct drm_atomic_state *old_state,
2736	uint32_t flags)
2737	{
2738	struct drm_crtc *crtc;
2739	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
2740	struct drm_plane *plane;
2741	struct drm_plane_state *old_plane_state, *new_plane_state;
2742	int i;
2743	bool active_only = flags & DRM_PLANE_COMMIT_ACTIVE_ONLY;
2744	bool no_disable = flags & DRM_PLANE_COMMIT_NO_DISABLE_AFTER_MODESET;
2745
2746	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
2747	const struct drm_crtc_helper_funcs *funcs;
2748
2749	funcs = crtc->helper_private;
2750
2751	if (!funcs || !funcs->atomic_begin)
2752	continue;
2753
2754	if (active_only && !new_crtc_state->active)
2755	continue;
2756
2757	funcs->atomic_begin(crtc, old_state);
2758	}
2759
2760	for_each_oldnew_plane_in_state(old_state, plane, old_plane_state, new_plane_state, i) {
2761	const struct drm_plane_helper_funcs *funcs;
2762	bool disabling;
2763
2764	funcs = plane->helper_private;
2765
2766	if (!funcs)
2767	continue;
2768
2769	disabling = drm_atomic_plane_disabling(old_plane_state,
2770	new_plane_state);
2771
2772	if (active_only) {
2773	/*
2774	* Skip planes related to inactive CRTCs. If the plane
2775	* is enabled use the state of the current CRTC. If the
2776	* plane is being disabled use the state of the old
2777	* CRTC to avoid skipping planes being disabled on an
2778	* active CRTC.
2779	*/
2780	if (!disabling && !plane_crtc_active(state: new_plane_state))
2781	continue;
2782	if (disabling && !plane_crtc_active(state: old_plane_state))
2783	continue;
2784	}
2785
2786	/*
2787	* Special-case disabling the plane if drivers support it.
2788	*/
2789	if (disabling && funcs->atomic_disable) {
2790	struct drm_crtc_state *crtc_state;
2791
2792	crtc_state = old_plane_state->crtc->state;
2793
2794	if (drm_atomic_crtc_needs_modeset(state: crtc_state) &&
2795	no_disable)
2796	continue;
2797
2798	funcs->atomic_disable(plane, old_state);
2799	} else if (new_plane_state->crtc || disabling) {
2800	funcs->atomic_update(plane, old_state);
2801
2802	if (!disabling && funcs->atomic_enable) {
2803	if (drm_atomic_plane_enabling(old_plane_state, new_plane_state))
2804	funcs->atomic_enable(plane, old_state);
2805	}
2806	}
2807	}
2808
2809	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
2810	const struct drm_crtc_helper_funcs *funcs;
2811
2812	funcs = crtc->helper_private;
2813
2814	if (!funcs || !funcs->atomic_flush)
2815	continue;
2816
2817	if (active_only && !new_crtc_state->active)
2818	continue;
2819
2820	funcs->atomic_flush(crtc, old_state);
2821	}
2822
2823	/*
2824	* Signal end of framebuffer access here before hw_done. After hw_done,
2825	* a later commit might have already released the plane state.
2826	*/
2827	for_each_old_plane_in_state(old_state, plane, old_plane_state, i) {
2828	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2829
2830	if (funcs->end_fb_access)
2831	funcs->end_fb_access(plane, old_plane_state);
2832	}
2833	}
2834	EXPORT_SYMBOL(drm_atomic_helper_commit_planes);
2835
2836	/**
2837	* drm_atomic_helper_commit_planes_on_crtc - commit plane state for a CRTC
2838	* @old_crtc_state: atomic state object with the old CRTC state
2839	*
2840	* This function commits the new plane state using the plane and atomic helper
2841	* functions for planes on the specific CRTC. It assumes that the atomic state
2842	* has already been pushed into the relevant object state pointers, since this
2843	* step can no longer fail.
2844	*
2845	* This function is useful when plane updates should be done CRTC-by-CRTC
2846	* instead of one global step like drm_atomic_helper_commit_planes() does.
2847	*
2848	* This function can only be savely used when planes are not allowed to move
2849	* between different CRTCs because this function doesn't handle inter-CRTC
2850	* dependencies. Callers need to ensure that either no such dependencies exist,
2851	* resolve them through ordering of commit calls or through some other means.
2852	*/
2853	void
2854	drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state)
2855	{
2856	const struct drm_crtc_helper_funcs *crtc_funcs;
2857	struct drm_crtc *crtc = old_crtc_state->crtc;
2858	struct drm_atomic_state *old_state = old_crtc_state->state;
2859	struct drm_crtc_state *new_crtc_state =
2860	drm_atomic_get_new_crtc_state(state: old_state, crtc);
2861	struct drm_plane *plane;
2862	unsigned int plane_mask;
2863
2864	plane_mask = old_crtc_state->plane_mask;
2865	plane_mask |= new_crtc_state->plane_mask;
2866
2867	crtc_funcs = crtc->helper_private;
2868	if (crtc_funcs && crtc_funcs->atomic_begin)
2869	crtc_funcs->atomic_begin(crtc, old_state);
2870
2871	drm_for_each_plane_mask(plane, crtc->dev, plane_mask) {
2872	struct drm_plane_state *old_plane_state =
2873	drm_atomic_get_old_plane_state(state: old_state, plane);
2874	struct drm_plane_state *new_plane_state =
2875	drm_atomic_get_new_plane_state(state: old_state, plane);
2876	const struct drm_plane_helper_funcs *plane_funcs;
2877	bool disabling;
2878
2879	plane_funcs = plane->helper_private;
2880
2881	if (!old_plane_state || !plane_funcs)
2882	continue;
2883
2884	WARN_ON(new_plane_state->crtc &&
2885	new_plane_state->crtc != crtc);
2886
2887	disabling = drm_atomic_plane_disabling(old_plane_state, new_plane_state);
2888
2889	if (disabling && plane_funcs->atomic_disable) {
2890	plane_funcs->atomic_disable(plane, old_state);
2891	} else if (new_plane_state->crtc || disabling) {
2892	plane_funcs->atomic_update(plane, old_state);
2893
2894	if (!disabling && plane_funcs->atomic_enable) {
2895	if (drm_atomic_plane_enabling(old_plane_state, new_plane_state))
2896	plane_funcs->atomic_enable(plane, old_state);
2897	}
2898	}
2899	}
2900
2901	if (crtc_funcs && crtc_funcs->atomic_flush)
2902	crtc_funcs->atomic_flush(crtc, old_state);
2903	}
2904	EXPORT_SYMBOL(drm_atomic_helper_commit_planes_on_crtc);
2905
2906	/**
2907	* drm_atomic_helper_disable_planes_on_crtc - helper to disable CRTC's planes
2908	* @old_crtc_state: atomic state object with the old CRTC state
2909	* @atomic: if set, synchronize with CRTC's atomic_begin/flush hooks
2910	*
2911	* Disables all planes associated with the given CRTC. This can be
2912	* used for instance in the CRTC helper atomic_disable callback to disable
2913	* all planes.
2914	*
2915	* If the atomic-parameter is set the function calls the CRTC's
2916	* atomic_begin hook before and atomic_flush hook after disabling the
2917	* planes.
2918	*
2919	* It is a bug to call this function without having implemented the
2920	* &drm_plane_helper_funcs.atomic_disable plane hook.
2921	*/
2922	void
2923	drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc_state *old_crtc_state,
2924	bool atomic)
2925	{
2926	struct drm_crtc *crtc = old_crtc_state->crtc;
2927	const struct drm_crtc_helper_funcs *crtc_funcs =
2928	crtc->helper_private;
2929	struct drm_plane *plane;
2930
2931	if (atomic && crtc_funcs && crtc_funcs->atomic_begin)
2932	crtc_funcs->atomic_begin(crtc, NULL);
2933
2934	drm_atomic_crtc_state_for_each_plane(plane, old_crtc_state) {
2935	const struct drm_plane_helper_funcs *plane_funcs =
2936	plane->helper_private;
2937
2938	if (!plane_funcs)
2939	continue;
2940
2941	WARN_ON(!plane_funcs->atomic_disable);
2942	if (plane_funcs->atomic_disable)
2943	plane_funcs->atomic_disable(plane, NULL);
2944	}
2945
2946	if (atomic && crtc_funcs && crtc_funcs->atomic_flush)
2947	crtc_funcs->atomic_flush(crtc, NULL);
2948	}
2949	EXPORT_SYMBOL(drm_atomic_helper_disable_planes_on_crtc);
2950
2951	/**
2952	* drm_atomic_helper_cleanup_planes - cleanup plane resources after commit
2953	* @dev: DRM device
2954	* @old_state: atomic state object with old state structures
2955	*
2956	* This function cleans up plane state, specifically framebuffers, from the old
2957	* configuration. Hence the old configuration must be perserved in @old_state to
2958	* be able to call this function.
2959	*
2960	* This function may not be called on the new state when the atomic update
2961	* fails at any point after calling drm_atomic_helper_prepare_planes(). Use
2962	* drm_atomic_helper_unprepare_planes() in this case.
2963	*/
2964	void drm_atomic_helper_cleanup_planes(struct drm_device *dev,
2965	struct drm_atomic_state *old_state)
2966	{
2967	struct drm_plane *plane;
2968	struct drm_plane_state *old_plane_state;
2969	int i;
2970
2971	for_each_old_plane_in_state(old_state, plane, old_plane_state, i) {
2972	const struct drm_plane_helper_funcs *funcs = plane->helper_private;
2973
2974	if (funcs->cleanup_fb)
2975	funcs->cleanup_fb(plane, old_plane_state);
2976	}
2977	}
2978	EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
2979
2980	/**
2981	* drm_atomic_helper_swap_state - store atomic state into current sw state
2982	* @state: atomic state
2983	* @stall: stall for preceding commits
2984	*
2985	* This function stores the atomic state into the current state pointers in all
2986	* driver objects. It should be called after all failing steps have been done
2987	* and succeeded, but before the actual hardware state is committed.
2988	*
2989	* For cleanup and error recovery the current state for all changed objects will
2990	* be swapped into @state.
2991	*
2992	* With that sequence it fits perfectly into the plane prepare/cleanup sequence:
2993	*
2994	* 1. Call drm_atomic_helper_prepare_planes() with the staged atomic state.
2995	*
2996	* 2. Do any other steps that might fail.
2997	*
2998	* 3. Put the staged state into the current state pointers with this function.
2999	*
3000	* 4. Actually commit the hardware state.
3001	*
3002	* 5. Call drm_atomic_helper_cleanup_planes() with @state, which since step 3
3003	* contains the old state. Also do any other cleanup required with that state.
3004	*
3005	* @stall must be set when nonblocking commits for this driver directly access
3006	* the &drm_plane.state, &drm_crtc.state or &drm_connector.state pointer. With
3007	* the current atomic helpers this is almost always the case, since the helpers
3008	* don't pass the right state structures to the callbacks.
3009	*
3010	* Returns:
3011	*
3012	* Returns 0 on success. Can return -ERESTARTSYS when @stall is true and the
3013	* waiting for the previous commits has been interrupted.
3014	*/
3015	int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
3016	bool stall)
3017	{
3018	int i, ret;
3019	struct drm_connector *connector;
3020	struct drm_connector_state *old_conn_state, *new_conn_state;
3021	struct drm_crtc *crtc;
3022	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
3023	struct drm_plane *plane;
3024	struct drm_plane_state *old_plane_state, *new_plane_state;
3025	struct drm_crtc_commit *commit;
3026	struct drm_private_obj *obj;
3027	struct drm_private_state *old_obj_state, *new_obj_state;
3028
3029	if (stall) {
3030	/*
3031	* We have to stall for hw_done here before
3032	* drm_atomic_helper_wait_for_dependencies() because flip
3033	* depth > 1 is not yet supported by all drivers. As long as
3034	* obj->state is directly dereferenced anywhere in the drivers
3035	* atomic_commit_tail function, then it's unsafe to swap state
3036	* before drm_atomic_helper_commit_hw_done() is called.
3037	*/
3038
3039	for_each_old_crtc_in_state(state, crtc, old_crtc_state, i) {
3040	commit = old_crtc_state->commit;
3041
3042	if (!commit)
3043	continue;
3044
3045	ret = wait_for_completion_interruptible(x: &commit->hw_done);
3046	if (ret)
3047	return ret;
3048	}
3049
3050	for_each_old_connector_in_state(state, connector, old_conn_state, i) {
3051	commit = old_conn_state->commit;
3052
3053	if (!commit)
3054	continue;
3055
3056	ret = wait_for_completion_interruptible(x: &commit->hw_done);
3057	if (ret)
3058	return ret;
3059	}
3060
3061	for_each_old_plane_in_state(state, plane, old_plane_state, i) {
3062	commit = old_plane_state->commit;
3063
3064	if (!commit)
3065	continue;
3066
3067	ret = wait_for_completion_interruptible(x: &commit->hw_done);
3068	if (ret)
3069	return ret;
3070	}
3071	}
3072
3073	for_each_oldnew_connector_in_state(state, connector, old_conn_state, new_conn_state, i) {
3074	WARN_ON(connector->state != old_conn_state);
3075
3076	old_conn_state->state = state;
3077	new_conn_state->state = NULL;
3078
3079	state->connectors[i].state = old_conn_state;
3080	connector->state = new_conn_state;
3081	}
3082
3083	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
3084	WARN_ON(crtc->state != old_crtc_state);
3085
3086	old_crtc_state->state = state;
3087	new_crtc_state->state = NULL;
3088
3089	state->crtcs[i].state = old_crtc_state;
3090	crtc->state = new_crtc_state;
3091
3092	if (new_crtc_state->commit) {
3093	spin_lock(lock: &crtc->commit_lock);
3094	list_add(new: &new_crtc_state->commit->commit_entry,
3095	head: &crtc->commit_list);
3096	spin_unlock(lock: &crtc->commit_lock);
3097
3098	new_crtc_state->commit->event = NULL;
3099	}
3100	}
3101
3102	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
3103	WARN_ON(plane->state != old_plane_state);
3104
3105	old_plane_state->state = state;
3106	new_plane_state->state = NULL;
3107
3108	state->planes[i].state = old_plane_state;
3109	plane->state = new_plane_state;
3110	}
3111
3112	for_each_oldnew_private_obj_in_state(state, obj, old_obj_state, new_obj_state, i) {
3113	WARN_ON(obj->state != old_obj_state);
3114
3115	old_obj_state->state = state;
3116	new_obj_state->state = NULL;
3117
3118	state->private_objs[i].state = old_obj_state;
3119	obj->state = new_obj_state;
3120	}
3121
3122	return 0;
3123	}
3124	EXPORT_SYMBOL(drm_atomic_helper_swap_state);
3125
3126	/**
3127	* drm_atomic_helper_update_plane - Helper for primary plane update using atomic
3128	* @plane: plane object to update
3129	* @crtc: owning CRTC of owning plane
3130	* @fb: framebuffer to flip onto plane
3131	* @crtc_x: x offset of primary plane on @crtc
3132	* @crtc_y: y offset of primary plane on @crtc
3133	* @crtc_w: width of primary plane rectangle on @crtc
3134	* @crtc_h: height of primary plane rectangle on @crtc
3135	* @src_x: x offset of @fb for panning
3136	* @src_y: y offset of @fb for panning
3137	* @src_w: width of source rectangle in @fb
3138	* @src_h: height of source rectangle in @fb
3139	* @ctx: lock acquire context
3140	*
3141	* Provides a default plane update handler using the atomic driver interface.
3142	*
3143	* RETURNS:
3144	* Zero on success, error code on failure
3145	*/
3146	int drm_atomic_helper_update_plane(struct drm_plane *plane,
3147	struct drm_crtc *crtc,
3148	struct drm_framebuffer *fb,
3149	int crtc_x, int crtc_y,
3150	unsigned int crtc_w, unsigned int crtc_h,
3151	uint32_t src_x, uint32_t src_y,
3152	uint32_t src_w, uint32_t src_h,
3153	struct drm_modeset_acquire_ctx *ctx)
3154	{
3155	struct drm_atomic_state *state;
3156	struct drm_plane_state *plane_state;
3157	int ret = 0;
3158
3159	state = drm_atomic_state_alloc(dev: plane->dev);
3160	if (!state)
3161	return -ENOMEM;
3162
3163	state->acquire_ctx = ctx;
3164	plane_state = drm_atomic_get_plane_state(state, plane);
3165	if (IS_ERR(ptr: plane_state)) {
3166	ret = PTR_ERR(ptr: plane_state);
3167	goto fail;
3168	}
3169
3170	ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
3171	if (ret != 0)
3172	goto fail;
3173	drm_atomic_set_fb_for_plane(plane_state, fb);
3174	plane_state->crtc_x = crtc_x;
3175	plane_state->crtc_y = crtc_y;
3176	plane_state->crtc_w = crtc_w;
3177	plane_state->crtc_h = crtc_h;
3178	plane_state->src_x = src_x;
3179	plane_state->src_y = src_y;
3180	plane_state->src_w = src_w;
3181	plane_state->src_h = src_h;
3182
3183	if (plane == crtc->cursor)
3184	state->legacy_cursor_update = true;
3185
3186	ret = drm_atomic_commit(state);
3187	fail:
3188	drm_atomic_state_put(state);
3189	return ret;
3190	}
3191	EXPORT_SYMBOL(drm_atomic_helper_update_plane);
3192
3193	/**
3194	* drm_atomic_helper_disable_plane - Helper for primary plane disable using atomic
3195	* @plane: plane to disable
3196	* @ctx: lock acquire context
3197	*
3198	* Provides a default plane disable handler using the atomic driver interface.
3199	*
3200	* RETURNS:
3201	* Zero on success, error code on failure
3202	*/
3203	int drm_atomic_helper_disable_plane(struct drm_plane *plane,
3204	struct drm_modeset_acquire_ctx *ctx)
3205	{
3206	struct drm_atomic_state *state;
3207	struct drm_plane_state *plane_state;
3208	int ret = 0;
3209
3210	state = drm_atomic_state_alloc(dev: plane->dev);
3211	if (!state)
3212	return -ENOMEM;
3213
3214	state->acquire_ctx = ctx;
3215	plane_state = drm_atomic_get_plane_state(state, plane);
3216	if (IS_ERR(ptr: plane_state)) {
3217	ret = PTR_ERR(ptr: plane_state);
3218	goto fail;
3219	}
3220
3221	if (plane_state->crtc && plane_state->crtc->cursor == plane)
3222	plane_state->state->legacy_cursor_update = true;
3223
3224	ret = __drm_atomic_helper_disable_plane(plane, plane_state);
3225	if (ret != 0)
3226	goto fail;
3227
3228	ret = drm_atomic_commit(state);
3229	fail:
3230	drm_atomic_state_put(state);
3231	return ret;
3232	}
3233	EXPORT_SYMBOL(drm_atomic_helper_disable_plane);
3234
3235	/**
3236	* drm_atomic_helper_set_config - set a new config from userspace
3237	* @set: mode set configuration
3238	* @ctx: lock acquisition context
3239	*
3240	* Provides a default CRTC set_config handler using the atomic driver interface.
3241	*
3242	* NOTE: For backwards compatibility with old userspace this automatically
3243	* resets the "link-status" property to GOOD, to force any link
3244	* re-training. The SETCRTC ioctl does not define whether an update does
3245	* need a full modeset or just a plane update, hence we're allowed to do
3246	* that. See also drm_connector_set_link_status_property().
3247	*
3248	* Returns:
3249	* Returns 0 on success, negative errno numbers on failure.
3250	*/
3251	int drm_atomic_helper_set_config(struct drm_mode_set *set,
3252	struct drm_modeset_acquire_ctx *ctx)
3253	{
3254	struct drm_atomic_state *state;
3255	struct drm_crtc *crtc = set->crtc;
3256	int ret = 0;
3257
3258	state = drm_atomic_state_alloc(dev: crtc->dev);
3259	if (!state)
3260	return -ENOMEM;
3261
3262	state->acquire_ctx = ctx;
3263	ret = __drm_atomic_helper_set_config(set, state);
3264	if (ret != 0)
3265	goto fail;
3266
3267	ret = handle_conflicting_encoders(state, disable_conflicting_encoders: true);
3268	if (ret)
3269	goto fail;
3270
3271	ret = drm_atomic_commit(state);
3272
3273	fail:
3274	drm_atomic_state_put(state);
3275	return ret;
3276	}
3277	EXPORT_SYMBOL(drm_atomic_helper_set_config);
3278
3279	/**
3280	* drm_atomic_helper_disable_all - disable all currently active outputs
3281	* @dev: DRM device
3282	* @ctx: lock acquisition context
3283	*
3284	* Loops through all connectors, finding those that aren't turned off and then
3285	* turns them off by setting their DPMS mode to OFF and deactivating the CRTC
3286	* that they are connected to.
3287	*
3288	* This is used for example in suspend/resume to disable all currently active
3289	* functions when suspending. If you just want to shut down everything at e.g.
3290	* driver unload, look at drm_atomic_helper_shutdown().
3291	*
3292	* Note that if callers haven't already acquired all modeset locks this might
3293	* return -EDEADLK, which must be handled by calling drm_modeset_backoff().
3294	*
3295	* Returns:
3296	* 0 on success or a negative error code on failure.
3297	*
3298	* See also:
3299	* drm_atomic_helper_suspend(), drm_atomic_helper_resume() and
3300	* drm_atomic_helper_shutdown().
3301	*/
3302	int drm_atomic_helper_disable_all(struct drm_device *dev,
3303	struct drm_modeset_acquire_ctx *ctx)
3304	{
3305	struct drm_atomic_state *state;
3306	struct drm_connector_state *conn_state;
3307	struct drm_connector *conn;
3308	struct drm_plane_state *plane_state;
3309	struct drm_plane *plane;
3310	struct drm_crtc_state *crtc_state;
3311	struct drm_crtc *crtc;
3312	int ret, i;
3313
3314	state = drm_atomic_state_alloc(dev);
3315	if (!state)
3316	return -ENOMEM;
3317
3318	state->acquire_ctx = ctx;
3319
3320	drm_for_each_crtc(crtc, dev) {
3321	crtc_state = drm_atomic_get_crtc_state(state, crtc);
3322	if (IS_ERR(ptr: crtc_state)) {
3323	ret = PTR_ERR(ptr: crtc_state);
3324	goto free;
3325	}
3326
3327	crtc_state->active = false;
3328
3329	ret = drm_atomic_set_mode_prop_for_crtc(state: crtc_state, NULL);
3330	if (ret < 0)
3331	goto free;
3332
3333	ret = drm_atomic_add_affected_planes(state, crtc);
3334	if (ret < 0)
3335	goto free;
3336
3337	ret = drm_atomic_add_affected_connectors(state, crtc);
3338	if (ret < 0)
3339	goto free;
3340	}
3341
3342	for_each_new_connector_in_state(state, conn, conn_state, i) {
3343	ret = drm_atomic_set_crtc_for_connector(conn_state, NULL);
3344	if (ret < 0)
3345	goto free;
3346	}
3347
3348	for_each_new_plane_in_state(state, plane, plane_state, i) {
3349	ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
3350	if (ret < 0)
3351	goto free;
3352
3353	drm_atomic_set_fb_for_plane(plane_state, NULL);
3354	}
3355
3356	ret = drm_atomic_commit(state);
3357	free:
3358	drm_atomic_state_put(state);
3359	return ret;
3360	}
3361	EXPORT_SYMBOL(drm_atomic_helper_disable_all);
3362
3363	/**
3364	* drm_atomic_helper_shutdown - shutdown all CRTC
3365	* @dev: DRM device
3366	*
3367	* This shuts down all CRTC, which is useful for driver unloading. Shutdown on
3368	* suspend should instead be handled with drm_atomic_helper_suspend(), since
3369	* that also takes a snapshot of the modeset state to be restored on resume.
3370	*
3371	* This is just a convenience wrapper around drm_atomic_helper_disable_all(),
3372	* and it is the atomic version of drm_helper_force_disable_all().
3373	*/
3374	void drm_atomic_helper_shutdown(struct drm_device *dev)
3375	{
3376	struct drm_modeset_acquire_ctx ctx;
3377	int ret;
3378
3379	if (dev == NULL)
3380	return;
3381
3382	DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret);
3383
3384	ret = drm_atomic_helper_disable_all(dev, &ctx);
3385	if (ret)
3386	drm_err(dev,
3387	"Disabling all crtc's during unload failed with %i\n",
3388	ret);
3389
3390	DRM_MODESET_LOCK_ALL_END(dev, ctx, ret);
3391	}
3392	EXPORT_SYMBOL(drm_atomic_helper_shutdown);
3393
3394	/**
3395	* drm_atomic_helper_duplicate_state - duplicate an atomic state object
3396	* @dev: DRM device
3397	* @ctx: lock acquisition context
3398	*
3399	* Makes a copy of the current atomic state by looping over all objects and
3400	* duplicating their respective states. This is used for example by suspend/
3401	* resume support code to save the state prior to suspend such that it can
3402	* be restored upon resume.
3403	*
3404	* Note that this treats atomic state as persistent between save and restore.
3405	* Drivers must make sure that this is possible and won't result in confusion
3406	* or erroneous behaviour.
3407	*
3408	* Note that if callers haven't already acquired all modeset locks this might
3409	* return -EDEADLK, which must be handled by calling drm_modeset_backoff().
3410	*
3411	* Returns:
3412	* A pointer to the copy of the atomic state object on success or an
3413	* ERR_PTR()-encoded error code on failure.
3414	*
3415	* See also:
3416	* drm_atomic_helper_suspend(), drm_atomic_helper_resume()
3417	*/
3418	struct drm_atomic_state *
3419	drm_atomic_helper_duplicate_state(struct drm_device *dev,
3420	struct drm_modeset_acquire_ctx *ctx)
3421	{
3422	struct drm_atomic_state *state;
3423	struct drm_connector *conn;
3424	struct drm_connector_list_iter conn_iter;
3425	struct drm_plane *plane;
3426	struct drm_crtc *crtc;
3427	int err = 0;
3428
3429	state = drm_atomic_state_alloc(dev);
3430	if (!state)
3431	return ERR_PTR(error: -ENOMEM);
3432
3433	state->acquire_ctx = ctx;
3434	state->duplicated = true;
3435
3436	drm_for_each_crtc(crtc, dev) {
3437	struct drm_crtc_state *crtc_state;
3438
3439	crtc_state = drm_atomic_get_crtc_state(state, crtc);
3440	if (IS_ERR(ptr: crtc_state)) {
3441	err = PTR_ERR(ptr: crtc_state);
3442	goto free;
3443	}
3444	}
3445
3446	drm_for_each_plane(plane, dev) {
3447	struct drm_plane_state *plane_state;
3448
3449	plane_state = drm_atomic_get_plane_state(state, plane);
3450	if (IS_ERR(ptr: plane_state)) {
3451	err = PTR_ERR(ptr: plane_state);
3452	goto free;
3453	}
3454	}
3455
3456	drm_connector_list_iter_begin(dev, iter: &conn_iter);
3457	drm_for_each_connector_iter(conn, &conn_iter) {
3458	struct drm_connector_state *conn_state;
3459
3460	conn_state = drm_atomic_get_connector_state(state, connector: conn);
3461	if (IS_ERR(ptr: conn_state)) {
3462	err = PTR_ERR(ptr: conn_state);
3463	drm_connector_list_iter_end(iter: &conn_iter);
3464	goto free;
3465	}
3466	}
3467	drm_connector_list_iter_end(iter: &conn_iter);
3468
3469	/* clear the acquire context so that it isn't accidentally reused */
3470	state->acquire_ctx = NULL;
3471
3472	free:
3473	if (err < 0) {
3474	drm_atomic_state_put(state);
3475	state = ERR_PTR(error: err);
3476	}
3477
3478	return state;
3479	}
3480	EXPORT_SYMBOL(drm_atomic_helper_duplicate_state);
3481
3482	/**
3483	* drm_atomic_helper_suspend - subsystem-level suspend helper
3484	* @dev: DRM device
3485	*
3486	* Duplicates the current atomic state, disables all active outputs and then
3487	* returns a pointer to the original atomic state to the caller. Drivers can
3488	* pass this pointer to the drm_atomic_helper_resume() helper upon resume to
3489	* restore the output configuration that was active at the time the system
3490	* entered suspend.
3491	*
3492	* Note that it is potentially unsafe to use this. The atomic state object
3493	* returned by this function is assumed to be persistent. Drivers must ensure
3494	* that this holds true. Before calling this function, drivers must make sure
3495	* to suspend fbdev emulation so that nothing can be using the device.
3496	*
3497	* Returns:
3498	* A pointer to a copy of the state before suspend on success or an ERR_PTR()-
3499	* encoded error code on failure. Drivers should store the returned atomic
3500	* state object and pass it to the drm_atomic_helper_resume() helper upon
3501	* resume.
3502	*
3503	* See also:
3504	* drm_atomic_helper_duplicate_state(), drm_atomic_helper_disable_all(),
3505	* drm_atomic_helper_resume(), drm_atomic_helper_commit_duplicated_state()
3506	*/
3507	struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev)
3508	{
3509	struct drm_modeset_acquire_ctx ctx;
3510	struct drm_atomic_state *state;
3511	int err;
3512
3513	/* This can never be returned, but it makes the compiler happy */
3514	state = ERR_PTR(error: -EINVAL);
3515
3516	DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, err);
3517
3518	state = drm_atomic_helper_duplicate_state(dev, &ctx);
3519	if (IS_ERR(ptr: state))
3520	goto unlock;
3521
3522	err = drm_atomic_helper_disable_all(dev, &ctx);
3523	if (err < 0) {
3524	drm_atomic_state_put(state);
3525	state = ERR_PTR(error: err);
3526	goto unlock;
3527	}
3528
3529	unlock:
3530	DRM_MODESET_LOCK_ALL_END(dev, ctx, err);
3531	if (err)
3532	return ERR_PTR(error: err);
3533
3534	return state;
3535	}
3536	EXPORT_SYMBOL(drm_atomic_helper_suspend);
3537
3538	/**
3539	* drm_atomic_helper_commit_duplicated_state - commit duplicated state
3540	* @state: duplicated atomic state to commit
3541	* @ctx: pointer to acquire_ctx to use for commit.
3542	*
3543	* The state returned by drm_atomic_helper_duplicate_state() and
3544	* drm_atomic_helper_suspend() is partially invalid, and needs to
3545	* be fixed up before commit.
3546	*
3547	* Returns:
3548	* 0 on success or a negative error code on failure.
3549	*
3550	* See also:
3551	* drm_atomic_helper_suspend()
3552	*/
3553	int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
3554	struct drm_modeset_acquire_ctx *ctx)
3555	{
3556	int i, ret;
3557	struct drm_plane *plane;
3558	struct drm_plane_state *new_plane_state;
3559	struct drm_connector *connector;
3560	struct drm_connector_state *new_conn_state;
3561	struct drm_crtc *crtc;
3562	struct drm_crtc_state *new_crtc_state;
3563
3564	state->acquire_ctx = ctx;
3565
3566	for_each_new_plane_in_state(state, plane, new_plane_state, i)
3567	state->planes[i].old_state = plane->state;
3568
3569	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
3570	state->crtcs[i].old_state = crtc->state;
3571
3572	for_each_new_connector_in_state(state, connector, new_conn_state, i)
3573	state->connectors[i].old_state = connector->state;
3574
3575	ret = drm_atomic_commit(state);
3576
3577	state->acquire_ctx = NULL;
3578
3579	return ret;
3580	}
3581	EXPORT_SYMBOL(drm_atomic_helper_commit_duplicated_state);
3582
3583	/**
3584	* drm_atomic_helper_resume - subsystem-level resume helper
3585	* @dev: DRM device
3586	* @state: atomic state to resume to
3587	*
3588	* Calls drm_mode_config_reset() to synchronize hardware and software states,
3589	* grabs all modeset locks and commits the atomic state object. This can be
3590	* used in conjunction with the drm_atomic_helper_suspend() helper to
3591	* implement suspend/resume for drivers that support atomic mode-setting.
3592	*
3593	* Returns:
3594	* 0 on success or a negative error code on failure.
3595	*
3596	* See also:
3597	* drm_atomic_helper_suspend()
3598	*/
3599	int drm_atomic_helper_resume(struct drm_device *dev,
3600	struct drm_atomic_state *state)
3601	{
3602	struct drm_modeset_acquire_ctx ctx;
3603	int err;
3604
3605	drm_mode_config_reset(dev);
3606
3607	DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, err);
3608
3609	err = drm_atomic_helper_commit_duplicated_state(state, &ctx);
3610
3611	DRM_MODESET_LOCK_ALL_END(dev, ctx, err);
3612	drm_atomic_state_put(state);
3613
3614	return err;
3615	}
3616	EXPORT_SYMBOL(drm_atomic_helper_resume);
3617
3618	static int page_flip_common(struct drm_atomic_state *state,
3619	struct drm_crtc *crtc,
3620	struct drm_framebuffer *fb,
3621	struct drm_pending_vblank_event *event,
3622	uint32_t flags)
3623	{
3624	struct drm_plane *plane = crtc->primary;
3625	struct drm_plane_state *plane_state;
3626	struct drm_crtc_state *crtc_state;
3627	int ret = 0;
3628
3629	crtc_state = drm_atomic_get_crtc_state(state, crtc);
3630	if (IS_ERR(ptr: crtc_state))
3631	return PTR_ERR(ptr: crtc_state);
3632
3633	crtc_state->event = event;
3634	crtc_state->async_flip = flags & DRM_MODE_PAGE_FLIP_ASYNC;
3635
3636	plane_state = drm_atomic_get_plane_state(state, plane);
3637	if (IS_ERR(ptr: plane_state))
3638	return PTR_ERR(ptr: plane_state);
3639
3640	ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
3641	if (ret != 0)
3642	return ret;
3643	drm_atomic_set_fb_for_plane(plane_state, fb);
3644
3645	/* Make sure we don't accidentally do a full modeset. */
3646	state->allow_modeset = false;
3647	if (!crtc_state->active) {
3648	drm_dbg_atomic(crtc->dev,
3649	"[CRTC:%d:%s] disabled, rejecting legacy flip\n",
3650	crtc->base.id, crtc->name);
3651	return -EINVAL;
3652	}
3653
3654	return ret;
3655	}
3656
3657	/**
3658	* drm_atomic_helper_page_flip - execute a legacy page flip
3659	* @crtc: DRM CRTC
3660	* @fb: DRM framebuffer
3661	* @event: optional DRM event to signal upon completion
3662	* @flags: flip flags for non-vblank sync'ed updates
3663	* @ctx: lock acquisition context
3664	*
3665	* Provides a default &drm_crtc_funcs.page_flip implementation
3666	* using the atomic driver interface.
3667	*
3668	* Returns:
3669	* Returns 0 on success, negative errno numbers on failure.
3670	*
3671	* See also:
3672	* drm_atomic_helper_page_flip_target()
3673	*/
3674	int drm_atomic_helper_page_flip(struct drm_crtc *crtc,
3675	struct drm_framebuffer *fb,
3676	struct drm_pending_vblank_event *event,
3677	uint32_t flags,
3678	struct drm_modeset_acquire_ctx *ctx)
3679	{
3680	struct drm_plane *plane = crtc->primary;
3681	struct drm_atomic_state *state;
3682	int ret = 0;
3683
3684	state = drm_atomic_state_alloc(dev: plane->dev);
3685	if (!state)
3686	return -ENOMEM;
3687
3688	state->acquire_ctx = ctx;
3689
3690	ret = page_flip_common(state, crtc, fb, event, flags);
3691	if (ret != 0)
3692	goto fail;
3693
3694	ret = drm_atomic_nonblocking_commit(state);
3695	fail:
3696	drm_atomic_state_put(state);
3697	return ret;
3698	}
3699	EXPORT_SYMBOL(drm_atomic_helper_page_flip);
3700
3701	/**
3702	* drm_atomic_helper_page_flip_target - do page flip on target vblank period.
3703	* @crtc: DRM CRTC
3704	* @fb: DRM framebuffer
3705	* @event: optional DRM event to signal upon completion
3706	* @flags: flip flags for non-vblank sync'ed updates
3707	* @target: specifying the target vblank period when the flip to take effect
3708	* @ctx: lock acquisition context
3709	*
3710	* Provides a default &drm_crtc_funcs.page_flip_target implementation.
3711	* Similar to drm_atomic_helper_page_flip() with extra parameter to specify
3712	* target vblank period to flip.
3713	*
3714	* Returns:
3715	* Returns 0 on success, negative errno numbers on failure.
3716	*/
3717	int drm_atomic_helper_page_flip_target(struct drm_crtc *crtc,
3718	struct drm_framebuffer *fb,
3719	struct drm_pending_vblank_event *event,
3720	uint32_t flags,
3721	uint32_t target,
3722	struct drm_modeset_acquire_ctx *ctx)
3723	{
3724	struct drm_plane *plane = crtc->primary;
3725	struct drm_atomic_state *state;
3726	struct drm_crtc_state *crtc_state;
3727	int ret = 0;
3728
3729	state = drm_atomic_state_alloc(dev: plane->dev);
3730	if (!state)
3731	return -ENOMEM;
3732
3733	state->acquire_ctx = ctx;
3734
3735	ret = page_flip_common(state, crtc, fb, event, flags);
3736	if (ret != 0)
3737	goto fail;
3738
3739	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
3740	if (WARN_ON(!crtc_state)) {
3741	ret = -EINVAL;
3742	goto fail;
3743	}
3744	crtc_state->target_vblank = target;
3745
3746	ret = drm_atomic_nonblocking_commit(state);
3747	fail:
3748	drm_atomic_state_put(state);
3749	return ret;
3750	}
3751	EXPORT_SYMBOL(drm_atomic_helper_page_flip_target);
3752
3753	/**
3754	* drm_atomic_helper_bridge_propagate_bus_fmt() - Propagate output format to
3755	* the input end of a bridge
3756	* @bridge: bridge control structure
3757	* @bridge_state: new bridge state
3758	* @crtc_state: new CRTC state
3759	* @conn_state: new connector state
3760	* @output_fmt: tested output bus format
3761	* @num_input_fmts: will contain the size of the returned array
3762	*
3763	* This helper is a pluggable implementation of the
3764	* &drm_bridge_funcs.atomic_get_input_bus_fmts operation for bridges that don't
3765	* modify the bus configuration between their input and their output. It
3766	* returns an array of input formats with a single element set to @output_fmt.
3767	*
3768	* RETURNS:
3769	* a valid format array of size @num_input_fmts, or NULL if the allocation
3770	* failed
3771	*/
3772	u32 *
3773	drm_atomic_helper_bridge_propagate_bus_fmt(struct drm_bridge *bridge,
3774	struct drm_bridge_state *bridge_state,
3775	struct drm_crtc_state *crtc_state,
3776	struct drm_connector_state *conn_state,
3777	u32 output_fmt,
3778	unsigned int *num_input_fmts)
3779	{
3780	u32 *input_fmts;
3781
3782	input_fmts = kzalloc(size: sizeof(*input_fmts), GFP_KERNEL);
3783	if (!input_fmts) {
3784	*num_input_fmts = 0;
3785	return NULL;
3786	}
3787
3788	*num_input_fmts = 1;
3789	input_fmts[0] = output_fmt;
3790	return input_fmts;
3791	}
3792	EXPORT_SYMBOL(drm_atomic_helper_bridge_propagate_bus_fmt);
3793