1	/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2	*/
3	/*
4	*
5	* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6	* All Rights Reserved.
7	*
8	* Permission is hereby granted, free of charge, to any person obtaining a
9	* copy of this software and associated documentation files (the
10	* "Software"), to deal in the Software without restriction, including
11	* without limitation the rights to use, copy, modify, merge, publish,
12	* distribute, sub license, and/or sell copies of the Software, and to
13	* permit persons to whom the Software is furnished to do so, subject to
14	* the following conditions:
15	*
16	* The above copyright notice and this permission notice (including the
17	* next paragraph) shall be included in all copies or substantial portions
18	* of the Software.
19	*
20	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23	* IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27	*
28	*/
29
30	#include <linux/aperture.h>
31	#include <linux/acpi.h>
32	#include <linux/device.h>
33	#include <linux/module.h>
34	#include <linux/oom.h>
35	#include <linux/pci.h>
36	#include <linux/pm.h>
37	#include <linux/pm_runtime.h>
38	#include <linux/slab.h>
39	#include <linux/string_helpers.h>
40	#include <linux/vga_switcheroo.h>
41	#include <linux/vt.h>
42
43	#include <drm/drm_atomic_helper.h>
44	#include <drm/drm_client.h>
45	#include <drm/drm_client_event.h>
46	#include <drm/drm_ioctl.h>
47	#include <drm/drm_managed.h>
48	#include <drm/drm_probe_helper.h>
49
50	#include "display/i9xx_display_sr.h"
51	#include "display/intel_bw.h"
52	#include "display/intel_cdclk.h"
53	#include "display/intel_crtc.h"
54	#include "display/intel_display_driver.h"
55	#include "display/intel_dmc.h"
56	#include "display/intel_dp.h"
57	#include "display/intel_dpt.h"
58	#include "display/intel_encoder.h"
59	#include "display/intel_fbdev.h"
60	#include "display/intel_hotplug.h"
61	#include "display/intel_overlay.h"
62	#include "display/intel_pch_refclk.h"
63	#include "display/intel_pps.h"
64	#include "display/intel_sprite_uapi.h"
65	#include "display/skl_watermark.h"
66
67	#include "gem/i915_gem_context.h"
68	#include "gem/i915_gem_create.h"
69	#include "gem/i915_gem_dmabuf.h"
70	#include "gem/i915_gem_ioctls.h"
71	#include "gem/i915_gem_mman.h"
72	#include "gem/i915_gem_pm.h"
73	#include "gt/intel_gt.h"
74	#include "gt/intel_gt_pm.h"
75	#include "gt/intel_gt_print.h"
76	#include "gt/intel_rc6.h"
77
78	#include "pxp/intel_pxp.h"
79	#include "pxp/intel_pxp_debugfs.h"
80	#include "pxp/intel_pxp_pm.h"
81
82	#include "soc/intel_dram.h"
83	#include "soc/intel_gmch.h"
84
85	#include "i915_debugfs.h"
86	#include "i915_driver.h"
87	#include "i915_drm_client.h"
88	#include "i915_drv.h"
89	#include "i915_file_private.h"
90	#include "i915_getparam.h"
91	#include "i915_hwmon.h"
92	#include "i915_ioc32.h"
93	#include "i915_ioctl.h"
94	#include "i915_irq.h"
95	#include "i915_memcpy.h"
96	#include "i915_perf.h"
97	#include "i915_query.h"
98	#include "i915_reg.h"
99	#include "i915_switcheroo.h"
100	#include "i915_sysfs.h"
101	#include "i915_utils.h"
102	#include "i915_vgpu.h"
103	#include "intel_clock_gating.h"
104	#include "intel_cpu_info.h"
105	#include "intel_gvt.h"
106	#include "intel_memory_region.h"
107	#include "intel_pci_config.h"
108	#include "intel_pcode.h"
109	#include "intel_region_ttm.h"
110	#include "intel_sbi.h"
111	#include "vlv_sideband.h"
112	#include "vlv_suspend.h"
113
114	static const struct drm_driver i915_drm_driver;
115
116	static int i915_workqueues_init(struct drm_i915_private *dev_priv)
117	{
118	/*
119	* The i915 workqueue is primarily used for batched retirement of
120	* requests (and thus managing bo) once the task has been completed
121	* by the GPU. i915_retire_requests() is called directly when we
122	* need high-priority retirement, such as waiting for an explicit
123	* bo.
124	*
125	* It is also used for periodic low-priority events, such as
126	* idle-timers and recording error state.
127	*
128	* All tasks on the workqueue are expected to acquire the dev mutex
129	* so there is no point in running more than one instance of the
130	* workqueue at any time. Use an ordered one.
131	*/
132	dev_priv->wq = alloc_ordered_workqueue("i915", 0);
133	if (dev_priv->wq == NULL)
134	goto out_err;
135
136	dev_priv->display.hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
137	if (dev_priv->display.hotplug.dp_wq == NULL)
138	goto out_free_wq;
139
140	/*
141	* The unordered i915 workqueue should be used for all work
142	* scheduling that do not require running in order, which used
143	* to be scheduled on the system_wq before moving to a driver
144	* instance due deprecation of flush_scheduled_work().
145	*/
146	dev_priv->unordered_wq = alloc_workqueue(fmt: "i915-unordered", flags: 0, max_active: 0);
147	if (dev_priv->unordered_wq == NULL)
148	goto out_free_dp_wq;
149
150	return 0;
151
152	out_free_dp_wq:
153	destroy_workqueue(wq: dev_priv->display.hotplug.dp_wq);
154	out_free_wq:
155	destroy_workqueue(wq: dev_priv->wq);
156	out_err:
157	drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n");
158
159	return -ENOMEM;
160	}
161
162	static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
163	{
164	destroy_workqueue(wq: dev_priv->unordered_wq);
165	destroy_workqueue(wq: dev_priv->display.hotplug.dp_wq);
166	destroy_workqueue(wq: dev_priv->wq);
167	}
168
169	/*
170	* We don't keep the workarounds for pre-production hardware, so we expect our
171	* driver to fail on these machines in one way or another. A little warning on
172	* dmesg may help both the user and the bug triagers.
173	*
174	* Our policy for removing pre-production workarounds is to keep the
175	* current gen workarounds as a guide to the bring-up of the next gen
176	* (workarounds have a habit of persisting!). Anything older than that
177	* should be removed along with the complications they introduce.
178	*/
179	static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
180	{
181	bool pre = false;
182
183	pre |= IS_HASWELL_EARLY_SDV(dev_priv);
184	pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6;
185	pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA;
186	pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
187	pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3;
188	pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7;
189	pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
190	pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
191	pre |= IS_DG2_G10(dev_priv) && INTEL_REVID(dev_priv) < 0x8;
192	pre |= IS_DG2_G11(dev_priv) && INTEL_REVID(dev_priv) < 0x5;
193	pre |= IS_DG2_G12(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
194
195	if (pre) {
196	drm_err(&dev_priv->drm, "This is a pre-production stepping. "
197	"It may not be fully functional.\n");
198	add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
199	}
200	}
201
202	static void sanitize_gpu(struct drm_i915_private *i915)
203	{
204	if (!intel_gt_gpu_reset_clobbers_display(gt: to_gt(i915))) {
205	struct intel_gt *gt;
206	unsigned int i;
207
208	for_each_gt(gt, i915, i)
209	intel_gt_reset_all_engines(gt);
210	}
211	}
212
213	/**
214	* i915_driver_early_probe - setup state not requiring device access
215	* @dev_priv: device private
216	*
217	* Initialize everything that is a "SW-only" state, that is state not
218	* requiring accessing the device or exposing the driver via kernel internal
219	* or userspace interfaces. Example steps belonging here: lock initialization,
220	* system memory allocation, setting up device specific attributes and
221	* function hooks not requiring accessing the device.
222	*/
223	static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
224	{
225	struct intel_display *display = &dev_priv->display;
226	int ret = 0;
227
228	if (i915_inject_probe_failure(dev_priv))
229	return -ENODEV;
230
231	intel_device_info_runtime_init_early(dev_priv);
232
233	intel_step_init(i915: dev_priv);
234
235	intel_uncore_mmio_debug_init_early(i915: dev_priv);
236
237	spin_lock_init(&dev_priv->gpu_error.lock);
238
239	intel_sbi_init(i915: dev_priv);
240	vlv_iosf_sb_init(i915: dev_priv);
241	mutex_init(&dev_priv->sb_lock);
242
243	i915_memcpy_init_early(i915: dev_priv);
244	intel_runtime_pm_init_early(rpm: &dev_priv->runtime_pm);
245
246	ret = i915_workqueues_init(dev_priv);
247	if (ret < 0)
248	return ret;
249
250	ret = vlv_suspend_init(i915: dev_priv);
251	if (ret < 0)
252	goto err_workqueues;
253
254	ret = intel_region_ttm_device_init(dev_priv);
255	if (ret)
256	goto err_ttm;
257
258	ret = intel_root_gt_init_early(i915: dev_priv);
259	if (ret < 0)
260	goto err_rootgt;
261
262	i915_gem_init_early(i915: dev_priv);
263
264	intel_irq_init(dev_priv);
265	intel_display_driver_early_probe(display);
266	intel_clock_gating_hooks_init(i915: dev_priv);
267
268	intel_detect_preproduction_hw(dev_priv);
269
270	return 0;
271
272	err_rootgt:
273	intel_region_ttm_device_fini(dev_priv);
274	err_ttm:
275	vlv_suspend_cleanup(i915: dev_priv);
276	err_workqueues:
277	i915_workqueues_cleanup(dev_priv);
278	return ret;
279	}
280
281	/**
282	* i915_driver_late_release - cleanup the setup done in
283	* i915_driver_early_probe()
284	* @dev_priv: device private
285	*/
286	static void i915_driver_late_release(struct drm_i915_private *dev_priv)
287	{
288	struct intel_display *display = &dev_priv->display;
289
290	intel_irq_fini(dev_priv);
291	intel_power_domains_cleanup(display);
292	i915_gem_cleanup_early(i915: dev_priv);
293	intel_gt_driver_late_release_all(i915: dev_priv);
294	intel_region_ttm_device_fini(dev_priv);
295	vlv_suspend_cleanup(i915: dev_priv);
296	i915_workqueues_cleanup(dev_priv);
297
298	mutex_destroy(lock: &dev_priv->sb_lock);
299	vlv_iosf_sb_fini(i915: dev_priv);
300	intel_sbi_fini(i915: dev_priv);
301
302	i915_params_free(params: &dev_priv->params);
303	}
304
305	/**
306	* i915_driver_mmio_probe - setup device MMIO
307	* @dev_priv: device private
308	*
309	* Setup minimal device state necessary for MMIO accesses later in the
310	* initialization sequence. The setup here should avoid any other device-wide
311	* side effects or exposing the driver via kernel internal or user space
312	* interfaces.
313	*/
314	static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
315	{
316	struct intel_display *display = &dev_priv->display;
317	struct intel_gt *gt;
318	int ret, i;
319
320	if (i915_inject_probe_failure(dev_priv))
321	return -ENODEV;
322
323	ret = intel_gmch_bridge_setup(i915: dev_priv);
324	if (ret < 0)
325	return ret;
326
327	for_each_gt(gt, dev_priv, i) {
328	ret = intel_uncore_init_mmio(uncore: gt->uncore);
329	if (ret)
330	return ret;
331
332	ret = drmm_add_action_or_reset(&dev_priv->drm,
333	intel_uncore_fini_mmio,
334	gt->uncore);
335	if (ret)
336	return ret;
337	}
338
339	/* Try to make sure MCHBAR is enabled before poking at it */
340	intel_gmch_bar_setup(i915: dev_priv);
341	intel_device_info_runtime_init(dev_priv);
342	intel_display_device_info_runtime_init(display);
343
344	for_each_gt(gt, dev_priv, i) {
345	ret = intel_gt_init_mmio(gt);
346	if (ret)
347	goto err_uncore;
348	}
349
350	/* As early as possible, scrub existing GPU state before clobbering */
351	sanitize_gpu(i915: dev_priv);
352
353	return 0;
354
355	err_uncore:
356	intel_gmch_bar_teardown(i915: dev_priv);
357
358	return ret;
359	}
360
361	/**
362	* i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe()
363	* @dev_priv: device private
364	*/
365	static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
366	{
367	intel_gmch_bar_teardown(i915: dev_priv);
368	}
369
370	/**
371	* i915_set_dma_info - set all relevant PCI dma info as configured for the
372	* platform
373	* @i915: valid i915 instance
374	*
375	* Set the dma max segment size, device and coherent masks. The dma mask set
376	* needs to occur before i915_ggtt_probe_hw.
377	*
378	* A couple of platforms have special needs. Address them as well.
379	*
380	*/
381	static int i915_set_dma_info(struct drm_i915_private *i915)
382	{
383	unsigned int mask_size = INTEL_INFO(i915)->dma_mask_size;
384	int ret;
385
386	GEM_BUG_ON(!mask_size);
387
388	/*
389	* We don't have a max segment size, so set it to the max so sg's
390	* debugging layer doesn't complain
391	*/
392	dma_set_max_seg_size(dev: i915->drm.dev, UINT_MAX);
393
394	ret = dma_set_mask(dev: i915->drm.dev, DMA_BIT_MASK(mask_size));
395	if (ret)
396	goto mask_err;
397
398	/* overlay on gen2 is broken and can't address above 1G */
399	if (GRAPHICS_VER(i915) == 2)
400	mask_size = 30;
401
402	/*
403	* 965GM sometimes incorrectly writes to hardware status page (HWS)
404	* using 32bit addressing, overwriting memory if HWS is located
405	* above 4GB.
406	*
407	* The documentation also mentions an issue with undefined
408	* behaviour if any general state is accessed within a page above 4GB,
409	* which also needs to be handled carefully.
410	*/
411	if (IS_I965G(i915) || IS_I965GM(i915))
412	mask_size = 32;
413
414	ret = dma_set_coherent_mask(dev: i915->drm.dev, DMA_BIT_MASK(mask_size));
415	if (ret)
416	goto mask_err;
417
418	return 0;
419
420	mask_err:
421	drm_err(&i915->drm, "Can't set DMA mask/consistent mask (%d)\n", ret);
422	return ret;
423	}
424
425	/* Wa_14022698537:dg2 */
426	static void i915_enable_g8(struct drm_i915_private *i915)
427	{
428	if (IS_DG2(i915)) {
429	if (IS_DG2_D(i915) && !intel_match_g8_cpu())
430	return;
431
432	snb_pcode_write_p(uncore: &i915->uncore, PCODE_POWER_SETUP,
433	POWER_SETUP_SUBCOMMAND_G8_ENABLE, p2: 0, val: 0);
434	}
435	}
436
437	static int i915_pcode_init(struct drm_i915_private *i915)
438	{
439	struct intel_gt *gt;
440	int id, ret;
441
442	for_each_gt(gt, i915, id) {
443	ret = intel_pcode_init(uncore: gt->uncore);
444	if (ret) {
445	gt_err(gt, "intel_pcode_init failed %d\n", ret);
446	return ret;
447	}
448	}
449
450	i915_enable_g8(i915);
451	return 0;
452	}
453
454	/**
455	* i915_driver_hw_probe - setup state requiring device access
456	* @dev_priv: device private
457	*
458	* Setup state that requires accessing the device, but doesn't require
459	* exposing the driver via kernel internal or userspace interfaces.
460	*/
461	static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
462	{
463	struct intel_display *display = &dev_priv->display;
464	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
465	int ret;
466
467	if (i915_inject_probe_failure(dev_priv))
468	return -ENODEV;
469
470	if (HAS_PPGTT(dev_priv)) {
471	if (intel_vgpu_active(i915: dev_priv) &&
472	!intel_vgpu_has_full_ppgtt(i915: dev_priv)) {
473	drm_err(&dev_priv->drm,
474	"incompatible vGPU found, support for isolated ppGTT required\n");
475	return -ENXIO;
476	}
477	}
478
479	if (HAS_EXECLISTS(dev_priv)) {
480	/*
481	* Older GVT emulation depends upon intercepting CSB mmio,
482	* which we no longer use, preferring to use the HWSP cache
483	* instead.
484	*/
485	if (intel_vgpu_active(i915: dev_priv) &&
486	!intel_vgpu_has_hwsp_emulation(i915: dev_priv)) {
487	drm_err(&dev_priv->drm,
488	"old vGPU host found, support for HWSP emulation required\n");
489	return -ENXIO;
490	}
491	}
492
493	/* needs to be done before ggtt probe */
494	intel_dram_edram_detect(i915: dev_priv);
495
496	ret = i915_set_dma_info(i915: dev_priv);
497	if (ret)
498	return ret;
499
500	ret = i915_perf_init(i915: dev_priv);
501	if (ret)
502	return ret;
503
504	ret = i915_ggtt_probe_hw(i915: dev_priv);
505	if (ret)
506	goto err_perf;
507
508	ret = aperture_remove_conflicting_pci_devices(pdev, name: dev_priv->drm.driver->name);
509	if (ret)
510	goto err_ggtt;
511
512	ret = i915_ggtt_init_hw(i915: dev_priv);
513	if (ret)
514	goto err_ggtt;
515
516	/*
517	* Make sure we probe lmem before we probe stolen-lmem. The BAR size
518	* might be different due to bar resizing.
519	*/
520	ret = intel_gt_tiles_init(i915: dev_priv);
521	if (ret)
522	goto err_ggtt;
523
524	ret = intel_memory_regions_hw_probe(i915: dev_priv);
525	if (ret)
526	goto err_ggtt;
527
528	ret = i915_ggtt_enable_hw(i915: dev_priv);
529	if (ret) {
530	drm_err(&dev_priv->drm, "failed to enable GGTT\n");
531	goto err_mem_regions;
532	}
533
534	pci_set_master(dev: pdev);
535
536	/* On the 945G/GM, the chipset reports the MSI capability on the
537	* integrated graphics even though the support isn't actually there
538	* according to the published specs. It doesn't appear to function
539	* correctly in testing on 945G.
540	* This may be a side effect of MSI having been made available for PEG
541	* and the registers being closely associated.
542	*
543	* According to chipset errata, on the 965GM, MSI interrupts may
544	* be lost or delayed, and was defeatured. MSI interrupts seem to
545	* get lost on g4x as well, and interrupt delivery seems to stay
546	* properly dead afterwards. So we'll just disable them for all
547	* pre-gen5 chipsets.
548	*
549	* dp aux and gmbus irq on gen4 seems to be able to generate legacy
550	* interrupts even when in MSI mode. This results in spurious
551	* interrupt warnings if the legacy irq no. is shared with another
552	* device. The kernel then disables that interrupt source and so
553	* prevents the other device from working properly.
554	*/
555	if (GRAPHICS_VER(dev_priv) >= 5) {
556	if (pci_enable_msi(dev: pdev) < 0)
557	drm_dbg(&dev_priv->drm, "can't enable MSI");
558	}
559
560	ret = intel_gvt_init(dev_priv);
561	if (ret)
562	goto err_msi;
563
564	intel_opregion_setup(display);
565
566	ret = i915_pcode_init(i915: dev_priv);
567	if (ret)
568	goto err_opregion;
569
570	/*
571	* Fill the dram structure to get the system dram info. This will be
572	* used for memory latency calculation.
573	*/
574	intel_dram_detect(i915: dev_priv);
575
576	intel_bw_init_hw(display);
577
578	return 0;
579
580	err_opregion:
581	intel_opregion_cleanup(display);
582	err_msi:
583	if (pdev->msi_enabled)
584	pci_disable_msi(dev: pdev);
585	err_mem_regions:
586	intel_memory_regions_driver_release(i915: dev_priv);
587	err_ggtt:
588	i915_ggtt_driver_release(i915: dev_priv);
589	i915_gem_drain_freed_objects(i915: dev_priv);
590	i915_ggtt_driver_late_release(i915: dev_priv);
591	err_perf:
592	i915_perf_fini(i915: dev_priv);
593	return ret;
594	}
595
596	/**
597	* i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe()
598	* @dev_priv: device private
599	*/
600	static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
601	{
602	struct intel_display *display = &dev_priv->display;
603	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
604
605	i915_perf_fini(i915: dev_priv);
606
607	intel_opregion_cleanup(display);
608
609	if (pdev->msi_enabled)
610	pci_disable_msi(dev: pdev);
611	}
612
613	/**
614	* i915_driver_register - register the driver with the rest of the system
615	* @dev_priv: device private
616	*
617	* Perform any steps necessary to make the driver available via kernel
618	* internal or userspace interfaces.
619	*/
620	static int i915_driver_register(struct drm_i915_private *dev_priv)
621	{
622	struct intel_display *display = &dev_priv->display;
623	struct intel_gt *gt;
624	unsigned int i;
625	int ret;
626
627	i915_gem_driver_register(i915: dev_priv);
628	i915_pmu_register(i915: dev_priv);
629
630	intel_vgpu_register(i915: dev_priv);
631
632	/* Reveal our presence to userspace */
633	ret = drm_dev_register(dev: &dev_priv->drm, flags: 0);
634	if (ret) {
635	i915_probe_error(dev_priv,
636	"Failed to register driver for userspace access!\n");
637	drm_dev_unregister(dev: &dev_priv->drm);
638	i915_pmu_unregister(i915: dev_priv);
639	i915_gem_driver_unregister(i915: dev_priv);
640	return ret;
641	}
642
643	i915_debugfs_register(dev_priv);
644	i915_setup_sysfs(i915: dev_priv);
645
646	/* Depends on sysfs having been initialized */
647	i915_perf_register(i915: dev_priv);
648
649	for_each_gt(gt, dev_priv, i)
650	intel_gt_driver_register(gt);
651
652	intel_pxp_debugfs_register(pxp: dev_priv->pxp);
653
654	i915_hwmon_register(i915: dev_priv);
655
656	intel_display_driver_register(display);
657
658	intel_power_domains_enable(display);
659	intel_runtime_pm_enable(rpm: &dev_priv->runtime_pm);
660
661	if (i915_switcheroo_register(i915: dev_priv))
662	drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n");
663
664	return 0;
665	}
666
667	/**
668	* i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
669	* @dev_priv: device private
670	*/
671	static void i915_driver_unregister(struct drm_i915_private *dev_priv)
672	{
673	struct intel_display *display = &dev_priv->display;
674	struct intel_gt *gt;
675	unsigned int i;
676
677	i915_switcheroo_unregister(i915: dev_priv);
678
679	intel_runtime_pm_disable(rpm: &dev_priv->runtime_pm);
680	intel_power_domains_disable(display);
681
682	intel_display_driver_unregister(display);
683
684	intel_pxp_fini(i915: dev_priv);
685
686	for_each_gt(gt, dev_priv, i)
687	intel_gt_driver_unregister(gt);
688
689	i915_hwmon_unregister(i915: dev_priv);
690
691	i915_perf_unregister(i915: dev_priv);
692	i915_pmu_unregister(i915: dev_priv);
693
694	i915_teardown_sysfs(i915: dev_priv);
695	drm_dev_unplug(dev: &dev_priv->drm);
696
697	i915_gem_driver_unregister(i915: dev_priv);
698	}
699
700	void
701	i915_print_iommu_status(struct drm_i915_private *i915, struct drm_printer *p)
702	{
703	drm_printf(p, f: "iommu: %s\n",
704	str_enabled_disabled(v: i915_vtd_active(i915)));
705	}
706
707	static void i915_welcome_messages(struct drm_i915_private *dev_priv)
708	{
709	if (drm_debug_enabled(DRM_UT_DRIVER)) {
710	struct drm_printer p = drm_dbg_printer(drm: &dev_priv->drm, category: DRM_UT_DRIVER,
711	prefix: "device info:");
712	struct intel_gt *gt;
713	unsigned int i;
714
715	drm_printf(p: &p, f: "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
716	INTEL_DEVID(dev_priv),
717	INTEL_REVID(dev_priv),
718	intel_platform_name(INTEL_INFO(dev_priv)->platform),
719	intel_subplatform(RUNTIME_INFO(dev_priv),
720	INTEL_INFO(dev_priv)->platform),
721	GRAPHICS_VER(dev_priv));
722
723	intel_device_info_print(INTEL_INFO(dev_priv),
724	RUNTIME_INFO(dev_priv), p: &p);
725	i915_print_iommu_status(i915: dev_priv, p: &p);
726	for_each_gt(gt, dev_priv, i)
727	intel_gt_info_print(info: &gt->info, p: &p);
728	}
729
730	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
731	drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n");
732	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
733	drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n");
734	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
735	drm_info(&dev_priv->drm,
736	"DRM_I915_DEBUG_RUNTIME_PM enabled\n");
737	}
738
739	static struct drm_i915_private *
740	i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
741	{
742	const struct intel_device_info *match_info =
743	(struct intel_device_info *)ent->driver_data;
744	struct drm_i915_private *i915;
745
746	i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver,
747	struct drm_i915_private, drm);
748	if (IS_ERR(ptr: i915))
749	return i915;
750
751	pci_set_drvdata(pdev, data: &i915->drm);
752
753	/* Device parameters start as a copy of module parameters. */
754	i915_params_copy(dest: &i915->params, src: &i915_modparams);
755
756	/* Set up device info and initial runtime info. */
757	intel_device_info_driver_create(i915, device_id: pdev->device, match_info);
758
759	intel_display_device_probe(pdev);
760
761	return i915;
762	}
763
764	/**
765	* i915_driver_probe - setup chip and create an initial config
766	* @pdev: PCI device
767	* @ent: matching PCI ID entry
768	*
769	* The driver probe routine has to do several things:
770	* - drive output discovery via intel_display_driver_probe()
771	* - initialize the memory manager
772	* - allocate initial config memory
773	* - setup the DRM framebuffer with the allocated memory
774	*/
775	int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
776	{
777	struct drm_i915_private *i915;
778	struct intel_display *display;
779	int ret;
780
781	ret = pci_enable_device(dev: pdev);
782	if (ret) {
783	pr_err("Failed to enable graphics device: %pe\n", ERR_PTR(ret));
784	return ret;
785	}
786
787	i915 = i915_driver_create(pdev, ent);
788	if (IS_ERR(ptr: i915)) {
789	pci_disable_device(dev: pdev);
790	return PTR_ERR(ptr: i915);
791	}
792
793	display = &i915->display;
794
795	ret = i915_driver_early_probe(dev_priv: i915);
796	if (ret < 0)
797	goto out_pci_disable;
798
799	disable_rpm_wakeref_asserts(rpm: &i915->runtime_pm);
800
801	intel_vgpu_detect(i915);
802
803	ret = intel_gt_probe_all(i915);
804	if (ret < 0)
805	goto out_runtime_pm_put;
806
807	ret = i915_driver_mmio_probe(dev_priv: i915);
808	if (ret < 0)
809	goto out_runtime_pm_put;
810
811	ret = i915_driver_hw_probe(dev_priv: i915);
812	if (ret < 0)
813	goto out_cleanup_mmio;
814
815	ret = intel_display_driver_probe_noirq(display);
816	if (ret < 0)
817	goto out_cleanup_hw;
818
819	ret = intel_irq_install(dev_priv: i915);
820	if (ret)
821	goto out_cleanup_modeset;
822
823	ret = intel_display_driver_probe_nogem(display);
824	if (ret)
825	goto out_cleanup_irq;
826
827	ret = i915_gem_init(i915);
828	if (ret)
829	goto out_cleanup_modeset2;
830
831	ret = intel_pxp_init(i915);
832	if (ret && ret != -ENODEV)
833	drm_dbg(&i915->drm, "pxp init failed with %d\n", ret);
834
835	ret = intel_display_driver_probe(display);
836	if (ret)
837	goto out_cleanup_gem;
838
839	ret = i915_driver_register(dev_priv: i915);
840	if (ret)
841	goto out_cleanup_gem;
842
843	enable_rpm_wakeref_asserts(rpm: &i915->runtime_pm);
844
845	i915_welcome_messages(dev_priv: i915);
846
847	i915->do_release = true;
848
849	return 0;
850
851	out_cleanup_gem:
852	intel_pxp_fini(i915);
853	i915_gem_suspend(i915);
854	i915_gem_driver_remove(i915);
855	i915_gem_driver_release(i915);
856	out_cleanup_modeset2:
857	/* FIXME clean up the error path */
858	intel_display_driver_remove(display);
859	intel_irq_uninstall(dev_priv: i915);
860	intel_display_driver_remove_noirq(display);
861	goto out_cleanup_modeset;
862	out_cleanup_irq:
863	intel_irq_uninstall(dev_priv: i915);
864	out_cleanup_modeset:
865	intel_display_driver_remove_nogem(display);
866	out_cleanup_hw:
867	i915_driver_hw_remove(dev_priv: i915);
868	intel_memory_regions_driver_release(i915);
869	i915_ggtt_driver_release(i915);
870	i915_gem_drain_freed_objects(i915);
871	i915_ggtt_driver_late_release(i915);
872	out_cleanup_mmio:
873	i915_driver_mmio_release(dev_priv: i915);
874	out_runtime_pm_put:
875	enable_rpm_wakeref_asserts(rpm: &i915->runtime_pm);
876	i915_driver_late_release(dev_priv: i915);
877	out_pci_disable:
878	pci_disable_device(dev: pdev);
879	i915_probe_error(i915, "Device initialization failed (%d)\n", ret);
880	return ret;
881	}
882
883	void i915_driver_remove(struct drm_i915_private *i915)
884	{
885	struct intel_display *display = &i915->display;
886	intel_wakeref_t wakeref;
887
888	wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm);
889
890	i915_driver_unregister(dev_priv: i915);
891
892	/* Flush any external code that still may be under the RCU lock */
893	synchronize_rcu();
894
895	i915_gem_suspend(i915);
896
897	intel_gvt_driver_remove(dev_priv: i915);
898
899	intel_display_driver_remove(display);
900
901	intel_irq_uninstall(dev_priv: i915);
902
903	intel_display_driver_remove_noirq(display);
904
905	i915_reset_error_state(i915);
906	i915_gem_driver_remove(i915);
907
908	intel_display_driver_remove_nogem(display);
909
910	i915_driver_hw_remove(dev_priv: i915);
911
912	intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref);
913	}
914
915	static void i915_driver_release(struct drm_device *dev)
916	{
917	struct drm_i915_private *dev_priv = to_i915(dev);
918	struct intel_display *display = &dev_priv->display;
919	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
920	intel_wakeref_t wakeref;
921
922	if (!dev_priv->do_release)
923	return;
924
925	wakeref = intel_runtime_pm_get(rpm);
926
927	i915_gem_driver_release(i915: dev_priv);
928
929	intel_memory_regions_driver_release(i915: dev_priv);
930	i915_ggtt_driver_release(i915: dev_priv);
931	i915_gem_drain_freed_objects(i915: dev_priv);
932	i915_ggtt_driver_late_release(i915: dev_priv);
933
934	i915_driver_mmio_release(dev_priv);
935
936	intel_runtime_pm_put(rpm, wref: wakeref);
937
938	intel_runtime_pm_driver_release(rpm);
939
940	i915_driver_late_release(dev_priv);
941
942	intel_display_device_remove(display);
943	}
944
945	static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
946	{
947	struct drm_i915_private *i915 = to_i915(dev);
948	int ret;
949
950	ret = i915_gem_open(i915, file);
951	if (ret)
952	return ret;
953
954	return 0;
955	}
956
957	static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
958	{
959	struct drm_i915_file_private *file_priv = file->driver_priv;
960
961	i915_gem_context_close(file);
962	i915_drm_client_put(client: file_priv->client);
963
964	kfree_rcu(file_priv, rcu);
965
966	/* Catch up with all the deferred frees from "this" client */
967	i915_gem_flush_free_objects(i915: to_i915(dev));
968	}
969
970	void i915_driver_shutdown(struct drm_i915_private *i915)
971	{
972	struct intel_display *display = &i915->display;
973
974	disable_rpm_wakeref_asserts(rpm: &i915->runtime_pm);
975	intel_runtime_pm_disable(rpm: &i915->runtime_pm);
976	intel_power_domains_disable(display);
977
978	drm_client_dev_suspend(dev: &i915->drm, holds_console_lock: false);
979	if (HAS_DISPLAY(i915)) {
980	drm_kms_helper_poll_disable(dev: &i915->drm);
981	intel_display_driver_disable_user_access(display);
982
983	drm_atomic_helper_shutdown(dev: &i915->drm);
984	}
985
986	intel_dp_mst_suspend(display);
987
988	intel_irq_suspend(i915);
989	intel_hpd_cancel_work(display);
990
991	if (HAS_DISPLAY(i915))
992	intel_display_driver_suspend_access(display);
993
994	intel_encoder_suspend_all(display: &i915->display);
995	intel_encoder_shutdown_all(display: &i915->display);
996
997	intel_dmc_suspend(display: &i915->display);
998
999	i915_gem_suspend(i915);
1000
1001	/*
1002	* The only requirement is to reboot with display DC states disabled,
1003	* for now leaving all display power wells in the INIT power domain
1004	* enabled.
1005	*
1006	* TODO:
1007	* - unify the pci_driver::shutdown sequence here with the
1008	* pci_driver.driver.pm.poweroff,poweroff_late sequence.
1009	* - unify the driver remove and system/runtime suspend sequences with
1010	* the above unified shutdown/poweroff sequence.
1011	*/
1012	intel_power_domains_driver_remove(display);
1013	enable_rpm_wakeref_asserts(rpm: &i915->runtime_pm);
1014
1015	intel_runtime_pm_driver_last_release(rpm: &i915->runtime_pm);
1016	}
1017
1018	static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1019	{
1020	#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1021	if (acpi_target_system_state() < ACPI_STATE_S3)
1022	return true;
1023	#endif
1024	return false;
1025	}
1026
1027	static void i915_drm_complete(struct drm_device *dev)
1028	{
1029	struct drm_i915_private *i915 = to_i915(dev);
1030
1031	intel_pxp_resume_complete(pxp: i915->pxp);
1032	}
1033
1034	static int i915_drm_prepare(struct drm_device *dev)
1035	{
1036	struct drm_i915_private *i915 = to_i915(dev);
1037
1038	intel_pxp_suspend_prepare(pxp: i915->pxp);
1039
1040	/*
1041	* NB intel_display_driver_suspend() may issue new requests after we've
1042	* ostensibly marked the GPU as ready-to-sleep here. We need to
1043	* split out that work and pull it forward so that after point,
1044	* the GPU is not woken again.
1045	*/
1046	return i915_gem_backup_suspend(i915);
1047	}
1048
1049	static int i915_drm_suspend(struct drm_device *dev)
1050	{
1051	struct drm_i915_private *dev_priv = to_i915(dev);
1052	struct intel_display *display = &dev_priv->display;
1053	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1054	pci_power_t opregion_target_state;
1055
1056	disable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1057
1058	/* We do a lot of poking in a lot of registers, make sure they work
1059	* properly. */
1060	intel_power_domains_disable(display);
1061	drm_client_dev_suspend(dev, holds_console_lock: false);
1062	if (HAS_DISPLAY(dev_priv)) {
1063	drm_kms_helper_poll_disable(dev);
1064	intel_display_driver_disable_user_access(display);
1065	}
1066
1067	pci_save_state(dev: pdev);
1068
1069	intel_display_driver_suspend(display);
1070
1071	intel_irq_suspend(i915: dev_priv);
1072	intel_hpd_cancel_work(display);
1073
1074	if (HAS_DISPLAY(dev_priv))
1075	intel_display_driver_suspend_access(display);
1076
1077	intel_encoder_suspend_all(display: &dev_priv->display);
1078
1079	/* Must be called before GGTT is suspended. */
1080	intel_dpt_suspend(display);
1081	i915_ggtt_suspend(gtt: to_gt(i915: dev_priv)->ggtt);
1082
1083	i9xx_display_sr_save(display);
1084
1085	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1086	intel_opregion_suspend(display, state: opregion_target_state);
1087
1088	dev_priv->suspend_count++;
1089
1090	intel_dmc_suspend(display);
1091
1092	enable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1093
1094	i915_gem_drain_freed_objects(i915: dev_priv);
1095
1096	return 0;
1097	}
1098
1099	static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1100	{
1101	struct drm_i915_private *dev_priv = to_i915(dev);
1102	struct intel_display *display = &dev_priv->display;
1103	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1104	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1105	struct intel_gt *gt;
1106	int ret, i;
1107	bool s2idle = !hibernation && suspend_to_idle(dev_priv);
1108
1109	disable_rpm_wakeref_asserts(rpm);
1110
1111	intel_pxp_suspend(pxp: dev_priv->pxp);
1112
1113	i915_gem_suspend_late(i915: dev_priv);
1114
1115	for_each_gt(gt, dev_priv, i)
1116	intel_uncore_suspend(uncore: gt->uncore);
1117
1118	intel_display_power_suspend_late(display, s2idle);
1119
1120	ret = vlv_suspend_complete(i915: dev_priv);
1121	if (ret) {
1122	drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);
1123	intel_display_power_resume_early(display);
1124
1125	goto out;
1126	}
1127
1128	pci_disable_device(dev: pdev);
1129	/*
1130	* During hibernation on some platforms the BIOS may try to access
1131	* the device even though it's already in D3 and hang the machine. So
1132	* leave the device in D0 on those platforms and hope the BIOS will
1133	* power down the device properly. The issue was seen on multiple old
1134	* GENs with different BIOS vendors, so having an explicit blacklist
1135	* is impractical; apply the workaround on everything pre GEN6. The
1136	* platforms where the issue was seen:
1137	* Lenovo Thinkpad X301, X61s, X60, T60, X41
1138	* Fujitsu FSC S7110
1139	* Acer Aspire 1830T
1140	*/
1141	if (!(hibernation && GRAPHICS_VER(dev_priv) < 6))
1142	pci_set_power_state(dev: pdev, PCI_D3hot);
1143
1144	out:
1145	enable_rpm_wakeref_asserts(rpm);
1146	if (!dev_priv->uncore.user_forcewake_count)
1147	intel_runtime_pm_driver_release(rpm);
1148
1149	return ret;
1150	}
1151
1152	int i915_driver_suspend_switcheroo(struct drm_i915_private *i915,
1153	pm_message_t state)
1154	{
1155	int error;
1156
1157	if (drm_WARN_ON_ONCE(&i915->drm, state.event != PM_EVENT_SUSPEND &&
1158	state.event != PM_EVENT_FREEZE))
1159	return -EINVAL;
1160
1161	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1162	return 0;
1163
1164	error = i915_drm_suspend(dev: &i915->drm);
1165	if (error)
1166	return error;
1167
1168	return i915_drm_suspend_late(dev: &i915->drm, hibernation: false);
1169	}
1170
1171	static int i915_drm_resume(struct drm_device *dev)
1172	{
1173	struct drm_i915_private *dev_priv = to_i915(dev);
1174	struct intel_display *display = &dev_priv->display;
1175	struct intel_gt *gt;
1176	int ret, i;
1177
1178	disable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1179
1180	ret = i915_pcode_init(i915: dev_priv);
1181	if (ret)
1182	return ret;
1183
1184	sanitize_gpu(i915: dev_priv);
1185
1186	ret = i915_ggtt_enable_hw(i915: dev_priv);
1187	if (ret)
1188	drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");
1189
1190	i915_ggtt_resume(ggtt: to_gt(i915: dev_priv)->ggtt);
1191
1192	for_each_gt(gt, dev_priv, i)
1193	if (GRAPHICS_VER(gt->i915) >= 8)
1194	setup_private_pat(gt);
1195
1196	/* Must be called after GGTT is resumed. */
1197	intel_dpt_resume(display);
1198
1199	intel_dmc_resume(display);
1200
1201	i9xx_display_sr_restore(display);
1202
1203	intel_gmbus_reset(display);
1204
1205	intel_pps_unlock_regs_wa(display);
1206
1207	intel_init_pch_refclk(display);
1208
1209	/*
1210	* Interrupts have to be enabled before any batches are run. If not the
1211	* GPU will hang. i915_gem_init_hw() will initiate batches to
1212	* update/restore the context.
1213	*
1214	* drm_mode_config_reset() needs AUX interrupts.
1215	*
1216	* Modeset enabling in intel_display_driver_init_hw() also needs working
1217	* interrupts.
1218	*/
1219	intel_irq_resume(i915: dev_priv);
1220
1221	if (HAS_DISPLAY(dev_priv))
1222	drm_mode_config_reset(dev);
1223
1224	i915_gem_resume(i915: dev_priv);
1225
1226	intel_display_driver_init_hw(display);
1227
1228	intel_clock_gating_init(i915: dev_priv);
1229
1230	if (HAS_DISPLAY(dev_priv))
1231	intel_display_driver_resume_access(display);
1232
1233	intel_hpd_init(display);
1234
1235	intel_display_driver_resume(display);
1236
1237	if (HAS_DISPLAY(dev_priv)) {
1238	intel_display_driver_enable_user_access(display);
1239	drm_kms_helper_poll_enable(dev);
1240	}
1241	intel_hpd_poll_disable(display);
1242
1243	intel_opregion_resume(display);
1244
1245	drm_client_dev_resume(dev, holds_console_lock: false);
1246
1247	intel_power_domains_enable(display);
1248
1249	intel_gvt_resume(dev_priv);
1250
1251	enable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1252
1253	return 0;
1254	}
1255
1256	static int i915_drm_resume_early(struct drm_device *dev)
1257	{
1258	struct drm_i915_private *dev_priv = to_i915(dev);
1259	struct intel_display *display = &dev_priv->display;
1260	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1261	struct intel_gt *gt;
1262	int ret, i;
1263
1264	/*
1265	* We have a resume ordering issue with the snd-hda driver also
1266	* requiring our device to be power up. Due to the lack of a
1267	* parent/child relationship we currently solve this with an early
1268	* resume hook.
1269	*
1270	* FIXME: This should be solved with a special hdmi sink device or
1271	* similar so that power domains can be employed.
1272	*/
1273
1274	/*
1275	* Note that we need to set the power state explicitly, since we
1276	* powered off the device during freeze and the PCI core won't power
1277	* it back up for us during thaw. Powering off the device during
1278	* freeze is not a hard requirement though, and during the
1279	* suspend/resume phases the PCI core makes sure we get here with the
1280	* device powered on. So in case we change our freeze logic and keep
1281	* the device powered we can also remove the following set power state
1282	* call.
1283	*/
1284	ret = pci_set_power_state(dev: pdev, PCI_D0);
1285	if (ret) {
1286	drm_err(&dev_priv->drm,
1287	"failed to set PCI D0 power state (%d)\n", ret);
1288	return ret;
1289	}
1290
1291	/*
1292	* Note that pci_enable_device() first enables any parent bridge
1293	* device and only then sets the power state for this device. The
1294	* bridge enabling is a nop though, since bridge devices are resumed
1295	* first. The order of enabling power and enabling the device is
1296	* imposed by the PCI core as described above, so here we preserve the
1297	* same order for the freeze/thaw phases.
1298	*
1299	* TODO: eventually we should remove pci_disable_device() /
1300	* pci_enable_enable_device() from suspend/resume. Due to how they
1301	* depend on the device enable refcount we can't anyway depend on them
1302	* disabling/enabling the device.
1303	*/
1304	if (pci_enable_device(dev: pdev))
1305	return -EIO;
1306
1307	pci_set_master(dev: pdev);
1308
1309	disable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1310
1311	ret = vlv_resume_prepare(i915: dev_priv, rpm_resume: false);
1312	if (ret)
1313	drm_err(&dev_priv->drm,
1314	"Resume prepare failed: %d, continuing anyway\n", ret);
1315
1316	for_each_gt(gt, dev_priv, i)
1317	intel_gt_resume_early(gt);
1318
1319	intel_display_power_resume_early(display);
1320
1321	enable_rpm_wakeref_asserts(rpm: &dev_priv->runtime_pm);
1322
1323	return ret;
1324	}
1325
1326	int i915_driver_resume_switcheroo(struct drm_i915_private *i915)
1327	{
1328	int ret;
1329
1330	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1331	return 0;
1332
1333	ret = i915_drm_resume_early(dev: &i915->drm);
1334	if (ret)
1335	return ret;
1336
1337	return i915_drm_resume(dev: &i915->drm);
1338	}
1339
1340	static int i915_pm_prepare(struct device *kdev)
1341	{
1342	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1343
1344	if (!i915) {
1345	dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1346	return -ENODEV;
1347	}
1348
1349	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1350	return 0;
1351
1352	return i915_drm_prepare(dev: &i915->drm);
1353	}
1354
1355	static int i915_pm_suspend(struct device *kdev)
1356	{
1357	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1358
1359	if (!i915) {
1360	dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1361	return -ENODEV;
1362	}
1363
1364	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1365	return 0;
1366
1367	return i915_drm_suspend(dev: &i915->drm);
1368	}
1369
1370	static int i915_pm_suspend_late(struct device *kdev)
1371	{
1372	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1373
1374	/*
1375	* We have a suspend ordering issue with the snd-hda driver also
1376	* requiring our device to be power up. Due to the lack of a
1377	* parent/child relationship we currently solve this with an late
1378	* suspend hook.
1379	*
1380	* FIXME: This should be solved with a special hdmi sink device or
1381	* similar so that power domains can be employed.
1382	*/
1383	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1384	return 0;
1385
1386	return i915_drm_suspend_late(dev: &i915->drm, hibernation: false);
1387	}
1388
1389	static int i915_pm_poweroff_late(struct device *kdev)
1390	{
1391	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1392
1393	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1394	return 0;
1395
1396	return i915_drm_suspend_late(dev: &i915->drm, hibernation: true);
1397	}
1398
1399	static int i915_pm_resume_early(struct device *kdev)
1400	{
1401	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1402
1403	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1404	return 0;
1405
1406	return i915_drm_resume_early(dev: &i915->drm);
1407	}
1408
1409	static int i915_pm_resume(struct device *kdev)
1410	{
1411	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1412
1413	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1414	return 0;
1415
1416	return i915_drm_resume(dev: &i915->drm);
1417	}
1418
1419	static void i915_pm_complete(struct device *kdev)
1420	{
1421	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1422
1423	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1424	return;
1425
1426	i915_drm_complete(dev: &i915->drm);
1427	}
1428
1429	/* freeze: before creating the hibernation_image */
1430	static int i915_pm_freeze(struct device *kdev)
1431	{
1432	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1433	int ret;
1434
1435	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
1436	ret = i915_drm_suspend(dev: &i915->drm);
1437	if (ret)
1438	return ret;
1439	}
1440
1441	ret = i915_gem_freeze(i915);
1442	if (ret)
1443	return ret;
1444
1445	return 0;
1446	}
1447
1448	static int i915_pm_freeze_late(struct device *kdev)
1449	{
1450	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1451	int ret;
1452
1453	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
1454	ret = i915_drm_suspend_late(dev: &i915->drm, hibernation: true);
1455	if (ret)
1456	return ret;
1457	}
1458
1459	ret = i915_gem_freeze_late(i915);
1460	if (ret)
1461	return ret;
1462
1463	return 0;
1464	}
1465
1466	/* thaw: called after creating the hibernation image, but before turning off. */
1467	static int i915_pm_thaw_early(struct device *kdev)
1468	{
1469	return i915_pm_resume_early(kdev);
1470	}
1471
1472	static int i915_pm_thaw(struct device *kdev)
1473	{
1474	return i915_pm_resume(kdev);
1475	}
1476
1477	/* restore: called after loading the hibernation image. */
1478	static int i915_pm_restore_early(struct device *kdev)
1479	{
1480	return i915_pm_resume_early(kdev);
1481	}
1482
1483	static int i915_pm_restore(struct device *kdev)
1484	{
1485	return i915_pm_resume(kdev);
1486	}
1487
1488	static int intel_runtime_suspend(struct device *kdev)
1489	{
1490	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1491	struct intel_display *display = &dev_priv->display;
1492	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1493	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1494	struct pci_dev *root_pdev;
1495	struct intel_gt *gt;
1496	int ret, i;
1497
1498	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1499	return -ENODEV;
1500
1501	drm_dbg(&dev_priv->drm, "Suspending device\n");
1502
1503	disable_rpm_wakeref_asserts(rpm);
1504
1505	/*
1506	* We are safe here against re-faults, since the fault handler takes
1507	* an RPM reference.
1508	*/
1509	i915_gem_runtime_suspend(i915: dev_priv);
1510
1511	intel_pxp_runtime_suspend(pxp: dev_priv->pxp);
1512
1513	for_each_gt(gt, dev_priv, i)
1514	intel_gt_runtime_suspend(gt);
1515
1516	intel_irq_suspend(i915: dev_priv);
1517
1518	for_each_gt(gt, dev_priv, i)
1519	intel_uncore_suspend(uncore: gt->uncore);
1520
1521	intel_display_power_suspend(display);
1522
1523	ret = vlv_suspend_complete(i915: dev_priv);
1524	if (ret) {
1525	drm_err(&dev_priv->drm,
1526	"Runtime suspend failed, disabling it (%d)\n", ret);
1527	intel_uncore_runtime_resume(uncore: &dev_priv->uncore);
1528
1529	intel_irq_resume(i915: dev_priv);
1530
1531	for_each_gt(gt, dev_priv, i)
1532	intel_gt_runtime_resume(gt);
1533
1534	enable_rpm_wakeref_asserts(rpm);
1535
1536	return ret;
1537	}
1538
1539	enable_rpm_wakeref_asserts(rpm);
1540	intel_runtime_pm_driver_release(rpm);
1541
1542	if (intel_uncore_arm_unclaimed_mmio_detection(uncore: &dev_priv->uncore))
1543	drm_err(&dev_priv->drm,
1544	"Unclaimed access detected prior to suspending\n");
1545
1546	/*
1547	* FIXME: Temporary hammer to avoid freezing the machine on our DGFX
1548	* This should be totally removed when we handle the pci states properly
1549	* on runtime PM.
1550	*/
1551	root_pdev = pcie_find_root_port(dev: pdev);
1552	if (root_pdev)
1553	pci_d3cold_disable(dev: root_pdev);
1554
1555	/*
1556	* FIXME: We really should find a document that references the arguments
1557	* used below!
1558	*/
1559	if (IS_BROADWELL(dev_priv)) {
1560	/*
1561	* On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
1562	* being detected, and the call we do at intel_runtime_resume()
1563	* won't be able to restore them. Since PCI_D3hot matches the
1564	* actual specification and appears to be working, use it.
1565	*/
1566	intel_opregion_notify_adapter(display, PCI_D3hot);
1567	} else {
1568	/*
1569	* current versions of firmware which depend on this opregion
1570	* notification have repurposed the D1 definition to mean
1571	* "runtime suspended" vs. what you would normally expect (D3)
1572	* to distinguish it from notifications that might be sent via
1573	* the suspend path.
1574	*/
1575	intel_opregion_notify_adapter(display, PCI_D1);
1576	}
1577
1578	assert_forcewakes_inactive(uncore: &dev_priv->uncore);
1579
1580	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
1581	intel_hpd_poll_enable(display);
1582
1583	drm_dbg(&dev_priv->drm, "Device suspended\n");
1584	return 0;
1585	}
1586
1587	static int intel_runtime_resume(struct device *kdev)
1588	{
1589	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1590	struct intel_display *display = &dev_priv->display;
1591	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1592	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1593	struct pci_dev *root_pdev;
1594	struct intel_gt *gt;
1595	int ret, i;
1596
1597	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1598	return -ENODEV;
1599
1600	drm_dbg(&dev_priv->drm, "Resuming device\n");
1601
1602	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));
1603	disable_rpm_wakeref_asserts(rpm);
1604
1605	intel_opregion_notify_adapter(display, PCI_D0);
1606
1607	root_pdev = pcie_find_root_port(dev: pdev);
1608	if (root_pdev)
1609	pci_d3cold_enable(dev: root_pdev);
1610
1611	if (intel_uncore_unclaimed_mmio(uncore: &dev_priv->uncore))
1612	drm_dbg(&dev_priv->drm,
1613	"Unclaimed access during suspend, bios?\n");
1614
1615	intel_display_power_resume(display);
1616
1617	ret = vlv_resume_prepare(i915: dev_priv, rpm_resume: true);
1618
1619	for_each_gt(gt, dev_priv, i)
1620	intel_uncore_runtime_resume(uncore: gt->uncore);
1621
1622	intel_irq_resume(i915: dev_priv);
1623
1624	/*
1625	* No point of rolling back things in case of an error, as the best
1626	* we can do is to hope that things will still work (and disable RPM).
1627	*/
1628	for_each_gt(gt, dev_priv, i)
1629	intel_gt_runtime_resume(gt);
1630
1631	intel_pxp_runtime_resume(pxp: dev_priv->pxp);
1632
1633	/*
1634	* On VLV/CHV display interrupts are part of the display
1635	* power well, so hpd is reinitialized from there. For
1636	* everyone else do it here.
1637	*/
1638	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
1639	intel_hpd_init(display);
1640	intel_hpd_poll_disable(display);
1641	}
1642
1643	skl_watermark_ipc_update(display);
1644
1645	enable_rpm_wakeref_asserts(rpm);
1646
1647	if (ret)
1648	drm_err(&dev_priv->drm,
1649	"Runtime resume failed, disabling it (%d)\n", ret);
1650	else
1651	drm_dbg(&dev_priv->drm, "Device resumed\n");
1652
1653	return ret;
1654	}
1655
1656	const struct dev_pm_ops i915_pm_ops = {
1657	/*
1658	* S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
1659	* PMSG_RESUME]
1660	*/
1661	.prepare = i915_pm_prepare,
1662	.suspend = i915_pm_suspend,
1663	.suspend_late = i915_pm_suspend_late,
1664	.resume_early = i915_pm_resume_early,
1665	.resume = i915_pm_resume,
1666	.complete = i915_pm_complete,
1667
1668	/*
1669	* S4 event handlers
1670	* @freeze, @freeze_late : called (1) before creating the
1671	* hibernation image [PMSG_FREEZE] and
1672	* (2) after rebooting, before restoring
1673	* the image [PMSG_QUIESCE]
1674	* @thaw, @thaw_early : called (1) after creating the hibernation
1675	* image, before writing it [PMSG_THAW]
1676	* and (2) after failing to create or
1677	* restore the image [PMSG_RECOVER]
1678	* @poweroff, @poweroff_late: called after writing the hibernation
1679	* image, before rebooting [PMSG_HIBERNATE]
1680	* @restore, @restore_early : called after rebooting and restoring the
1681	* hibernation image [PMSG_RESTORE]
1682	*/
1683	.freeze = i915_pm_freeze,
1684	.freeze_late = i915_pm_freeze_late,
1685	.thaw_early = i915_pm_thaw_early,
1686	.thaw = i915_pm_thaw,
1687	.poweroff = i915_pm_suspend,
1688	.poweroff_late = i915_pm_poweroff_late,
1689	.restore_early = i915_pm_restore_early,
1690	.restore = i915_pm_restore,
1691
1692	/* S0ix (via runtime suspend) event handlers */
1693	.runtime_suspend = intel_runtime_suspend,
1694	.runtime_resume = intel_runtime_resume,
1695	};
1696
1697	static const struct file_operations i915_driver_fops = {
1698	.owner = THIS_MODULE,
1699	.open = drm_open,
1700	.release = drm_release_noglobal,
1701	.unlocked_ioctl = drm_ioctl,
1702	.mmap = i915_gem_mmap,
1703	.poll = drm_poll,
1704	.read = drm_read,
1705	.compat_ioctl = i915_ioc32_compat_ioctl,
1706	.llseek = noop_llseek,
1707	#ifdef CONFIG_PROC_FS
1708	.show_fdinfo = drm_show_fdinfo,
1709	#endif
1710	.fop_flags = FOP_UNSIGNED_OFFSET,
1711	};
1712
1713	static int
1714	i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
1715	struct drm_file *file)
1716	{
1717	return -ENODEV;
1718	}
1719
1720	static const struct drm_ioctl_desc i915_ioctls[] = {
1721	DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1722	DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
1723	DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
1724	DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
1725	DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
1726	DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
1727	DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW),
1728	DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1729	DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
1730	DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
1731	DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1732	DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
1733	DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1734	DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1735	DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
1736	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
1737	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1738	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1739	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, drm_invalid_op, DRM_AUTH),
1740	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
1741	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1742	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1743	DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW),
1744	DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
1745	DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
1746	DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW),
1747	DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1748	DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1749	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
1750	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE_EXT, i915_gem_create_ext_ioctl, DRM_RENDER_ALLOW),
1751	DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
1752	DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
1753	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
1754	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_OFFSET, i915_gem_mmap_offset_ioctl, DRM_RENDER_ALLOW),
1755	DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
1756	DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
1757	DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW),
1758	DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW),
1759	DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
1760	DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_crtc_get_pipe_from_crtc_id_ioctl, 0),
1761	DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
1762	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER),
1763	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER),
1764	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
1765	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
1766	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW),
1767	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
1768	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
1769	DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
1770	DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
1771	DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
1772	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
1773	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
1774	DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
1775	DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW),
1776	DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW),
1777	DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW),
1778	DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW),
1779	DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW),
1780	};
1781
1782	/*
1783	* Interface history:
1784	*
1785	* 1.1: Original.
1786	* 1.2: Add Power Management
1787	* 1.3: Add vblank support
1788	* 1.4: Fix cmdbuffer path, add heap destroy
1789	* 1.5: Add vblank pipe configuration
1790	* 1.6: - New ioctl for scheduling buffer swaps on vertical blank
1791	* - Support vertical blank on secondary display pipe
1792	*/
1793	#define DRIVER_MAJOR 1
1794	#define DRIVER_MINOR 6
1795	#define DRIVER_PATCHLEVEL 0
1796
1797	static const struct drm_driver i915_drm_driver = {
1798	/* Don't use MTRRs here; the Xserver or userspace app should
1799	* deal with them for Intel hardware.
1800	*/
1801	.driver_features =
1802	DRIVER_GEM |
1803	DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ |
1804	DRIVER_SYNCOBJ_TIMELINE,
1805	.release = i915_driver_release,
1806	.open = i915_driver_open,
1807	.postclose = i915_driver_postclose,
1808	.show_fdinfo = PTR_IF(IS_ENABLED(CONFIG_PROC_FS), i915_drm_client_fdinfo),
1809
1810	.gem_prime_import = i915_gem_prime_import,
1811
1812	.dumb_create = i915_gem_dumb_create,
1813	.dumb_map_offset = i915_gem_dumb_mmap_offset,
1814
1815	INTEL_FBDEV_DRIVER_OPS,
1816
1817	.ioctls = i915_ioctls,
1818	.num_ioctls = ARRAY_SIZE(i915_ioctls),
1819	.fops = &i915_driver_fops,
1820	.name = DRIVER_NAME,
1821	.desc = DRIVER_DESC,
1822	.major = DRIVER_MAJOR,
1823	.minor = DRIVER_MINOR,
1824	.patchlevel = DRIVER_PATCHLEVEL,
1825	};
1826