1	/*
2	* Copyright © 2015-2016 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*
23	* Authors:
24	* Robert Bragg <robert@sixbynine.org>
25	*/
26
27
28	/**
29	* DOC: i915 Perf Overview
30	*
31	* Gen graphics supports a large number of performance counters that can help
32	* driver and application developers understand and optimize their use of the
33	* GPU.
34	*
35	* This i915 perf interface enables userspace to configure and open a file
36	* descriptor representing a stream of GPU metrics which can then be read() as
37	* a stream of sample records.
38	*
39	* The interface is particularly suited to exposing buffered metrics that are
40	* captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
41	*
42	* Streams representing a single context are accessible to applications with a
43	* corresponding drm file descriptor, such that OpenGL can use the interface
44	* without special privileges. Access to system-wide metrics requires root
45	* privileges by default, unless changed via the dev.i915.perf_event_paranoid
46	* sysctl option.
47	*
48	*/
49
50	/**
51	* DOC: i915 Perf History and Comparison with Core Perf
52	*
53	* The interface was initially inspired by the core Perf infrastructure but
54	* some notable differences are:
55	*
56	* i915 perf file descriptors represent a "stream" instead of an "event"; where
57	* a perf event primarily corresponds to a single 64bit value, while a stream
58	* might sample sets of tightly-coupled counters, depending on the
59	* configuration. For example the Gen OA unit isn't designed to support
60	* orthogonal configurations of individual counters; it's configured for a set
61	* of related counters. Samples for an i915 perf stream capturing OA metrics
62	* will include a set of counter values packed in a compact HW specific format.
63	* The OA unit supports a number of different packing formats which can be
64	* selected by the user opening the stream. Perf has support for grouping
65	* events, but each event in the group is configured, validated and
66	* authenticated individually with separate system calls.
67	*
68	* i915 perf stream configurations are provided as an array of u64 (key,value)
69	* pairs, instead of a fixed struct with multiple miscellaneous config members,
70	* interleaved with event-type specific members.
71	*
72	* i915 perf doesn't support exposing metrics via an mmap'd circular buffer.
73	* The supported metrics are being written to memory by the GPU unsynchronized
74	* with the CPU, using HW specific packing formats for counter sets. Sometimes
75	* the constraints on HW configuration require reports to be filtered before it
76	* would be acceptable to expose them to unprivileged applications - to hide
77	* the metrics of other processes/contexts. For these use cases a read() based
78	* interface is a good fit, and provides an opportunity to filter data as it
79	* gets copied from the GPU mapped buffers to userspace buffers.
80	*
81	*
82	* Issues hit with first prototype based on Core Perf
83	* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
84	*
85	* The first prototype of this driver was based on the core perf
86	* infrastructure, and while we did make that mostly work, with some changes to
87	* perf, we found we were breaking or working around too many assumptions baked
88	* into perf's currently cpu centric design.
89	*
90	* In the end we didn't see a clear benefit to making perf's implementation and
91	* interface more complex by changing design assumptions while we knew we still
92	* wouldn't be able to use any existing perf based userspace tools.
93	*
94	* Also considering the Gen specific nature of the Observability hardware and
95	* how userspace will sometimes need to combine i915 perf OA metrics with
96	* side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
97	* expecting the interface to be used by a platform specific userspace such as
98	* OpenGL or tools. This is to say; we aren't inherently missing out on having
99	* a standard vendor/architecture agnostic interface by not using perf.
100	*
101	*
102	* For posterity, in case we might re-visit trying to adapt core perf to be
103	* better suited to exposing i915 metrics these were the main pain points we
104	* hit:
105	*
106	* - The perf based OA PMU driver broke some significant design assumptions:
107	*
108	* Existing perf pmus are used for profiling work on a cpu and we were
109	* introducing the idea of _IS_DEVICE pmus with different security
110	* implications, the need to fake cpu-related data (such as user/kernel
111	* registers) to fit with perf's current design, and adding _DEVICE records
112	* as a way to forward device-specific status records.
113	*
114	* The OA unit writes reports of counters into a circular buffer, without
115	* involvement from the CPU, making our PMU driver the first of a kind.
116	*
117	* Given the way we were periodically forward data from the GPU-mapped, OA
118	* buffer to perf's buffer, those bursts of sample writes looked to perf like
119	* we were sampling too fast and so we had to subvert its throttling checks.
120	*
121	* Perf supports groups of counters and allows those to be read via
122	* transactions internally but transactions currently seem designed to be
123	* explicitly initiated from the cpu (say in response to a userspace read())
124	* and while we could pull a report out of the OA buffer we can't
125	* trigger a report from the cpu on demand.
126	*
127	* Related to being report based; the OA counters are configured in HW as a
128	* set while perf generally expects counter configurations to be orthogonal.
129	* Although counters can be associated with a group leader as they are
130	* opened, there's no clear precedent for being able to provide group-wide
131	* configuration attributes (for example we want to let userspace choose the
132	* OA unit report format used to capture all counters in a set, or specify a
133	* GPU context to filter metrics on). We avoided using perf's grouping
134	* feature and forwarded OA reports to userspace via perf's 'raw' sample
135	* field. This suited our userspace well considering how coupled the counters
136	* are when dealing with normalizing. It would be inconvenient to split
137	* counters up into separate events, only to require userspace to recombine
138	* them. For Mesa it's also convenient to be forwarded raw, periodic reports
139	* for combining with the side-band raw reports it captures using
140	* MI_REPORT_PERF_COUNT commands.
141	*
142	* - As a side note on perf's grouping feature; there was also some concern
143	* that using PERF_FORMAT_GROUP as a way to pack together counter values
144	* would quite drastically inflate our sample sizes, which would likely
145	* lower the effective sampling resolutions we could use when the available
146	* memory bandwidth is limited.
147	*
148	* With the OA unit's report formats, counters are packed together as 32
149	* or 40bit values, with the largest report size being 256 bytes.
150	*
151	* PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a
152	* documented ordering to the values, implying PERF_FORMAT_ID must also be
153	* used to add a 64bit ID before each value; giving 16 bytes per counter.
154	*
155	* Related to counter orthogonality; we can't time share the OA unit, while
156	* event scheduling is a central design idea within perf for allowing
157	* userspace to open + enable more events than can be configured in HW at any
158	* one time. The OA unit is not designed to allow re-configuration while in
159	* use. We can't reconfigure the OA unit without losing internal OA unit
160	* state which we can't access explicitly to save and restore. Reconfiguring
161	* the OA unit is also relatively slow, involving ~100 register writes. From
162	* userspace Mesa also depends on a stable OA configuration when emitting
163	* MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be
164	* disabled while there are outstanding MI_RPC commands lest we hang the
165	* command streamer.
166	*
167	* The contents of sample records aren't extensible by device drivers (i.e.
168	* the sample_type bits). As an example; Sourab Gupta had been looking to
169	* attach GPU timestamps to our OA samples. We were shoehorning OA reports
170	* into sample records by using the 'raw' field, but it's tricky to pack more
171	* than one thing into this field because events/core.c currently only lets a
172	* pmu give a single raw data pointer plus len which will be copied into the
173	* ring buffer. To include more than the OA report we'd have to copy the
174	* report into an intermediate larger buffer. I'd been considering allowing a
175	* vector of data+len values to be specified for copying the raw data, but
176	* it felt like a kludge to being using the raw field for this purpose.
177	*
178	* - It felt like our perf based PMU was making some technical compromises
179	* just for the sake of using perf:
180	*
181	* perf_event_open() requires events to either relate to a pid or a specific
182	* cpu core, while our device pmu related to neither. Events opened with a
183	* pid will be automatically enabled/disabled according to the scheduling of
184	* that process - so not appropriate for us. When an event is related to a
185	* cpu id, perf ensures pmu methods will be invoked via an inter process
186	* interrupt on that core. To avoid invasive changes our userspace opened OA
187	* perf events for a specific cpu. This was workable but it meant the
188	* majority of the OA driver ran in atomic context, including all OA report
189	* forwarding, which wasn't really necessary in our case and seems to make
190	* our locking requirements somewhat complex as we handled the interaction
191	* with the rest of the i915 driver.
192	*/
193
194	#include <linux/anon_inodes.h>
195	#include <linux/nospec.h>
196	#include <linux/sizes.h>
197	#include <linux/uuid.h>
198
199	#include "gem/i915_gem_context.h"
200	#include "gem/i915_gem_internal.h"
201	#include "gt/intel_engine_pm.h"
202	#include "gt/intel_engine_regs.h"
203	#include "gt/intel_engine_user.h"
204	#include "gt/intel_execlists_submission.h"
205	#include "gt/intel_gpu_commands.h"
206	#include "gt/intel_gt.h"
207	#include "gt/intel_gt_clock_utils.h"
208	#include "gt/intel_gt_mcr.h"
209	#include "gt/intel_gt_print.h"
210	#include "gt/intel_gt_regs.h"
211	#include "gt/intel_lrc.h"
212	#include "gt/intel_lrc_reg.h"
213	#include "gt/intel_rc6.h"
214	#include "gt/intel_ring.h"
215	#include "gt/uc/intel_guc_slpc.h"
216
217	#include "i915_drv.h"
218	#include "i915_file_private.h"
219	#include "i915_perf.h"
220	#include "i915_perf_oa_regs.h"
221	#include "i915_reg.h"
222
223	/* HW requires this to be a power of two, between 128k and 16M, though driver
224	* is currently generally designed assuming the largest 16M size is used such
225	* that the overflow cases are unlikely in normal operation.
226	*/
227	#define OA_BUFFER_SIZE SZ_16M
228
229	#define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
230
231	/**
232	* DOC: OA Tail Pointer Race
233	*
234	* There's a HW race condition between OA unit tail pointer register updates and
235	* writes to memory whereby the tail pointer can sometimes get ahead of what's
236	* been written out to the OA buffer so far (in terms of what's visible to the
237	* CPU).
238	*
239	* Although this can be observed explicitly while copying reports to userspace
240	* by checking for a zeroed report-id field in tail reports, we want to account
241	* for this earlier, as part of the oa_buffer_check_unlocked to avoid lots of
242	* redundant read() attempts.
243	*
244	* We workaround this issue in oa_buffer_check_unlocked() by reading the reports
245	* in the OA buffer, starting from the tail reported by the HW until we find a
246	* report with its first 2 dwords not 0 meaning its previous report is
247	* completely in memory and ready to be read. Those dwords are also set to 0
248	* once read and the whole buffer is cleared upon OA buffer initialization. The
249	* first dword is the reason for this report while the second is the timestamp,
250	* making the chances of having those 2 fields at 0 fairly unlikely. A more
251	* detailed explanation is available in oa_buffer_check_unlocked().
252	*
253	* Most of the implementation details for this workaround are in
254	* oa_buffer_check_unlocked() and _append_oa_reports()
255	*
256	* Note for posterity: previously the driver used to define an effective tail
257	* pointer that lagged the real pointer by a 'tail margin' measured in bytes
258	* derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency.
259	* This was flawed considering that the OA unit may also automatically generate
260	* non-periodic reports (such as on context switch) or the OA unit may be
261	* enabled without any periodic sampling.
262	*/
263	#define OA_TAIL_MARGIN_NSEC 100000ULL
264	#define INVALID_TAIL_PTR 0xffffffff
265
266	/* The default frequency for checking whether the OA unit has written new
267	* reports to the circular OA buffer...
268	*/
269	#define DEFAULT_POLL_FREQUENCY_HZ 200
270	#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
271
272	/* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */
273	static u32 i915_perf_stream_paranoid = true;
274
275	/* The maximum exponent the hardware accepts is 63 (essentially it selects one
276	* of the 64bit timestamp bits to trigger reports from) but there's currently
277	* no known use case for sampling as infrequently as once per 47 thousand years.
278	*
279	* Since the timestamps included in OA reports are only 32bits it seems
280	* reasonable to limit the OA exponent where it's still possible to account for
281	* overflow in OA report timestamps.
282	*/
283	#define OA_EXPONENT_MAX 31
284
285	#define INVALID_CTX_ID 0xffffffff
286
287	/* On Gen8+ automatically triggered OA reports include a 'reason' field... */
288	#define OAREPORT_REASON_MASK 0x3f
289	#define OAREPORT_REASON_MASK_EXTENDED 0x7f
290	#define OAREPORT_REASON_SHIFT 19
291	#define OAREPORT_REASON_TIMER (1<<0)
292	#define OAREPORT_REASON_CTX_SWITCH (1<<3)
293	#define OAREPORT_REASON_CLK_RATIO (1<<5)
294
295	#define HAS_MI_SET_PREDICATE(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
296
297	/* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
298	*
299	* The highest sampling frequency we can theoretically program the OA unit
300	* with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell.
301	*
302	* Initialized just before we register the sysctl parameter.
303	*/
304	static int oa_sample_rate_hard_limit;
305
306	/* Theoretically we can program the OA unit to sample every 160ns but don't
307	* allow that by default unless root...
308	*
309	* The default threshold of 100000Hz is based on perf's similar
310	* kernel.perf_event_max_sample_rate sysctl parameter.
311	*/
312	static u32 i915_oa_max_sample_rate = 100000;
313
314	/* XXX: beware if future OA HW adds new report formats that the current
315	* code assumes all reports have a power-of-two size and ~(size - 1) can
316	* be used as a mask to align the OA tail pointer.
317	*/
318	static const struct i915_oa_format oa_formats[I915_OA_FORMAT_MAX] = {
319	[I915_OA_FORMAT_A13] = { 0, 64 },
320	[I915_OA_FORMAT_A29] = { .format: 1, .size: 128 },
321	[I915_OA_FORMAT_A13_B8_C8] = { .format: 2, .size: 128 },
322	/* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */
323	[I915_OA_FORMAT_B4_C8] = { .format: 4, .size: 64 },
324	[I915_OA_FORMAT_A45_B8_C8] = { .format: 5, .size: 256 },
325	[I915_OA_FORMAT_B4_C8_A16] = { .format: 6, .size: 128 },
326	[I915_OA_FORMAT_C4_B8] = { .format: 7, .size: 64 },
327	[I915_OA_FORMAT_A12] = { .format: 0, .size: 64 },
328	[I915_OA_FORMAT_A12_B8_C8] = { .format: 2, .size: 128 },
329	[I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { .format: 5, .size: 256 },
330	[I915_OAR_FORMAT_A32u40_A4u32_B8_C8] = { .format: 5, .size: 256 },
331	[I915_OA_FORMAT_A24u40_A14u32_B8_C8] = { .format: 5, .size: 256 },
332	[I915_OAM_FORMAT_MPEC8u64_B8_C8] = { .format: 1, .size: 192, .type: TYPE_OAM, .header: HDR_64_BIT },
333	[I915_OAM_FORMAT_MPEC8u32_B8_C8] = { .format: 2, .size: 128, .type: TYPE_OAM, .header: HDR_64_BIT },
334	};
335
336	static const u32 mtl_oa_base[] = {
337	[PERF_GROUP_OAM_SAMEDIA_0] = 0x393000,
338	};
339
340	#define SAMPLE_OA_REPORT (1<<0)
341
342	/**
343	* struct perf_open_properties - for validated properties given to open a stream
344	* @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags
345	* @single_context: Whether a single or all gpu contexts should be monitored
346	* @hold_preemption: Whether the preemption is disabled for the filtered
347	* context
348	* @ctx_handle: A gem ctx handle for use with @single_context
349	* @metrics_set: An ID for an OA unit metric set advertised via sysfs
350	* @oa_format: An OA unit HW report format
351	* @oa_periodic: Whether to enable periodic OA unit sampling
352	* @oa_period_exponent: The OA unit sampling period is derived from this
353	* @engine: The engine (typically rcs0) being monitored by the OA unit
354	* @has_sseu: Whether @sseu was specified by userspace
355	* @sseu: internal SSEU configuration computed either from the userspace
356	* specified configuration in the opening parameters or a default value
357	* (see get_default_sseu_config())
358	* @poll_oa_period: The period in nanoseconds at which the CPU will check for OA
359	* data availability
360	*
361	* As read_properties_unlocked() enumerates and validates the properties given
362	* to open a stream of metrics the configuration is built up in the structure
363	* which starts out zero initialized.
364	*/
365	struct perf_open_properties {
366	u32 sample_flags;
367
368	u64 single_context:1;
369	u64 hold_preemption:1;
370	u64 ctx_handle;
371
372	/* OA sampling state */
373	int metrics_set;
374	int oa_format;
375	bool oa_periodic;
376	int oa_period_exponent;
377
378	struct intel_engine_cs *engine;
379
380	bool has_sseu;
381	struct intel_sseu sseu;
382
383	u64 poll_oa_period;
384	};
385
386	struct i915_oa_config_bo {
387	struct llist_node node;
388
389	struct i915_oa_config *oa_config;
390	struct i915_vma *vma;
391	};
392
393	static struct ctl_table_header *sysctl_header;
394
395	static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer);
396
397	void i915_oa_config_release(struct kref *ref)
398	{
399	struct i915_oa_config *oa_config =
400	container_of(ref, typeof(*oa_config), ref);
401
402	kfree(objp: oa_config->flex_regs);
403	kfree(objp: oa_config->b_counter_regs);
404	kfree(objp: oa_config->mux_regs);
405
406	kfree_rcu(oa_config, rcu);
407	}
408
409	struct i915_oa_config *
410	i915_perf_get_oa_config(struct i915_perf *perf, int metrics_set)
411	{
412	struct i915_oa_config *oa_config;
413
414	rcu_read_lock();
415	oa_config = idr_find(&perf->metrics_idr, id: metrics_set);
416	if (oa_config)
417	oa_config = i915_oa_config_get(oa_config);
418	rcu_read_unlock();
419
420	return oa_config;
421	}
422
423	static void free_oa_config_bo(struct i915_oa_config_bo *oa_bo)
424	{
425	i915_oa_config_put(oa_config: oa_bo->oa_config);
426	i915_vma_put(vma: oa_bo->vma);
427	kfree(objp: oa_bo);
428	}
429
430	static inline const
431	struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream)
432	{
433	return &stream->engine->oa_group->regs;
434	}
435
436	static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
437	{
438	struct intel_uncore *uncore = stream->uncore;
439
440	return intel_uncore_read(uncore, reg: __oa_regs(stream)->oa_tail_ptr) &
441	GEN12_OAG_OATAILPTR_MASK;
442	}
443
444	static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
445	{
446	struct intel_uncore *uncore = stream->uncore;
447
448	return intel_uncore_read(uncore, GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK;
449	}
450
451	static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
452	{
453	struct intel_uncore *uncore = stream->uncore;
454	u32 oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
455
456	return oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
457	}
458
459	#define oa_report_header_64bit(__s) \
460	((__s)->oa_buffer.format->header == HDR_64_BIT)
461
462	static u64 oa_report_id(struct i915_perf_stream *stream, void *report)
463	{
464	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
465	}
466
467	static u64 oa_report_reason(struct i915_perf_stream *stream, void *report)
468	{
469	return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) &
470	(GRAPHICS_VER(stream->perf->i915) == 12 ?
471	OAREPORT_REASON_MASK_EXTENDED :
472	OAREPORT_REASON_MASK);
473	}
474
475	static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report)
476	{
477	if (oa_report_header_64bit(stream))
478	*(u64 *)report = 0;
479	else
480	*report = 0;
481	}
482
483	static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report)
484	{
485	return !(oa_report_id(stream, report) &
486	stream->perf->gen8_valid_ctx_bit);
487	}
488
489	static u64 oa_timestamp(struct i915_perf_stream *stream, void *report)
490	{
491	return oa_report_header_64bit(stream) ?
492	*((u64 *)report + 1) :
493	*((u32 *)report + 1);
494	}
495
496	static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report)
497	{
498	if (oa_report_header_64bit(stream))
499	*(u64 *)&report[2] = 0;
500	else
501	report[1] = 0;
502	}
503
504	static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report)
505	{
506	u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2];
507
508	return ctx_id & stream->specific_ctx_id_mask;
509	}
510
511	static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report)
512	{
513	if (oa_report_header_64bit(stream))
514	report[4] = INVALID_CTX_ID;
515	else
516	report[2] = INVALID_CTX_ID;
517	}
518
519	/**
520	* oa_buffer_check_unlocked - check for data and update tail ptr state
521	* @stream: i915 stream instance
522	*
523	* This is either called via fops (for blocking reads in user ctx) or the poll
524	* check hrtimer (atomic ctx) to check the OA buffer tail pointer and check
525	* if there is data available for userspace to read.
526	*
527	* This function is central to providing a workaround for the OA unit tail
528	* pointer having a race with respect to what data is visible to the CPU.
529	* It is responsible for reading tail pointers from the hardware and giving
530	* the pointers time to 'age' before they are made available for reading.
531	* (See description of OA_TAIL_MARGIN_NSEC above for further details.)
532	*
533	* Besides returning true when there is data available to read() this function
534	* also updates the tail in the oa_buffer object.
535	*
536	* Note: It's safe to read OA config state here unlocked, assuming that this is
537	* only called while the stream is enabled, while the global OA configuration
538	* can't be modified.
539	*
540	* Returns: %true if the OA buffer contains data, else %false
541	*/
542	static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
543	{
544	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
545	int report_size = stream->oa_buffer.format->size;
546	u32 tail, hw_tail;
547	unsigned long flags;
548	bool pollin;
549	u32 partial_report_size;
550
551	/* We have to consider the (unlikely) possibility that read() errors
552	* could result in an OA buffer reset which might reset the head and
553	* tail state.
554	*/
555	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
556
557	hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
558	hw_tail -= gtt_offset;
559
560	/* The tail pointer increases in 64 byte increments, not in report_size
561	* steps. Also the report size may not be a power of 2. Compute
562	* potentially partially landed report in the OA buffer
563	*/
564	partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail);
565	partial_report_size %= report_size;
566
567	/* Subtract partial amount off the tail */
568	hw_tail = OA_TAKEN(hw_tail, partial_report_size);
569
570	tail = hw_tail;
571
572	/* Walk the stream backward until we find a report with report
573	* id and timestmap not at 0. Since the circular buffer pointers
574	* progress by increments of 64 bytes and that reports can be up
575	* to 256 bytes long, we can't tell whether a report has fully
576	* landed in memory before the report id and timestamp of the
577	* following report have effectively landed.
578	*
579	* This is assuming that the writes of the OA unit land in
580	* memory in the order they were written to.
581	* If not : (╯°□°）╯︵ ┻━┻
582	*/
583	while (OA_TAKEN(tail, stream->oa_buffer.tail) >= report_size) {
584	void *report = stream->oa_buffer.vaddr + tail;
585
586	if (oa_report_id(stream, report) ||
587	oa_timestamp(stream, report))
588	break;
589
590	tail = (tail - report_size) & (OA_BUFFER_SIZE - 1);
591	}
592
593	if (OA_TAKEN(hw_tail, tail) > report_size &&
594	__ratelimit(&stream->perf->tail_pointer_race))
595	drm_notice(&stream->uncore->i915->drm,
596	"unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n",
597	stream->oa_buffer.head, tail, hw_tail);
598
599	stream->oa_buffer.tail = tail;
600
601	pollin = OA_TAKEN(stream->oa_buffer.tail,
602	stream->oa_buffer.head) >= report_size;
603
604	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
605
606	return pollin;
607	}
608
609	/**
610	* append_oa_status - Appends a status record to a userspace read() buffer.
611	* @stream: An i915-perf stream opened for OA metrics
612	* @buf: destination buffer given by userspace
613	* @count: the number of bytes userspace wants to read
614	* @offset: (inout): the current position for writing into @buf
615	* @type: The kind of status to report to userspace
616	*
617	* Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`)
618	* into the userspace read() buffer.
619	*
620	* The @buf @offset will only be updated on success.
621	*
622	* Returns: 0 on success, negative error code on failure.
623	*/
624	static int append_oa_status(struct i915_perf_stream *stream,
625	char __user *buf,
626	size_t count,
627	size_t *offset,
628	enum drm_i915_perf_record_type type)
629	{
630	struct drm_i915_perf_record_header header = { type, 0, sizeof(header) };
631
632	if ((count - *offset) < header.size)
633	return -ENOSPC;
634
635	if (copy_to_user(to: buf + *offset, from: &header, n: sizeof(header)))
636	return -EFAULT;
637
638	(*offset) += header.size;
639
640	return 0;
641	}
642
643	/**
644	* append_oa_sample - Copies single OA report into userspace read() buffer.
645	* @stream: An i915-perf stream opened for OA metrics
646	* @buf: destination buffer given by userspace
647	* @count: the number of bytes userspace wants to read
648	* @offset: (inout): the current position for writing into @buf
649	* @report: A single OA report to (optionally) include as part of the sample
650	*
651	* The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*`
652	* properties when opening a stream, tracked as `stream->sample_flags`. This
653	* function copies the requested components of a single sample to the given
654	* read() @buf.
655	*
656	* The @buf @offset will only be updated on success.
657	*
658	* Returns: 0 on success, negative error code on failure.
659	*/
660	static int append_oa_sample(struct i915_perf_stream *stream,
661	char __user *buf,
662	size_t count,
663	size_t *offset,
664	const u8 *report)
665	{
666	int report_size = stream->oa_buffer.format->size;
667	struct drm_i915_perf_record_header header;
668	int report_size_partial;
669	u8 *oa_buf_end;
670
671	header.type = DRM_I915_PERF_RECORD_SAMPLE;
672	header.pad = 0;
673	header.size = stream->sample_size;
674
675	if ((count - *offset) < header.size)
676	return -ENOSPC;
677
678	buf += *offset;
679	if (copy_to_user(to: buf, from: &header, n: sizeof(header)))
680	return -EFAULT;
681	buf += sizeof(header);
682
683	oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE;
684	report_size_partial = oa_buf_end - report;
685
686	if (report_size_partial < report_size) {
687	if (copy_to_user(to: buf, from: report, n: report_size_partial))
688	return -EFAULT;
689	buf += report_size_partial;
690
691	if (copy_to_user(to: buf, from: stream->oa_buffer.vaddr,
692	n: report_size - report_size_partial))
693	return -EFAULT;
694	} else if (copy_to_user(to: buf, from: report, n: report_size)) {
695	return -EFAULT;
696	}
697
698	(*offset) += header.size;
699
700	return 0;
701	}
702
703	/**
704	* gen8_append_oa_reports - Copies all buffered OA reports into
705	* userspace read() buffer.
706	* @stream: An i915-perf stream opened for OA metrics
707	* @buf: destination buffer given by userspace
708	* @count: the number of bytes userspace wants to read
709	* @offset: (inout): the current position for writing into @buf
710	*
711	* Notably any error condition resulting in a short read (-%ENOSPC or
712	* -%EFAULT) will be returned even though one or more records may
713	* have been successfully copied. In this case it's up to the caller
714	* to decide if the error should be squashed before returning to
715	* userspace.
716	*
717	* Note: reports are consumed from the head, and appended to the
718	* tail, so the tail chases the head?... If you think that's mad
719	* and back-to-front you're not alone, but this follows the
720	* Gen PRM naming convention.
721	*
722	* Returns: 0 on success, negative error code on failure.
723	*/
724	static int gen8_append_oa_reports(struct i915_perf_stream *stream,
725	char __user *buf,
726	size_t count,
727	size_t *offset)
728	{
729	struct intel_uncore *uncore = stream->uncore;
730	int report_size = stream->oa_buffer.format->size;
731	u8 *oa_buf_base = stream->oa_buffer.vaddr;
732	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
733	u32 mask = (OA_BUFFER_SIZE - 1);
734	size_t start_offset = *offset;
735	unsigned long flags;
736	u32 head, tail;
737	int ret = 0;
738
739	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
740	return -EIO;
741
742	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
743
744	head = stream->oa_buffer.head;
745	tail = stream->oa_buffer.tail;
746
747	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
748
749	/*
750	* An out of bounds or misaligned head or tail pointer implies a driver
751	* bug since we validate + align the tail pointers we read from the
752	* hardware and we are in full control of the head pointer which should
753	* only be incremented by multiples of the report size.
754	*/
755	if (drm_WARN_ONCE(&uncore->i915->drm,
756	head > OA_BUFFER_SIZE ||
757	tail > OA_BUFFER_SIZE,
758	"Inconsistent OA buffer pointers: head = %u, tail = %u\n",
759	head, tail))
760	return -EIO;
761
762
763	for (/* none */;
764	OA_TAKEN(tail, head);
765	head = (head + report_size) & mask) {
766	u8 *report = oa_buf_base + head;
767	u32 *report32 = (void *)report;
768	u32 ctx_id;
769	u64 reason;
770
771	/*
772	* The reason field includes flags identifying what
773	* triggered this specific report (mostly timer
774	* triggered or e.g. due to a context switch).
775	*
776	* In MMIO triggered reports, some platforms do not set the
777	* reason bit in this field and it is valid to have a reason
778	* field of zero.
779	*/
780	reason = oa_report_reason(stream, report);
781	ctx_id = oa_context_id(stream, report: report32);
782
783	/*
784	* Squash whatever is in the CTX_ID field if it's marked as
785	* invalid to be sure we avoid false-positive, single-context
786	* filtering below...
787	*
788	* Note: that we don't clear the valid_ctx_bit so userspace can
789	* understand that the ID has been squashed by the kernel.
790	*/
791	if (oa_report_ctx_invalid(stream, report)) {
792	ctx_id = INVALID_CTX_ID;
793	oa_context_id_squash(stream, report: report32);
794	}
795
796	/*
797	* NB: For Gen 8 the OA unit no longer supports clock gating
798	* off for a specific context and the kernel can't securely
799	* stop the counters from updating as system-wide / global
800	* values.
801	*
802	* Automatic reports now include a context ID so reports can be
803	* filtered on the cpu but it's not worth trying to
804	* automatically subtract/hide counter progress for other
805	* contexts while filtering since we can't stop userspace
806	* issuing MI_REPORT_PERF_COUNT commands which would still
807	* provide a side-band view of the real values.
808	*
809	* To allow userspace (such as Mesa/GL_INTEL_performance_query)
810	* to normalize counters for a single filtered context then it
811	* needs be forwarded bookend context-switch reports so that it
812	* can track switches in between MI_REPORT_PERF_COUNT commands
813	* and can itself subtract/ignore the progress of counters
814	* associated with other contexts. Note that the hardware
815	* automatically triggers reports when switching to a new
816	* context which are tagged with the ID of the newly active
817	* context. To avoid the complexity (and likely fragility) of
818	* reading ahead while parsing reports to try and minimize
819	* forwarding redundant context switch reports (i.e. between
820	* other, unrelated contexts) we simply elect to forward them
821	* all.
822	*
823	* We don't rely solely on the reason field to identify context
824	* switches since it's not-uncommon for periodic samples to
825	* identify a switch before any 'context switch' report.
826	*/
827	if (!stream->ctx ||
828	stream->specific_ctx_id == ctx_id ||
829	stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
830	reason & OAREPORT_REASON_CTX_SWITCH) {
831
832	/*
833	* While filtering for a single context we avoid
834	* leaking the IDs of other contexts.
835	*/
836	if (stream->ctx &&
837	stream->specific_ctx_id != ctx_id) {
838	oa_context_id_squash(stream, report: report32);
839	}
840
841	ret = append_oa_sample(stream, buf, count, offset,
842	report);
843	if (ret)
844	break;
845
846	stream->oa_buffer.last_ctx_id = ctx_id;
847	}
848
849	if (is_power_of_2(n: report_size)) {
850	/*
851	* Clear out the report id and timestamp as a means
852	* to detect unlanded reports.
853	*/
854	oa_report_id_clear(stream, report: report32);
855	oa_timestamp_clear(stream, report: report32);
856	} else {
857	u8 *oa_buf_end = stream->oa_buffer.vaddr +
858	OA_BUFFER_SIZE;
859	u32 part = oa_buf_end - (u8 *)report32;
860
861	/* Zero out the entire report */
862	if (report_size <= part) {
863	memset(report32, 0, report_size);
864	} else {
865	memset(report32, 0, part);
866	memset(oa_buf_base, 0, report_size - part);
867	}
868	}
869	}
870
871	if (start_offset != *offset) {
872	i915_reg_t oaheadptr;
873
874	oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?
875	__oa_regs(stream)->oa_head_ptr :
876	GEN8_OAHEADPTR;
877
878	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
879
880	/*
881	* We removed the gtt_offset for the copy loop above, indexing
882	* relative to oa_buf_base so put back here...
883	*/
884	intel_uncore_write(uncore, reg: oaheadptr,
885	val: (head + gtt_offset) & GEN12_OAG_OAHEADPTR_MASK);
886	stream->oa_buffer.head = head;
887
888	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
889	}
890
891	return ret;
892	}
893
894	/**
895	* gen8_oa_read - copy status records then buffered OA reports
896	* @stream: An i915-perf stream opened for OA metrics
897	* @buf: destination buffer given by userspace
898	* @count: the number of bytes userspace wants to read
899	* @offset: (inout): the current position for writing into @buf
900	*
901	* Checks OA unit status registers and if necessary appends corresponding
902	* status records for userspace (such as for a buffer full condition) and then
903	* initiate appending any buffered OA reports.
904	*
905	* Updates @offset according to the number of bytes successfully copied into
906	* the userspace buffer.
907	*
908	* NB: some data may be successfully copied to the userspace buffer
909	* even if an error is returned, and this is reflected in the
910	* updated @offset.
911	*
912	* Returns: zero on success or a negative error code
913	*/
914	static int gen8_oa_read(struct i915_perf_stream *stream,
915	char __user *buf,
916	size_t count,
917	size_t *offset)
918	{
919	struct intel_uncore *uncore = stream->uncore;
920	u32 oastatus;
921	i915_reg_t oastatus_reg;
922	int ret;
923
924	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
925	return -EIO;
926
927	oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?
928	__oa_regs(stream)->oa_status :
929	GEN8_OASTATUS;
930
931	oastatus = intel_uncore_read(uncore, reg: oastatus_reg);
932
933	/*
934	* We treat OABUFFER_OVERFLOW as a significant error:
935	*
936	* Although theoretically we could handle this more gracefully
937	* sometimes, some Gens don't correctly suppress certain
938	* automatically triggered reports in this condition and so we
939	* have to assume that old reports are now being trampled
940	* over.
941	*
942	* Considering how we don't currently give userspace control
943	* over the OA buffer size and always configure a large 16MB
944	* buffer, then a buffer overflow does anyway likely indicate
945	* that something has gone quite badly wrong.
946	*/
947	if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) {
948	ret = append_oa_status(stream, buf, count, offset,
949	type: DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
950	if (ret)
951	return ret;
952
953	drm_dbg(&stream->perf->i915->drm,
954	"OA buffer overflow (exponent = %d): force restart\n",
955	stream->period_exponent);
956
957	stream->perf->ops.oa_disable(stream);
958	stream->perf->ops.oa_enable(stream);
959
960	/*
961	* Note: .oa_enable() is expected to re-init the oabuffer and
962	* reset GEN8_OASTATUS for us
963	*/
964	oastatus = intel_uncore_read(uncore, reg: oastatus_reg);
965	}
966
967	if (oastatus & GEN8_OASTATUS_REPORT_LOST) {
968	ret = append_oa_status(stream, buf, count, offset,
969	type: DRM_I915_PERF_RECORD_OA_REPORT_LOST);
970	if (ret)
971	return ret;
972
973	intel_uncore_rmw(uncore, reg: oastatus_reg,
974	GEN8_OASTATUS_COUNTER_OVERFLOW |
975	GEN8_OASTATUS_REPORT_LOST,
976	IS_GRAPHICS_VER(uncore->i915, 8, 11) ?
977	(GEN8_OASTATUS_HEAD_POINTER_WRAP |
978	GEN8_OASTATUS_TAIL_POINTER_WRAP) : 0);
979	}
980
981	return gen8_append_oa_reports(stream, buf, count, offset);
982	}
983
984	/**
985	* gen7_append_oa_reports - Copies all buffered OA reports into
986	* userspace read() buffer.
987	* @stream: An i915-perf stream opened for OA metrics
988	* @buf: destination buffer given by userspace
989	* @count: the number of bytes userspace wants to read
990	* @offset: (inout): the current position for writing into @buf
991	*
992	* Notably any error condition resulting in a short read (-%ENOSPC or
993	* -%EFAULT) will be returned even though one or more records may
994	* have been successfully copied. In this case it's up to the caller
995	* to decide if the error should be squashed before returning to
996	* userspace.
997	*
998	* Note: reports are consumed from the head, and appended to the
999	* tail, so the tail chases the head?... If you think that's mad
1000	* and back-to-front you're not alone, but this follows the
1001	* Gen PRM naming convention.
1002	*
1003	* Returns: 0 on success, negative error code on failure.
1004	*/
1005	static int gen7_append_oa_reports(struct i915_perf_stream *stream,
1006	char __user *buf,
1007	size_t count,
1008	size_t *offset)
1009	{
1010	struct intel_uncore *uncore = stream->uncore;
1011	int report_size = stream->oa_buffer.format->size;
1012	u8 *oa_buf_base = stream->oa_buffer.vaddr;
1013	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1014	u32 mask = (OA_BUFFER_SIZE - 1);
1015	size_t start_offset = *offset;
1016	unsigned long flags;
1017	u32 head, tail;
1018	int ret = 0;
1019
1020	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
1021	return -EIO;
1022
1023	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1024
1025	head = stream->oa_buffer.head;
1026	tail = stream->oa_buffer.tail;
1027
1028	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1029
1030	/* An out of bounds or misaligned head or tail pointer implies a driver
1031	* bug since we validate + align the tail pointers we read from the
1032	* hardware and we are in full control of the head pointer which should
1033	* only be incremented by multiples of the report size (notably also
1034	* all a power of two).
1035	*/
1036	if (drm_WARN_ONCE(&uncore->i915->drm,
1037	head > OA_BUFFER_SIZE || head % report_size ||
1038	tail > OA_BUFFER_SIZE || tail % report_size,
1039	"Inconsistent OA buffer pointers: head = %u, tail = %u\n",
1040	head, tail))
1041	return -EIO;
1042
1043
1044	for (/* none */;
1045	OA_TAKEN(tail, head);
1046	head = (head + report_size) & mask) {
1047	u8 *report = oa_buf_base + head;
1048	u32 *report32 = (void *)report;
1049
1050	/* All the report sizes factor neatly into the buffer
1051	* size so we never expect to see a report split
1052	* between the beginning and end of the buffer.
1053	*
1054	* Given the initial alignment check a misalignment
1055	* here would imply a driver bug that would result
1056	* in an overrun.
1057	*/
1058	if (drm_WARN_ON(&uncore->i915->drm,
1059	(OA_BUFFER_SIZE - head) < report_size)) {
1060	drm_err(&uncore->i915->drm,
1061	"Spurious OA head ptr: non-integral report offset\n");
1062	break;
1063	}
1064
1065	/* The report-ID field for periodic samples includes
1066	* some undocumented flags related to what triggered
1067	* the report and is never expected to be zero so we
1068	* can check that the report isn't invalid before
1069	* copying it to userspace...
1070	*/
1071	if (report32[0] == 0) {
1072	if (__ratelimit(&stream->perf->spurious_report_rs))
1073	drm_notice(&uncore->i915->drm,
1074	"Skipping spurious, invalid OA report\n");
1075	continue;
1076	}
1077
1078	ret = append_oa_sample(stream, buf, count, offset, report);
1079	if (ret)
1080	break;
1081
1082	/* Clear out the first 2 dwords as a mean to detect unlanded
1083	* reports.
1084	*/
1085	report32[0] = 0;
1086	report32[1] = 0;
1087	}
1088
1089	if (start_offset != *offset) {
1090	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1091
1092	intel_uncore_write(uncore, GEN7_OASTATUS2,
1093	val: ((head + gtt_offset) & GEN7_OASTATUS2_HEAD_MASK) |
1094	GEN7_OASTATUS2_MEM_SELECT_GGTT);
1095	stream->oa_buffer.head = head;
1096
1097	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1098	}
1099
1100	return ret;
1101	}
1102
1103	/**
1104	* gen7_oa_read - copy status records then buffered OA reports
1105	* @stream: An i915-perf stream opened for OA metrics
1106	* @buf: destination buffer given by userspace
1107	* @count: the number of bytes userspace wants to read
1108	* @offset: (inout): the current position for writing into @buf
1109	*
1110	* Checks Gen 7 specific OA unit status registers and if necessary appends
1111	* corresponding status records for userspace (such as for a buffer full
1112	* condition) and then initiate appending any buffered OA reports.
1113	*
1114	* Updates @offset according to the number of bytes successfully copied into
1115	* the userspace buffer.
1116	*
1117	* Returns: zero on success or a negative error code
1118	*/
1119	static int gen7_oa_read(struct i915_perf_stream *stream,
1120	char __user *buf,
1121	size_t count,
1122	size_t *offset)
1123	{
1124	struct intel_uncore *uncore = stream->uncore;
1125	u32 oastatus1;
1126	int ret;
1127
1128	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1129	return -EIO;
1130
1131	oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1132
1133	/* XXX: On Haswell we don't have a safe way to clear oastatus1
1134	* bits while the OA unit is enabled (while the tail pointer
1135	* may be updated asynchronously) so we ignore status bits
1136	* that have already been reported to userspace.
1137	*/
1138	oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1139
1140	/* We treat OABUFFER_OVERFLOW as a significant error:
1141	*
1142	* - The status can be interpreted to mean that the buffer is
1143	* currently full (with a higher precedence than OA_TAKEN()
1144	* which will start to report a near-empty buffer after an
1145	* overflow) but it's awkward that we can't clear the status
1146	* on Haswell, so without a reset we won't be able to catch
1147	* the state again.
1148	*
1149	* - Since it also implies the HW has started overwriting old
1150	* reports it may also affect our sanity checks for invalid
1151	* reports when copying to userspace that assume new reports
1152	* are being written to cleared memory.
1153	*
1154	* - In the future we may want to introduce a flight recorder
1155	* mode where the driver will automatically maintain a safe
1156	* guard band between head/tail, avoiding this overflow
1157	* condition, but we avoid the added driver complexity for
1158	* now.
1159	*/
1160	if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) {
1161	ret = append_oa_status(stream, buf, count, offset,
1162	type: DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
1163	if (ret)
1164	return ret;
1165
1166	drm_dbg(&stream->perf->i915->drm,
1167	"OA buffer overflow (exponent = %d): force restart\n",
1168	stream->period_exponent);
1169
1170	stream->perf->ops.oa_disable(stream);
1171	stream->perf->ops.oa_enable(stream);
1172
1173	oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1174	}
1175
1176	if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) {
1177	ret = append_oa_status(stream, buf, count, offset,
1178	type: DRM_I915_PERF_RECORD_OA_REPORT_LOST);
1179	if (ret)
1180	return ret;
1181	stream->perf->gen7_latched_oastatus1 |=
1182	GEN7_OASTATUS1_REPORT_LOST;
1183	}
1184
1185	return gen7_append_oa_reports(stream, buf, count, offset);
1186	}
1187
1188	/**
1189	* i915_oa_wait_unlocked - handles blocking IO until OA data available
1190	* @stream: An i915-perf stream opened for OA metrics
1191	*
1192	* Called when userspace tries to read() from a blocking stream FD opened
1193	* for OA metrics. It waits until the hrtimer callback finds a non-empty
1194	* OA buffer and wakes us.
1195	*
1196	* Note: it's acceptable to have this return with some false positives
1197	* since any subsequent read handling will return -EAGAIN if there isn't
1198	* really data ready for userspace yet.
1199	*
1200	* Returns: zero on success or a negative error code
1201	*/
1202	static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1203	{
1204	/* We would wait indefinitely if periodic sampling is not enabled */
1205	if (!stream->periodic)
1206	return -EIO;
1207
1208	return wait_event_interruptible(stream->poll_wq,
1209	oa_buffer_check_unlocked(stream));
1210	}
1211
1212	/**
1213	* i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1214	* @stream: An i915-perf stream opened for OA metrics
1215	* @file: An i915 perf stream file
1216	* @wait: poll() state table
1217	*
1218	* For handling userspace polling on an i915 perf stream opened for OA metrics,
1219	* this starts a poll_wait with the wait queue that our hrtimer callback wakes
1220	* when it sees data ready to read in the circular OA buffer.
1221	*/
1222	static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1223	struct file *file,
1224	poll_table *wait)
1225	{
1226	poll_wait(filp: file, wait_address: &stream->poll_wq, p: wait);
1227	}
1228
1229	/**
1230	* i915_oa_read - just calls through to &i915_oa_ops->read
1231	* @stream: An i915-perf stream opened for OA metrics
1232	* @buf: destination buffer given by userspace
1233	* @count: the number of bytes userspace wants to read
1234	* @offset: (inout): the current position for writing into @buf
1235	*
1236	* Updates @offset according to the number of bytes successfully copied into
1237	* the userspace buffer.
1238	*
1239	* Returns: zero on success or a negative error code
1240	*/
1241	static int i915_oa_read(struct i915_perf_stream *stream,
1242	char __user *buf,
1243	size_t count,
1244	size_t *offset)
1245	{
1246	return stream->perf->ops.read(stream, buf, count, offset);
1247	}
1248
1249	static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1250	{
1251	struct i915_gem_engines_iter it;
1252	struct i915_gem_context *ctx = stream->ctx;
1253	struct intel_context *ce;
1254	struct i915_gem_ww_ctx ww;
1255	int err = -ENODEV;
1256
1257	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1258	if (ce->engine != stream->engine) /* first match! */
1259	continue;
1260
1261	err = 0;
1262	break;
1263	}
1264	i915_gem_context_unlock_engines(ctx);
1265
1266	if (err)
1267	return ERR_PTR(error: err);
1268
1269	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
1270	retry:
1271	/*
1272	* As the ID is the gtt offset of the context's vma we
1273	* pin the vma to ensure the ID remains fixed.
1274	*/
1275	err = intel_context_pin_ww(ce, ww: &ww);
1276	if (err == -EDEADLK) {
1277	err = i915_gem_ww_ctx_backoff(ctx: &ww);
1278	if (!err)
1279	goto retry;
1280	}
1281	i915_gem_ww_ctx_fini(ctx: &ww);
1282
1283	if (err)
1284	return ERR_PTR(error: err);
1285
1286	stream->pinned_ctx = ce;
1287	return stream->pinned_ctx;
1288	}
1289
1290	static int
1291	__store_reg_to_mem(struct i915_request *rq, i915_reg_t reg, u32 ggtt_offset)
1292	{
1293	u32 *cs, cmd;
1294
1295	cmd = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1296	if (GRAPHICS_VER(rq->i915) >= 8)
1297	cmd++;
1298
1299	cs = intel_ring_begin(rq, num_dwords: 4);
1300	if (IS_ERR(ptr: cs))
1301	return PTR_ERR(ptr: cs);
1302
1303	*cs++ = cmd;
1304	*cs++ = i915_mmio_reg_offset(reg);
1305	*cs++ = ggtt_offset;
1306	*cs++ = 0;
1307
1308	intel_ring_advance(rq, cs);
1309
1310	return 0;
1311	}
1312
1313	static int
1314	__read_reg(struct intel_context *ce, i915_reg_t reg, u32 ggtt_offset)
1315	{
1316	struct i915_request *rq;
1317	int err;
1318
1319	rq = i915_request_create(ce);
1320	if (IS_ERR(ptr: rq))
1321	return PTR_ERR(ptr: rq);
1322
1323	i915_request_get(rq);
1324
1325	err = __store_reg_to_mem(rq, reg, ggtt_offset);
1326
1327	i915_request_add(rq);
1328	if (!err && i915_request_wait(rq, flags: 0, HZ / 2) < 0)
1329	err = -ETIME;
1330
1331	i915_request_put(rq);
1332
1333	return err;
1334	}
1335
1336	static int
1337	gen12_guc_sw_ctx_id(struct intel_context *ce, u32 *ctx_id)
1338	{
1339	struct i915_vma *scratch;
1340	u32 *val;
1341	int err;
1342
1343	scratch = __vm_create_scratch_for_read_pinned(vm: &ce->engine->gt->ggtt->vm, size: 4);
1344	if (IS_ERR(ptr: scratch))
1345	return PTR_ERR(ptr: scratch);
1346
1347	err = i915_vma_sync(vma: scratch);
1348	if (err)
1349	goto err_scratch;
1350
1351	err = __read_reg(ce, RING_EXECLIST_STATUS_HI(ce->engine->mmio_base),
1352	ggtt_offset: i915_ggtt_offset(vma: scratch));
1353	if (err)
1354	goto err_scratch;
1355
1356	val = i915_gem_object_pin_map_unlocked(obj: scratch->obj, type: I915_MAP_WB);
1357	if (IS_ERR(ptr: val)) {
1358	err = PTR_ERR(ptr: val);
1359	goto err_scratch;
1360	}
1361
1362	*ctx_id = *val;
1363	i915_gem_object_unpin_map(obj: scratch->obj);
1364
1365	err_scratch:
1366	i915_vma_unpin_and_release(p_vma: &scratch, flags: 0);
1367	return err;
1368	}
1369
1370	/*
1371	* For execlist mode of submission, pick an unused context id
1372	* 0 - (NUM_CONTEXT_TAG -1) are used by other contexts
1373	* XXX_MAX_CONTEXT_HW_ID is used by idle context
1374	*
1375	* For GuC mode of submission read context id from the upper dword of the
1376	* EXECLIST_STATUS register. Note that we read this value only once and expect
1377	* that the value stays fixed for the entire OA use case. There are cases where
1378	* GuC KMD implementation may deregister a context to reuse it's context id, but
1379	* we prevent that from happening to the OA context by pinning it.
1380	*/
1381	static int gen12_get_render_context_id(struct i915_perf_stream *stream)
1382	{
1383	u32 ctx_id, mask;
1384	int ret;
1385
1386	if (intel_engine_uses_guc(engine: stream->engine)) {
1387	ret = gen12_guc_sw_ctx_id(ce: stream->pinned_ctx, ctx_id: &ctx_id);
1388	if (ret)
1389	return ret;
1390
1391	mask = ((1U << GEN12_GUC_SW_CTX_ID_WIDTH) - 1) <<
1392	(GEN12_GUC_SW_CTX_ID_SHIFT - 32);
1393	} else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) {
1394	ctx_id = (XEHP_MAX_CONTEXT_HW_ID - 1) <<
1395	(XEHP_SW_CTX_ID_SHIFT - 32);
1396
1397	mask = ((1U << XEHP_SW_CTX_ID_WIDTH) - 1) <<
1398	(XEHP_SW_CTX_ID_SHIFT - 32);
1399	} else {
1400	ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) <<
1401	(GEN11_SW_CTX_ID_SHIFT - 32);
1402
1403	mask = ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) <<
1404	(GEN11_SW_CTX_ID_SHIFT - 32);
1405	}
1406	stream->specific_ctx_id = ctx_id & mask;
1407	stream->specific_ctx_id_mask = mask;
1408
1409	return 0;
1410	}
1411
1412	static bool oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end)
1413	{
1414	u32 idx = *offset;
1415	u32 len = min(MI_LRI_LEN(state[idx]) + idx, end);
1416	bool found = false;
1417
1418	idx++;
1419	for (; idx < len; idx += 2) {
1420	if (state[idx] == reg) {
1421	found = true;
1422	break;
1423	}
1424	}
1425
1426	*offset = idx;
1427	return found;
1428	}
1429
1430	static u32 oa_context_image_offset(struct intel_context *ce, u32 reg)
1431	{
1432	u32 offset, len = (ce->engine->context_size - PAGE_SIZE) / 4;
1433	u32 *state = ce->lrc_reg_state;
1434
1435	if (drm_WARN_ON(&ce->engine->i915->drm, !state))
1436	return U32_MAX;
1437
1438	for (offset = 0; offset < len; ) {
1439	if (IS_MI_LRI_CMD(state[offset])) {
1440	/*
1441	* We expect reg-value pairs in MI_LRI command, so
1442	* MI_LRI_LEN() should be even, if not, issue a warning.
1443	*/
1444	drm_WARN_ON(&ce->engine->i915->drm,
1445	MI_LRI_LEN(state[offset]) & 0x1);
1446
1447	if (oa_find_reg_in_lri(state, reg, offset: &offset, end: len))
1448	break;
1449	} else {
1450	offset++;
1451	}
1452	}
1453
1454	return offset < len ? offset : U32_MAX;
1455	}
1456
1457	static int set_oa_ctx_ctrl_offset(struct intel_context *ce)
1458	{
1459	i915_reg_t reg = GEN12_OACTXCONTROL(ce->engine->mmio_base);
1460	struct i915_perf *perf = &ce->engine->i915->perf;
1461	u32 offset = perf->ctx_oactxctrl_offset;
1462
1463	/* Do this only once. Failure is stored as offset of U32_MAX */
1464	if (offset)
1465	goto exit;
1466
1467	offset = oa_context_image_offset(ce, i915_mmio_reg_offset(reg));
1468	perf->ctx_oactxctrl_offset = offset;
1469
1470	drm_dbg(&ce->engine->i915->drm,
1471	"%s oa ctx control at 0x%08x dword offset\n",
1472	ce->engine->name, offset);
1473
1474	exit:
1475	return offset && offset != U32_MAX ? 0 : -ENODEV;
1476	}
1477
1478	static bool engine_supports_mi_query(struct intel_engine_cs *engine)
1479	{
1480	return engine->class == RENDER_CLASS;
1481	}
1482
1483	/**
1484	* oa_get_render_ctx_id - determine and hold ctx hw id
1485	* @stream: An i915-perf stream opened for OA metrics
1486	*
1487	* Determine the render context hw id, and ensure it remains fixed for the
1488	* lifetime of the stream. This ensures that we don't have to worry about
1489	* updating the context ID in OACONTROL on the fly.
1490	*
1491	* Returns: zero on success or a negative error code
1492	*/
1493	static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1494	{
1495	struct intel_context *ce;
1496	int ret = 0;
1497
1498	ce = oa_pin_context(stream);
1499	if (IS_ERR(ptr: ce))
1500	return PTR_ERR(ptr: ce);
1501
1502	if (engine_supports_mi_query(engine: stream->engine) &&
1503	HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) {
1504	/*
1505	* We are enabling perf query here. If we don't find the context
1506	* offset here, just return an error.
1507	*/
1508	ret = set_oa_ctx_ctrl_offset(ce);
1509	if (ret) {
1510	intel_context_unpin(ce);
1511	drm_err(&stream->perf->i915->drm,
1512	"Enabling perf query failed for %s\n",
1513	stream->engine->name);
1514	return ret;
1515	}
1516	}
1517
1518	switch (GRAPHICS_VER(ce->engine->i915)) {
1519	case 7: {
1520	/*
1521	* On Haswell we don't do any post processing of the reports
1522	* and don't need to use the mask.
1523	*/
1524	stream->specific_ctx_id = i915_ggtt_offset(vma: ce->state);
1525	stream->specific_ctx_id_mask = 0;
1526	break;
1527	}
1528
1529	case 8:
1530	case 9:
1531	if (intel_engine_uses_guc(engine: ce->engine)) {
1532	/*
1533	* When using GuC, the context descriptor we write in
1534	* i915 is read by GuC and rewritten before it's
1535	* actually written into the hardware. The LRCA is
1536	* what is put into the context id field of the
1537	* context descriptor by GuC. Because it's aligned to
1538	* a page, the lower 12bits are always at 0 and
1539	* dropped by GuC. They won't be part of the context
1540	* ID in the OA reports, so squash those lower bits.
1541	*/
1542	stream->specific_ctx_id = ce->lrc.lrca >> 12;
1543
1544	/*
1545	* GuC uses the top bit to signal proxy submission, so
1546	* ignore that bit.
1547	*/
1548	stream->specific_ctx_id_mask =
1549	(1U << (GEN8_CTX_ID_WIDTH - 1)) - 1;
1550	} else {
1551	stream->specific_ctx_id_mask =
1552	(1U << GEN8_CTX_ID_WIDTH) - 1;
1553	stream->specific_ctx_id = stream->specific_ctx_id_mask;
1554	}
1555	break;
1556
1557	case 11:
1558	case 12:
1559	ret = gen12_get_render_context_id(stream);
1560	break;
1561
1562	default:
1563	MISSING_CASE(GRAPHICS_VER(ce->engine->i915));
1564	}
1565
1566	ce->tag = stream->specific_ctx_id;
1567
1568	drm_dbg(&stream->perf->i915->drm,
1569	"filtering on ctx_id=0x%x ctx_id_mask=0x%x\n",
1570	stream->specific_ctx_id,
1571	stream->specific_ctx_id_mask);
1572
1573	return ret;
1574	}
1575
1576	/**
1577	* oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1578	* @stream: An i915-perf stream opened for OA metrics
1579	*
1580	* In case anything needed doing to ensure the context HW ID would remain valid
1581	* for the lifetime of the stream, then that can be undone here.
1582	*/
1583	static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1584	{
1585	struct intel_context *ce;
1586
1587	ce = fetch_and_zero(&stream->pinned_ctx);
1588	if (ce) {
1589	ce->tag = 0; /* recomputed on next submission after parking */
1590	intel_context_unpin(ce);
1591	}
1592
1593	stream->specific_ctx_id = INVALID_CTX_ID;
1594	stream->specific_ctx_id_mask = 0;
1595	}
1596
1597	static void
1598	free_oa_buffer(struct i915_perf_stream *stream)
1599	{
1600	i915_vma_unpin_and_release(p_vma: &stream->oa_buffer.vma,
1601	I915_VMA_RELEASE_MAP);
1602
1603	stream->oa_buffer.vaddr = NULL;
1604	}
1605
1606	static void
1607	free_oa_configs(struct i915_perf_stream *stream)
1608	{
1609	struct i915_oa_config_bo *oa_bo, *tmp;
1610
1611	i915_oa_config_put(oa_config: stream->oa_config);
1612	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1613	free_oa_config_bo(oa_bo);
1614	}
1615
1616	static void
1617	free_noa_wait(struct i915_perf_stream *stream)
1618	{
1619	i915_vma_unpin_and_release(p_vma: &stream->noa_wait, flags: 0);
1620	}
1621
1622	static bool engine_supports_oa(const struct intel_engine_cs *engine)
1623	{
1624	return engine->oa_group;
1625	}
1626
1627	static bool engine_supports_oa_format(struct intel_engine_cs *engine, int type)
1628	{
1629	return engine->oa_group && engine->oa_group->type == type;
1630	}
1631
1632	static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1633	{
1634	struct i915_perf *perf = stream->perf;
1635	struct intel_gt *gt = stream->engine->gt;
1636	struct i915_perf_group *g = stream->engine->oa_group;
1637
1638	if (WARN_ON(stream != g->exclusive_stream))
1639	return;
1640
1641	/*
1642	* Unset exclusive_stream first, it will be checked while disabling
1643	* the metric set on gen8+.
1644	*
1645	* See i915_oa_init_reg_state() and lrc_configure_all_contexts()
1646	*/
1647	WRITE_ONCE(g->exclusive_stream, NULL);
1648	perf->ops.disable_metric_set(stream);
1649
1650	free_oa_buffer(stream);
1651
1652	intel_uncore_forcewake_put(uncore: stream->uncore, domains: FORCEWAKE_ALL);
1653	intel_engine_pm_put(engine: stream->engine);
1654
1655	if (stream->ctx)
1656	oa_put_render_ctx_id(stream);
1657
1658	free_oa_configs(stream);
1659	free_noa_wait(stream);
1660
1661	if (perf->spurious_report_rs.missed) {
1662	gt_notice(gt, "%d spurious OA report notices suppressed due to ratelimiting\n",
1663	perf->spurious_report_rs.missed);
1664	}
1665	}
1666
1667	static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1668	{
1669	struct intel_uncore *uncore = stream->uncore;
1670	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1671	unsigned long flags;
1672
1673	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1674
1675	/* Pre-DevBDW: OABUFFER must be set with counters off,
1676	* before OASTATUS1, but after OASTATUS2
1677	*/
1678	intel_uncore_write(uncore, GEN7_OASTATUS2, /* head */
1679	val: gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT);
1680	stream->oa_buffer.head = 0;
1681
1682	intel_uncore_write(uncore, GEN7_OABUFFER, val: gtt_offset);
1683
1684	intel_uncore_write(uncore, GEN7_OASTATUS1, /* tail */
1685	val: gtt_offset | OABUFFER_SIZE_16M);
1686
1687	/* Mark that we need updated tail pointers to read from... */
1688	stream->oa_buffer.tail = 0;
1689
1690	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1691
1692	/* On Haswell we have to track which OASTATUS1 flags we've
1693	* already seen since they can't be cleared while periodic
1694	* sampling is enabled.
1695	*/
1696	stream->perf->gen7_latched_oastatus1 = 0;
1697
1698	/* NB: although the OA buffer will initially be allocated
1699	* zeroed via shmfs (and so this memset is redundant when
1700	* first allocating), we may re-init the OA buffer, either
1701	* when re-enabling a stream or in error/reset paths.
1702	*
1703	* The reason we clear the buffer for each re-init is for the
1704	* sanity check in gen7_append_oa_reports() that looks at the
1705	* report-id field to make sure it's non-zero which relies on
1706	* the assumption that new reports are being written to zeroed
1707	* memory...
1708	*/
1709	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1710	}
1711
1712	static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1713	{
1714	struct intel_uncore *uncore = stream->uncore;
1715	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1716	unsigned long flags;
1717
1718	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1719
1720	intel_uncore_write(uncore, GEN8_OASTATUS, val: 0);
1721	intel_uncore_write(uncore, GEN8_OAHEADPTR, val: gtt_offset);
1722	stream->oa_buffer.head = 0;
1723
1724	intel_uncore_write(uncore, GEN8_OABUFFER_UDW, val: 0);
1725
1726	/*
1727	* PRM says:
1728	*
1729	* "This MMIO must be set before the OATAILPTR
1730	* register and after the OAHEADPTR register. This is
1731	* to enable proper functionality of the overflow
1732	* bit."
1733	*/
1734	intel_uncore_write(uncore, GEN8_OABUFFER, val: gtt_offset |
1735	OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1736	intel_uncore_write(uncore, GEN8_OATAILPTR, val: gtt_offset & GEN8_OATAILPTR_MASK);
1737
1738	/* Mark that we need updated tail pointers to read from... */
1739	stream->oa_buffer.tail = 0;
1740
1741	/*
1742	* Reset state used to recognise context switches, affecting which
1743	* reports we will forward to userspace while filtering for a single
1744	* context.
1745	*/
1746	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1747
1748	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1749
1750	/*
1751	* NB: although the OA buffer will initially be allocated
1752	* zeroed via shmfs (and so this memset is redundant when
1753	* first allocating), we may re-init the OA buffer, either
1754	* when re-enabling a stream or in error/reset paths.
1755	*
1756	* The reason we clear the buffer for each re-init is for the
1757	* sanity check in gen8_append_oa_reports() that looks at the
1758	* reason field to make sure it's non-zero which relies on
1759	* the assumption that new reports are being written to zeroed
1760	* memory...
1761	*/
1762	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1763	}
1764
1765	static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1766	{
1767	struct intel_uncore *uncore = stream->uncore;
1768	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1769	unsigned long flags;
1770
1771	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1772
1773	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_status, val: 0);
1774	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_head_ptr,
1775	val: gtt_offset & GEN12_OAG_OAHEADPTR_MASK);
1776	stream->oa_buffer.head = 0;
1777
1778	/*
1779	* PRM says:
1780	*
1781	* "This MMIO must be set before the OATAILPTR
1782	* register and after the OAHEADPTR register. This is
1783	* to enable proper functionality of the overflow
1784	* bit."
1785	*/
1786	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_buffer, val: gtt_offset |
1787	OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1788	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_tail_ptr,
1789	val: gtt_offset & GEN12_OAG_OATAILPTR_MASK);
1790
1791	/* Mark that we need updated tail pointers to read from... */
1792	stream->oa_buffer.tail = 0;
1793
1794	/*
1795	* Reset state used to recognise context switches, affecting which
1796	* reports we will forward to userspace while filtering for a single
1797	* context.
1798	*/
1799	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1800
1801	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1802
1803	/*
1804	* NB: although the OA buffer will initially be allocated
1805	* zeroed via shmfs (and so this memset is redundant when
1806	* first allocating), we may re-init the OA buffer, either
1807	* when re-enabling a stream or in error/reset paths.
1808	*
1809	* The reason we clear the buffer for each re-init is for the
1810	* sanity check in gen8_append_oa_reports() that looks at the
1811	* reason field to make sure it's non-zero which relies on
1812	* the assumption that new reports are being written to zeroed
1813	* memory...
1814	*/
1815	memset(stream->oa_buffer.vaddr, 0,
1816	stream->oa_buffer.vma->size);
1817	}
1818
1819	static int alloc_oa_buffer(struct i915_perf_stream *stream)
1820	{
1821	struct drm_i915_private *i915 = stream->perf->i915;
1822	struct intel_gt *gt = stream->engine->gt;
1823	struct drm_i915_gem_object *bo;
1824	struct i915_vma *vma;
1825	int ret;
1826
1827	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1828	return -ENODEV;
1829
1830	BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
1831	BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
1832
1833	bo = i915_gem_object_create_shmem(i915: stream->perf->i915, OA_BUFFER_SIZE);
1834	if (IS_ERR(ptr: bo)) {
1835	drm_err(&i915->drm, "Failed to allocate OA buffer\n");
1836	return PTR_ERR(ptr: bo);
1837	}
1838
1839	i915_gem_object_set_cache_coherency(obj: bo, cache_level: I915_CACHE_LLC);
1840
1841	/* PreHSW required 512K alignment, HSW requires 16M */
1842	vma = i915_vma_instance(obj: bo, vm: &gt->ggtt->vm, NULL);
1843	if (IS_ERR(ptr: vma)) {
1844	ret = PTR_ERR(ptr: vma);
1845	goto err_unref;
1846	}
1847
1848	/*
1849	* PreHSW required 512K alignment.
1850	* HSW and onwards, align to requested size of OA buffer.
1851	*/
1852	ret = i915_vma_pin(vma, size: 0, SZ_16M, PIN_GLOBAL | PIN_HIGH);
1853	if (ret) {
1854	gt_err(gt, "Failed to pin OA buffer %d\n", ret);
1855	goto err_unref;
1856	}
1857
1858	stream->oa_buffer.vma = vma;
1859
1860	stream->oa_buffer.vaddr =
1861	i915_gem_object_pin_map_unlocked(obj: bo, type: I915_MAP_WB);
1862	if (IS_ERR(ptr: stream->oa_buffer.vaddr)) {
1863	ret = PTR_ERR(ptr: stream->oa_buffer.vaddr);
1864	goto err_unpin;
1865	}
1866
1867	return 0;
1868
1869	err_unpin:
1870	__i915_vma_unpin(vma);
1871
1872	err_unref:
1873	i915_gem_object_put(obj: bo);
1874
1875	stream->oa_buffer.vaddr = NULL;
1876	stream->oa_buffer.vma = NULL;
1877
1878	return ret;
1879	}
1880
1881	static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1882	bool save, i915_reg_t reg, u32 offset,
1883	u32 dword_count)
1884	{
1885	u32 cmd;
1886	u32 d;
1887
1888	cmd = save ? MI_STORE_REGISTER_MEM : MI_LOAD_REGISTER_MEM;
1889	cmd |= MI_SRM_LRM_GLOBAL_GTT;
1890	if (GRAPHICS_VER(stream->perf->i915) >= 8)
1891	cmd++;
1892
1893	for (d = 0; d < dword_count; d++) {
1894	*cs++ = cmd;
1895	*cs++ = i915_mmio_reg_offset(reg) + 4 * d;
1896	*cs++ = i915_ggtt_offset(vma: stream->noa_wait) + offset + 4 * d;
1897	*cs++ = 0;
1898	}
1899
1900	return cs;
1901	}
1902
1903	static int alloc_noa_wait(struct i915_perf_stream *stream)
1904	{
1905	struct drm_i915_private *i915 = stream->perf->i915;
1906	struct intel_gt *gt = stream->engine->gt;
1907	struct drm_i915_gem_object *bo;
1908	struct i915_vma *vma;
1909	const u64 delay_ticks = 0xffffffffffffffff -
1910	intel_gt_ns_to_clock_interval(gt: to_gt(i915: stream->perf->i915),
1911	ns: atomic64_read(v: &stream->perf->noa_programming_delay));
1912	const u32 base = stream->engine->mmio_base;
1913	#define CS_GPR(x) GEN8_RING_CS_GPR(base, x)
1914	u32 *batch, *ts0, *cs, *jump;
1915	struct i915_gem_ww_ctx ww;
1916	int ret, i;
1917	enum {
1918	START_TS,
1919	NOW_TS,
1920	DELTA_TS,
1921	JUMP_PREDICATE,
1922	DELTA_TARGET,
1923	N_CS_GPR
1924	};
1925	i915_reg_t mi_predicate_result = HAS_MI_SET_PREDICATE(i915) ?
1926	MI_PREDICATE_RESULT_2_ENGINE(base) :
1927	MI_PREDICATE_RESULT_1(RENDER_RING_BASE);
1928
1929	/*
1930	* gt->scratch was being used to save/restore the GPR registers, but on
1931	* MTL the scratch uses stolen lmem. An MI_SRM to this memory region
1932	* causes an engine hang. Instead allocate an additional page here to
1933	* save/restore GPR registers
1934	*/
1935	bo = i915_gem_object_create_internal(i915, size: 8192);
1936	if (IS_ERR(ptr: bo)) {
1937	drm_err(&i915->drm,
1938	"Failed to allocate NOA wait batchbuffer\n");
1939	return PTR_ERR(ptr: bo);
1940	}
1941
1942	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
1943	retry:
1944	ret = i915_gem_object_lock(obj: bo, ww: &ww);
1945	if (ret)
1946	goto out_ww;
1947
1948	/*
1949	* We pin in GGTT because we jump into this buffer now because
1950	* multiple OA config BOs will have a jump to this address and it
1951	* needs to be fixed during the lifetime of the i915/perf stream.
1952	*/
1953	vma = i915_vma_instance(obj: bo, vm: &gt->ggtt->vm, NULL);
1954	if (IS_ERR(ptr: vma)) {
1955	ret = PTR_ERR(ptr: vma);
1956	goto out_ww;
1957	}
1958
1959	ret = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0, PIN_GLOBAL | PIN_HIGH);
1960	if (ret)
1961	goto out_ww;
1962
1963	batch = cs = i915_gem_object_pin_map(obj: bo, type: I915_MAP_WB);
1964	if (IS_ERR(ptr: batch)) {
1965	ret = PTR_ERR(ptr: batch);
1966	goto err_unpin;
1967	}
1968
1969	stream->noa_wait = vma;
1970
1971	#define GPR_SAVE_OFFSET 4096
1972	#define PREDICATE_SAVE_OFFSET 4160
1973
1974	/* Save registers. */
1975	for (i = 0; i < N_CS_GPR; i++)
1976	cs = save_restore_register(
1977	stream, cs, save: true /* save */, CS_GPR(i),
1978	GPR_SAVE_OFFSET + 8 * i, dword_count: 2);
1979	cs = save_restore_register(
1980	stream, cs, save: true /* save */, reg: mi_predicate_result,
1981	PREDICATE_SAVE_OFFSET, dword_count: 1);
1982
1983	/* First timestamp snapshot location. */
1984	ts0 = cs;
1985
1986	/*
1987	* Initial snapshot of the timestamp register to implement the wait.
1988	* We work with 32b values, so clear out the top 32b bits of the
1989	* register because the ALU works 64bits.
1990	*/
1991	*cs++ = MI_LOAD_REGISTER_IMM(1);
1992	*cs++ = i915_mmio_reg_offset(CS_GPR(START_TS)) + 4;
1993	*cs++ = 0;
1994	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
1995	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
1996	*cs++ = i915_mmio_reg_offset(CS_GPR(START_TS));
1997
1998	/*
1999	* This is the location we're going to jump back into until the
2000	* required amount of time has passed.
2001	*/
2002	jump = cs;
2003
2004	/*
2005	* Take another snapshot of the timestamp register. Take care to clear
2006	* up the top 32bits of CS_GPR(1) as we're using it for other
2007	* operations below.
2008	*/
2009	*cs++ = MI_LOAD_REGISTER_IMM(1);
2010	*cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS)) + 4;
2011	*cs++ = 0;
2012	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2013	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
2014	*cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS));
2015
2016	/*
2017	* Do a diff between the 2 timestamps and store the result back into
2018	* CS_GPR(1).
2019	*/
2020	*cs++ = MI_MATH(5);
2021	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(NOW_TS));
2022	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(START_TS));
2023	*cs++ = MI_MATH_SUB;
2024	*cs++ = MI_MATH_STORE(MI_MATH_REG(DELTA_TS), MI_MATH_REG_ACCU);
2025	*cs++ = MI_MATH_STORE(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
2026
2027	/*
2028	* Transfer the carry flag (set to 1 if ts1 < ts0, meaning the
2029	* timestamp have rolled over the 32bits) into the predicate register
2030	* to be used for the predicated jump.
2031	*/
2032	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2033	*cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
2034	*cs++ = i915_mmio_reg_offset(mi_predicate_result);
2035
2036	if (HAS_MI_SET_PREDICATE(i915))
2037	*cs++ = MI_SET_PREDICATE | 1;
2038
2039	/* Restart from the beginning if we had timestamps roll over. */
2040	*cs++ = (GRAPHICS_VER(i915) < 8 ?
2041	MI_BATCH_BUFFER_START :
2042	MI_BATCH_BUFFER_START_GEN8) |
2043	MI_BATCH_PREDICATE;
2044	*cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4;
2045	*cs++ = 0;
2046
2047	if (HAS_MI_SET_PREDICATE(i915))
2048	*cs++ = MI_SET_PREDICATE;
2049
2050	/*
2051	* Now add the diff between to previous timestamps and add it to :
2052	* (((1 * << 64) - 1) - delay_ns)
2053	*
2054	* When the Carry Flag contains 1 this means the elapsed time is
2055	* longer than the expected delay, and we can exit the wait loop.
2056	*/
2057	*cs++ = MI_LOAD_REGISTER_IMM(2);
2058	*cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET));
2059	*cs++ = lower_32_bits(delay_ticks);
2060	*cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET)) + 4;
2061	*cs++ = upper_32_bits(delay_ticks);
2062
2063	*cs++ = MI_MATH(4);
2064	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(DELTA_TS));
2065	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(DELTA_TARGET));
2066	*cs++ = MI_MATH_ADD;
2067	*cs++ = MI_MATH_STOREINV(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
2068
2069	*cs++ = MI_ARB_CHECK;
2070
2071	/*
2072	* Transfer the result into the predicate register to be used for the
2073	* predicated jump.
2074	*/
2075	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2076	*cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
2077	*cs++ = i915_mmio_reg_offset(mi_predicate_result);
2078
2079	if (HAS_MI_SET_PREDICATE(i915))
2080	*cs++ = MI_SET_PREDICATE | 1;
2081
2082	/* Predicate the jump. */
2083	*cs++ = (GRAPHICS_VER(i915) < 8 ?
2084	MI_BATCH_BUFFER_START :
2085	MI_BATCH_BUFFER_START_GEN8) |
2086	MI_BATCH_PREDICATE;
2087	*cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4;
2088	*cs++ = 0;
2089
2090	if (HAS_MI_SET_PREDICATE(i915))
2091	*cs++ = MI_SET_PREDICATE;
2092
2093	/* Restore registers. */
2094	for (i = 0; i < N_CS_GPR; i++)
2095	cs = save_restore_register(
2096	stream, cs, save: false /* restore */, CS_GPR(i),
2097	GPR_SAVE_OFFSET + 8 * i, dword_count: 2);
2098	cs = save_restore_register(
2099	stream, cs, save: false /* restore */, reg: mi_predicate_result,
2100	PREDICATE_SAVE_OFFSET, dword_count: 1);
2101
2102	/* And return to the ring. */
2103	*cs++ = MI_BATCH_BUFFER_END;
2104
2105	GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch));
2106
2107	i915_gem_object_flush_map(obj: bo);
2108	__i915_gem_object_release_map(obj: bo);
2109
2110	goto out_ww;
2111
2112	err_unpin:
2113	i915_vma_unpin_and_release(p_vma: &vma, flags: 0);
2114	out_ww:
2115	if (ret == -EDEADLK) {
2116	ret = i915_gem_ww_ctx_backoff(ctx: &ww);
2117	if (!ret)
2118	goto retry;
2119	}
2120	i915_gem_ww_ctx_fini(ctx: &ww);
2121	if (ret)
2122	i915_gem_object_put(obj: bo);
2123	return ret;
2124	}
2125
2126	static u32 *write_cs_mi_lri(u32 *cs,
2127	const struct i915_oa_reg *reg_data,
2128	u32 n_regs)
2129	{
2130	u32 i;
2131
2132	for (i = 0; i < n_regs; i++) {
2133	if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
2134	u32 n_lri = min_t(u32,
2135	n_regs - i,
2136	MI_LOAD_REGISTER_IMM_MAX_REGS);
2137
2138	*cs++ = MI_LOAD_REGISTER_IMM(n_lri);
2139	}
2140	*cs++ = i915_mmio_reg_offset(reg_data[i].addr);
2141	*cs++ = reg_data[i].value;
2142	}
2143
2144	return cs;
2145	}
2146
2147	static int num_lri_dwords(int num_regs)
2148	{
2149	int count = 0;
2150
2151	if (num_regs > 0) {
2152	count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
2153	count += num_regs * 2;
2154	}
2155
2156	return count;
2157	}
2158
2159	static struct i915_oa_config_bo *
2160	alloc_oa_config_buffer(struct i915_perf_stream *stream,
2161	struct i915_oa_config *oa_config)
2162	{
2163	struct drm_i915_gem_object *obj;
2164	struct i915_oa_config_bo *oa_bo;
2165	struct i915_gem_ww_ctx ww;
2166	size_t config_length = 0;
2167	u32 *cs;
2168	int err;
2169
2170	oa_bo = kzalloc(size: sizeof(*oa_bo), GFP_KERNEL);
2171	if (!oa_bo)
2172	return ERR_PTR(error: -ENOMEM);
2173
2174	config_length += num_lri_dwords(num_regs: oa_config->mux_regs_len);
2175	config_length += num_lri_dwords(num_regs: oa_config->b_counter_regs_len);
2176	config_length += num_lri_dwords(num_regs: oa_config->flex_regs_len);
2177	config_length += 3; /* MI_BATCH_BUFFER_START */
2178	config_length = ALIGN(sizeof(u32) * config_length, I915_GTT_PAGE_SIZE);
2179
2180	obj = i915_gem_object_create_shmem(i915: stream->perf->i915, size: config_length);
2181	if (IS_ERR(ptr: obj)) {
2182	err = PTR_ERR(ptr: obj);
2183	goto err_free;
2184	}
2185
2186	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
2187	retry:
2188	err = i915_gem_object_lock(obj, ww: &ww);
2189	if (err)
2190	goto out_ww;
2191
2192	cs = i915_gem_object_pin_map(obj, type: I915_MAP_WB);
2193	if (IS_ERR(ptr: cs)) {
2194	err = PTR_ERR(ptr: cs);
2195	goto out_ww;
2196	}
2197
2198	cs = write_cs_mi_lri(cs,
2199	reg_data: oa_config->mux_regs,
2200	n_regs: oa_config->mux_regs_len);
2201	cs = write_cs_mi_lri(cs,
2202	reg_data: oa_config->b_counter_regs,
2203	n_regs: oa_config->b_counter_regs_len);
2204	cs = write_cs_mi_lri(cs,
2205	reg_data: oa_config->flex_regs,
2206	n_regs: oa_config->flex_regs_len);
2207
2208	/* Jump into the active wait. */
2209	*cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ?
2210	MI_BATCH_BUFFER_START :
2211	MI_BATCH_BUFFER_START_GEN8);
2212	*cs++ = i915_ggtt_offset(vma: stream->noa_wait);
2213	*cs++ = 0;
2214
2215	i915_gem_object_flush_map(obj);
2216	__i915_gem_object_release_map(obj);
2217
2218	oa_bo->vma = i915_vma_instance(obj,
2219	vm: &stream->engine->gt->ggtt->vm,
2220	NULL);
2221	if (IS_ERR(ptr: oa_bo->vma)) {
2222	err = PTR_ERR(ptr: oa_bo->vma);
2223	goto out_ww;
2224	}
2225
2226	oa_bo->oa_config = i915_oa_config_get(oa_config);
2227	llist_add(new: &oa_bo->node, head: &stream->oa_config_bos);
2228
2229	out_ww:
2230	if (err == -EDEADLK) {
2231	err = i915_gem_ww_ctx_backoff(ctx: &ww);
2232	if (!err)
2233	goto retry;
2234	}
2235	i915_gem_ww_ctx_fini(ctx: &ww);
2236
2237	if (err)
2238	i915_gem_object_put(obj);
2239	err_free:
2240	if (err) {
2241	kfree(objp: oa_bo);
2242	return ERR_PTR(error: err);
2243	}
2244	return oa_bo;
2245	}
2246
2247	static struct i915_vma *
2248	get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
2249	{
2250	struct i915_oa_config_bo *oa_bo;
2251
2252	/*
2253	* Look for the buffer in the already allocated BOs attached
2254	* to the stream.
2255	*/
2256	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
2257	if (oa_bo->oa_config == oa_config &&
2258	memcmp(p: oa_bo->oa_config->uuid,
2259	q: oa_config->uuid,
2260	size: sizeof(oa_config->uuid)) == 0)
2261	goto out;
2262	}
2263
2264	oa_bo = alloc_oa_config_buffer(stream, oa_config);
2265	if (IS_ERR(ptr: oa_bo))
2266	return ERR_CAST(ptr: oa_bo);
2267
2268	out:
2269	return i915_vma_get(vma: oa_bo->vma);
2270	}
2271
2272	static int
2273	emit_oa_config(struct i915_perf_stream *stream,
2274	struct i915_oa_config *oa_config,
2275	struct intel_context *ce,
2276	struct i915_active *active)
2277	{
2278	struct i915_request *rq;
2279	struct i915_vma *vma;
2280	struct i915_gem_ww_ctx ww;
2281	int err;
2282
2283	vma = get_oa_vma(stream, oa_config);
2284	if (IS_ERR(ptr: vma))
2285	return PTR_ERR(ptr: vma);
2286
2287	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
2288	retry:
2289	err = i915_gem_object_lock(obj: vma->obj, ww: &ww);
2290	if (err)
2291	goto err;
2292
2293	err = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0, PIN_GLOBAL | PIN_HIGH);
2294	if (err)
2295	goto err;
2296
2297	intel_engine_pm_get(engine: ce->engine);
2298	rq = i915_request_create(ce);
2299	intel_engine_pm_put(engine: ce->engine);
2300	if (IS_ERR(ptr: rq)) {
2301	err = PTR_ERR(ptr: rq);
2302	goto err_vma_unpin;
2303	}
2304
2305	if (!IS_ERR_OR_NULL(ptr: active)) {
2306	/* After all individual context modifications */
2307	err = i915_request_await_active(rq, ref: active,
2308	I915_ACTIVE_AWAIT_ACTIVE);
2309	if (err)
2310	goto err_add_request;
2311
2312	err = i915_active_add_request(ref: active, rq);
2313	if (err)
2314	goto err_add_request;
2315	}
2316
2317	err = i915_vma_move_to_active(vma, rq, flags: 0);
2318	if (err)
2319	goto err_add_request;
2320
2321	err = rq->engine->emit_bb_start(rq,
2322	i915_vma_offset(vma), 0,
2323	I915_DISPATCH_SECURE);
2324	if (err)
2325	goto err_add_request;
2326
2327	err_add_request:
2328	i915_request_add(rq);
2329	err_vma_unpin:
2330	i915_vma_unpin(vma);
2331	err:
2332	if (err == -EDEADLK) {
2333	err = i915_gem_ww_ctx_backoff(ctx: &ww);
2334	if (!err)
2335	goto retry;
2336	}
2337
2338	i915_gem_ww_ctx_fini(ctx: &ww);
2339	i915_vma_put(vma);
2340	return err;
2341	}
2342
2343	static struct intel_context *oa_context(struct i915_perf_stream *stream)
2344	{
2345	return stream->pinned_ctx ?: stream->engine->kernel_context;
2346	}
2347
2348	static int
2349	hsw_enable_metric_set(struct i915_perf_stream *stream,
2350	struct i915_active *active)
2351	{
2352	struct intel_uncore *uncore = stream->uncore;
2353
2354	/*
2355	* PRM:
2356	*
2357	* OA unit is using “crclk” for its functionality. When trunk
2358	* level clock gating takes place, OA clock would be gated,
2359	* unable to count the events from non-render clock domain.
2360	* Render clock gating must be disabled when OA is enabled to
2361	* count the events from non-render domain. Unit level clock
2362	* gating for RCS should also be disabled.
2363	*/
2364	intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2365	GEN7_DOP_CLOCK_GATE_ENABLE, set: 0);
2366	intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2367	clear: 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
2368
2369	return emit_oa_config(stream,
2370	oa_config: stream->oa_config, ce: oa_context(stream),
2371	active);
2372	}
2373
2374	static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2375	{
2376	struct intel_uncore *uncore = stream->uncore;
2377
2378	intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2379	GEN6_CSUNIT_CLOCK_GATE_DISABLE, set: 0);
2380	intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2381	clear: 0, GEN7_DOP_CLOCK_GATE_ENABLE);
2382
2383	intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, set: 0);
2384	}
2385
2386	static u32 oa_config_flex_reg(const struct i915_oa_config *oa_config,
2387	i915_reg_t reg)
2388	{
2389	u32 mmio = i915_mmio_reg_offset(reg);
2390	int i;
2391
2392	/*
2393	* This arbitrary default will select the 'EU FPU0 Pipeline
2394	* Active' event. In the future it's anticipated that there
2395	* will be an explicit 'No Event' we can select, but not yet...
2396	*/
2397	if (!oa_config)
2398	return 0;
2399
2400	for (i = 0; i < oa_config->flex_regs_len; i++) {
2401	if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio)
2402	return oa_config->flex_regs[i].value;
2403	}
2404
2405	return 0;
2406	}
2407	/*
2408	* NB: It must always remain pointer safe to run this even if the OA unit
2409	* has been disabled.
2410	*
2411	* It's fine to put out-of-date values into these per-context registers
2412	* in the case that the OA unit has been disabled.
2413	*/
2414	static void
2415	gen8_update_reg_state_unlocked(const struct intel_context *ce,
2416	const struct i915_perf_stream *stream)
2417	{
2418	u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2419	u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2420	/* The MMIO offsets for Flex EU registers aren't contiguous */
2421	static const i915_reg_t flex_regs[] = {
2422	EU_PERF_CNTL0,
2423	EU_PERF_CNTL1,
2424	EU_PERF_CNTL2,
2425	EU_PERF_CNTL3,
2426	EU_PERF_CNTL4,
2427	EU_PERF_CNTL5,
2428	EU_PERF_CNTL6,
2429	};
2430	u32 *reg_state = ce->lrc_reg_state;
2431	int i;
2432
2433	reg_state[ctx_oactxctrl + 1] =
2434	(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2435	(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2436	GEN8_OA_COUNTER_RESUME;
2437
2438	for (i = 0; i < ARRAY_SIZE(flex_regs); i++)
2439	reg_state[ctx_flexeu0 + i * 2 + 1] =
2440	oa_config_flex_reg(oa_config: stream->oa_config, reg: flex_regs[i]);
2441	}
2442
2443	struct flex {
2444	i915_reg_t reg;
2445	u32 offset;
2446	u32 value;
2447	};
2448
2449	static int
2450	gen8_store_flex(struct i915_request *rq,
2451	struct intel_context *ce,
2452	const struct flex *flex, unsigned int count)
2453	{
2454	u32 offset;
2455	u32 *cs;
2456
2457	cs = intel_ring_begin(rq, num_dwords: 4 * count);
2458	if (IS_ERR(ptr: cs))
2459	return PTR_ERR(ptr: cs);
2460
2461	offset = i915_ggtt_offset(vma: ce->state) + LRC_STATE_OFFSET;
2462	do {
2463	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
2464	*cs++ = offset + flex->offset * sizeof(u32);
2465	*cs++ = 0;
2466	*cs++ = flex->value;
2467	} while (flex++, --count);
2468
2469	intel_ring_advance(rq, cs);
2470
2471	return 0;
2472	}
2473
2474	static int
2475	gen8_load_flex(struct i915_request *rq,
2476	struct intel_context *ce,
2477	const struct flex *flex, unsigned int count)
2478	{
2479	u32 *cs;
2480
2481	GEM_BUG_ON(!count || count > 63);
2482
2483	cs = intel_ring_begin(rq, num_dwords: 2 * count + 2);
2484	if (IS_ERR(ptr: cs))
2485	return PTR_ERR(ptr: cs);
2486
2487	*cs++ = MI_LOAD_REGISTER_IMM(count);
2488	do {
2489	*cs++ = i915_mmio_reg_offset(flex->reg);
2490	*cs++ = flex->value;
2491	} while (flex++, --count);
2492	*cs++ = MI_NOOP;
2493
2494	intel_ring_advance(rq, cs);
2495
2496	return 0;
2497	}
2498
2499	static int gen8_modify_context(struct intel_context *ce,
2500	const struct flex *flex, unsigned int count)
2501	{
2502	struct i915_request *rq;
2503	int err;
2504
2505	rq = intel_engine_create_kernel_request(engine: ce->engine);
2506	if (IS_ERR(ptr: rq))
2507	return PTR_ERR(ptr: rq);
2508
2509	/* Serialise with the remote context */
2510	err = intel_context_prepare_remote_request(ce, rq);
2511	if (err == 0)
2512	err = gen8_store_flex(rq, ce, flex, count);
2513
2514	i915_request_add(rq);
2515	return err;
2516	}
2517
2518	static int
2519	gen8_modify_self(struct intel_context *ce,
2520	const struct flex *flex, unsigned int count,
2521	struct i915_active *active)
2522	{
2523	struct i915_request *rq;
2524	int err;
2525
2526	intel_engine_pm_get(engine: ce->engine);
2527	rq = i915_request_create(ce);
2528	intel_engine_pm_put(engine: ce->engine);
2529	if (IS_ERR(ptr: rq))
2530	return PTR_ERR(ptr: rq);
2531
2532	if (!IS_ERR_OR_NULL(ptr: active)) {
2533	err = i915_active_add_request(ref: active, rq);
2534	if (err)
2535	goto err_add_request;
2536	}
2537
2538	err = gen8_load_flex(rq, ce, flex, count);
2539	if (err)
2540	goto err_add_request;
2541
2542	err_add_request:
2543	i915_request_add(rq);
2544	return err;
2545	}
2546
2547	static int gen8_configure_context(struct i915_perf_stream *stream,
2548	struct i915_gem_context *ctx,
2549	struct flex *flex, unsigned int count)
2550	{
2551	struct i915_gem_engines_iter it;
2552	struct intel_context *ce;
2553	int err = 0;
2554
2555	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
2556	GEM_BUG_ON(ce == ce->engine->kernel_context);
2557
2558	if (ce->engine->class != RENDER_CLASS)
2559	continue;
2560
2561	/* Otherwise OA settings will be set upon first use */
2562	if (!intel_context_pin_if_active(ce))
2563	continue;
2564
2565	flex->value = intel_sseu_make_rpcs(gt: ce->engine->gt, req_sseu: &ce->sseu);
2566	err = gen8_modify_context(ce, flex, count);
2567
2568	intel_context_unpin(ce);
2569	if (err)
2570	break;
2571	}
2572	i915_gem_context_unlock_engines(ctx);
2573
2574	return err;
2575	}
2576
2577	static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2578	struct i915_active *active)
2579	{
2580	int err;
2581	struct intel_context *ce = stream->pinned_ctx;
2582	u32 format = stream->oa_buffer.format->format;
2583	u32 offset = stream->perf->ctx_oactxctrl_offset;
2584	struct flex regs_context[] = {
2585	{
2586	GEN8_OACTXCONTROL,
2587	offset + 1,
2588	active ? GEN8_OA_COUNTER_RESUME : 0,
2589	},
2590	};
2591	/* Offsets in regs_lri are not used since this configuration is only
2592	* applied using LRI. Initialize the correct offsets for posterity.
2593	*/
2594	#define GEN12_OAR_OACONTROL_OFFSET 0x5B0
2595	struct flex regs_lri[] = {
2596	{
2597	GEN12_OAR_OACONTROL,
2598	GEN12_OAR_OACONTROL_OFFSET + 1,
2599	(format << GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT) |
2600	(active ? GEN12_OAR_OACONTROL_COUNTER_ENABLE : 0)
2601	},
2602	{
2603	RING_CONTEXT_CONTROL(ce->engine->mmio_base),
2604	CTX_CONTEXT_CONTROL,
2605	_MASKED_FIELD(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE,
2606	active ?
2607	GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE :
2608	0)
2609	},
2610	};
2611
2612	/* Modify the context image of pinned context with regs_context */
2613	err = intel_context_lock_pinned(ce);
2614	if (err)
2615	return err;
2616
2617	err = gen8_modify_context(ce, flex: regs_context,
2618	ARRAY_SIZE(regs_context));
2619	intel_context_unlock_pinned(ce);
2620	if (err)
2621	return err;
2622
2623	/* Apply regs_lri using LRI with pinned context */
2624	return gen8_modify_self(ce, flex: regs_lri, ARRAY_SIZE(regs_lri), active);
2625	}
2626
2627	/*
2628	* Manages updating the per-context aspects of the OA stream
2629	* configuration across all contexts.
2630	*
2631	* The awkward consideration here is that OACTXCONTROL controls the
2632	* exponent for periodic sampling which is primarily used for system
2633	* wide profiling where we'd like a consistent sampling period even in
2634	* the face of context switches.
2635	*
2636	* Our approach of updating the register state context (as opposed to
2637	* say using a workaround batch buffer) ensures that the hardware
2638	* won't automatically reload an out-of-date timer exponent even
2639	* transiently before a WA BB could be parsed.
2640	*
2641	* This function needs to:
2642	* - Ensure the currently running context's per-context OA state is
2643	* updated
2644	* - Ensure that all existing contexts will have the correct per-context
2645	* OA state if they are scheduled for use.
2646	* - Ensure any new contexts will be initialized with the correct
2647	* per-context OA state.
2648	*
2649	* Note: it's only the RCS/Render context that has any OA state.
2650	* Note: the first flex register passed must always be R_PWR_CLK_STATE
2651	*/
2652	static int
2653	oa_configure_all_contexts(struct i915_perf_stream *stream,
2654	struct flex *regs,
2655	size_t num_regs,
2656	struct i915_active *active)
2657	{
2658	struct drm_i915_private *i915 = stream->perf->i915;
2659	struct intel_engine_cs *engine;
2660	struct intel_gt *gt = stream->engine->gt;
2661	struct i915_gem_context *ctx, *cn;
2662	int err;
2663
2664	lockdep_assert_held(&gt->perf.lock);
2665
2666	/*
2667	* The OA register config is setup through the context image. This image
2668	* might be written to by the GPU on context switch (in particular on
2669	* lite-restore). This means we can't safely update a context's image,
2670	* if this context is scheduled/submitted to run on the GPU.
2671	*
2672	* We could emit the OA register config through the batch buffer but
2673	* this might leave small interval of time where the OA unit is
2674	* configured at an invalid sampling period.
2675	*
2676	* Note that since we emit all requests from a single ring, there
2677	* is still an implicit global barrier here that may cause a high
2678	* priority context to wait for an otherwise independent low priority
2679	* context. Contexts idle at the time of reconfiguration are not
2680	* trapped behind the barrier.
2681	*/
2682	spin_lock(lock: &i915->gem.contexts.lock);
2683	list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) {
2684	if (!kref_get_unless_zero(kref: &ctx->ref))
2685	continue;
2686
2687	spin_unlock(lock: &i915->gem.contexts.lock);
2688
2689	err = gen8_configure_context(stream, ctx, flex: regs, count: num_regs);
2690	if (err) {
2691	i915_gem_context_put(ctx);
2692	return err;
2693	}
2694
2695	spin_lock(lock: &i915->gem.contexts.lock);
2696	list_safe_reset_next(ctx, cn, link);
2697	i915_gem_context_put(ctx);
2698	}
2699	spin_unlock(lock: &i915->gem.contexts.lock);
2700
2701	/*
2702	* After updating all other contexts, we need to modify ourselves.
2703	* If we don't modify the kernel_context, we do not get events while
2704	* idle.
2705	*/
2706	for_each_uabi_engine(engine, i915) {
2707	struct intel_context *ce = engine->kernel_context;
2708
2709	if (engine->class != RENDER_CLASS)
2710	continue;
2711
2712	regs[0].value = intel_sseu_make_rpcs(gt: engine->gt, req_sseu: &ce->sseu);
2713
2714	err = gen8_modify_self(ce, flex: regs, count: num_regs, active);
2715	if (err)
2716	return err;
2717	}
2718
2719	return 0;
2720	}
2721
2722	static int
2723	gen12_configure_all_contexts(struct i915_perf_stream *stream,
2724	const struct i915_oa_config *oa_config,
2725	struct i915_active *active)
2726	{
2727	struct flex regs[] = {
2728	{
2729	GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE),
2730	CTX_R_PWR_CLK_STATE,
2731	},
2732	};
2733
2734	if (stream->engine->class != RENDER_CLASS)
2735	return 0;
2736
2737	return oa_configure_all_contexts(stream,
2738	regs, ARRAY_SIZE(regs),
2739	active);
2740	}
2741
2742	static int
2743	lrc_configure_all_contexts(struct i915_perf_stream *stream,
2744	const struct i915_oa_config *oa_config,
2745	struct i915_active *active)
2746	{
2747	u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2748	/* The MMIO offsets for Flex EU registers aren't contiguous */
2749	const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2750	#define ctx_flexeuN(N) (ctx_flexeu0 + 2 * (N) + 1)
2751	struct flex regs[] = {
2752	{
2753	GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE),
2754	CTX_R_PWR_CLK_STATE,
2755	},
2756	{
2757	GEN8_OACTXCONTROL,
2758	ctx_oactxctrl + 1,
2759	},
2760	{ EU_PERF_CNTL0, ctx_flexeuN(0) },
2761	{ EU_PERF_CNTL1, ctx_flexeuN(1) },
2762	{ EU_PERF_CNTL2, ctx_flexeuN(2) },
2763	{ EU_PERF_CNTL3, ctx_flexeuN(3) },
2764	{ EU_PERF_CNTL4, ctx_flexeuN(4) },
2765	{ EU_PERF_CNTL5, ctx_flexeuN(5) },
2766	{ EU_PERF_CNTL6, ctx_flexeuN(6) },
2767	};
2768	#undef ctx_flexeuN
2769	int i;
2770
2771	regs[1].value =
2772	(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2773	(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2774	GEN8_OA_COUNTER_RESUME;
2775
2776	for (i = 2; i < ARRAY_SIZE(regs); i++)
2777	regs[i].value = oa_config_flex_reg(oa_config, reg: regs[i].reg);
2778
2779	return oa_configure_all_contexts(stream,
2780	regs, ARRAY_SIZE(regs),
2781	active);
2782	}
2783
2784	static int
2785	gen8_enable_metric_set(struct i915_perf_stream *stream,
2786	struct i915_active *active)
2787	{
2788	struct intel_uncore *uncore = stream->uncore;
2789	struct i915_oa_config *oa_config = stream->oa_config;
2790	int ret;
2791
2792	/*
2793	* We disable slice/unslice clock ratio change reports on SKL since
2794	* they are too noisy. The HW generates a lot of redundant reports
2795	* where the ratio hasn't really changed causing a lot of redundant
2796	* work to processes and increasing the chances we'll hit buffer
2797	* overruns.
2798	*
2799	* Although we don't currently use the 'disable overrun' OABUFFER
2800	* feature it's worth noting that clock ratio reports have to be
2801	* disabled before considering to use that feature since the HW doesn't
2802	* correctly block these reports.
2803	*
2804	* Currently none of the high-level metrics we have depend on knowing
2805	* this ratio to normalize.
2806	*
2807	* Note: This register is not power context saved and restored, but
2808	* that's OK considering that we disable RC6 while the OA unit is
2809	* enabled.
2810	*
2811	* The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to
2812	* be read back from automatically triggered reports, as part of the
2813	* RPT_ID field.
2814	*/
2815	if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {
2816	intel_uncore_write(uncore, GEN8_OA_DEBUG,
2817	_MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2818	GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
2819	}
2820
2821	/*
2822	* Update all contexts prior writing the mux configurations as we need
2823	* to make sure all slices/subslices are ON before writing to NOA
2824	* registers.
2825	*/
2826	ret = lrc_configure_all_contexts(stream, oa_config, active);
2827	if (ret)
2828	return ret;
2829
2830	return emit_oa_config(stream,
2831	oa_config: stream->oa_config, ce: oa_context(stream),
2832	active);
2833	}
2834
2835	static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2836	{
2837	return _MASKED_FIELD(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
2838	(stream->sample_flags & SAMPLE_OA_REPORT) ?
2839	0 : GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
2840	}
2841
2842	static int
2843	gen12_enable_metric_set(struct i915_perf_stream *stream,
2844	struct i915_active *active)
2845	{
2846	struct drm_i915_private *i915 = stream->perf->i915;
2847	struct intel_uncore *uncore = stream->uncore;
2848	struct i915_oa_config *oa_config = stream->oa_config;
2849	bool periodic = stream->periodic;
2850	u32 period_exponent = stream->period_exponent;
2851	u32 sqcnt1;
2852	int ret;
2853
2854	/*
2855	* Wa_1508761755:xehpsdv, dg2
2856	* EU NOA signals behave incorrectly if EU clock gating is enabled.
2857	* Disable thread stall DOP gating and EU DOP gating.
2858	*/
2859	if (IS_XEHPSDV(i915) || IS_DG2(i915)) {
2860	intel_gt_mcr_multicast_write(gt: uncore->gt, GEN8_ROW_CHICKEN,
2861	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
2862	intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
2863	_MASKED_BIT_ENABLE(GEN12_DISABLE_DOP_GATING));
2864	}
2865
2866	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_debug,
2867	/* Disable clk ratio reports, like previous Gens. */
2868	_MASKED_BIT_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2869	GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
2870	/*
2871	* If the user didn't require OA reports, instruct
2872	* the hardware not to emit ctx switch reports.
2873	*/
2874	oag_report_ctx_switches(stream));
2875
2876	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_ctx_ctrl, val: periodic ?
2877	(GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME |
2878	GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE |
2879	(period_exponent << GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT))
2880	: 0);
2881
2882	/*
2883	* Initialize Super Queue Internal Cnt Register
2884	* Set PMON Enable in order to collect valid metrics.
2885	* Enable byets per clock reporting in OA for XEHPSDV onward.
2886	*/
2887	sqcnt1 = GEN12_SQCNT1_PMON_ENABLE |
2888	(HAS_OA_BPC_REPORTING(i915) ? GEN12_SQCNT1_OABPC : 0);
2889
2890	intel_uncore_rmw(uncore, GEN12_SQCNT1, clear: 0, set: sqcnt1);
2891
2892	/*
2893	* Update all contexts prior writing the mux configurations as we need
2894	* to make sure all slices/subslices are ON before writing to NOA
2895	* registers.
2896	*/
2897	ret = gen12_configure_all_contexts(stream, oa_config, active);
2898	if (ret)
2899	return ret;
2900
2901	/*
2902	* For Gen12, performance counters are context
2903	* saved/restored. Only enable it for the context that
2904	* requested this.
2905	*/
2906	if (stream->ctx) {
2907	ret = gen12_configure_oar_context(stream, active);
2908	if (ret)
2909	return ret;
2910	}
2911
2912	return emit_oa_config(stream,
2913	oa_config: stream->oa_config, ce: oa_context(stream),
2914	active);
2915	}
2916
2917	static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2918	{
2919	struct intel_uncore *uncore = stream->uncore;
2920
2921	/* Reset all contexts' slices/subslices configurations. */
2922	lrc_configure_all_contexts(stream, NULL, NULL);
2923
2924	intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, set: 0);
2925	}
2926
2927	static void gen11_disable_metric_set(struct i915_perf_stream *stream)
2928	{
2929	struct intel_uncore *uncore = stream->uncore;
2930
2931	/* Reset all contexts' slices/subslices configurations. */
2932	lrc_configure_all_contexts(stream, NULL, NULL);
2933
2934	/* Make sure we disable noa to save power. */
2935	intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, set: 0);
2936	}
2937
2938	static void gen12_disable_metric_set(struct i915_perf_stream *stream)
2939	{
2940	struct intel_uncore *uncore = stream->uncore;
2941	struct drm_i915_private *i915 = stream->perf->i915;
2942	u32 sqcnt1;
2943
2944	/*
2945	* Wa_1508761755:xehpsdv, dg2
2946	* Enable thread stall DOP gating and EU DOP gating.
2947	*/
2948	if (IS_XEHPSDV(i915) || IS_DG2(i915)) {
2949	intel_gt_mcr_multicast_write(gt: uncore->gt, GEN8_ROW_CHICKEN,
2950	_MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
2951	intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
2952	_MASKED_BIT_DISABLE(GEN12_DISABLE_DOP_GATING));
2953	}
2954
2955	/* Reset all contexts' slices/subslices configurations. */
2956	gen12_configure_all_contexts(stream, NULL, NULL);
2957
2958	/* disable the context save/restore or OAR counters */
2959	if (stream->ctx)
2960	gen12_configure_oar_context(stream, NULL);
2961
2962	/* Make sure we disable noa to save power. */
2963	intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, set: 0);
2964
2965	sqcnt1 = GEN12_SQCNT1_PMON_ENABLE |
2966	(HAS_OA_BPC_REPORTING(i915) ? GEN12_SQCNT1_OABPC : 0);
2967
2968	/* Reset PMON Enable to save power. */
2969	intel_uncore_rmw(uncore, GEN12_SQCNT1, clear: sqcnt1, set: 0);
2970	}
2971
2972	static void gen7_oa_enable(struct i915_perf_stream *stream)
2973	{
2974	struct intel_uncore *uncore = stream->uncore;
2975	struct i915_gem_context *ctx = stream->ctx;
2976	u32 ctx_id = stream->specific_ctx_id;
2977	bool periodic = stream->periodic;
2978	u32 period_exponent = stream->period_exponent;
2979	u32 report_format = stream->oa_buffer.format->format;
2980
2981	/*
2982	* Reset buf pointers so we don't forward reports from before now.
2983	*
2984	* Think carefully if considering trying to avoid this, since it
2985	* also ensures status flags and the buffer itself are cleared
2986	* in error paths, and we have checks for invalid reports based
2987	* on the assumption that certain fields are written to zeroed
2988	* memory which this helps maintains.
2989	*/
2990	gen7_init_oa_buffer(stream);
2991
2992	intel_uncore_write(uncore, GEN7_OACONTROL,
2993	val: (ctx_id & GEN7_OACONTROL_CTX_MASK) |
2994	(period_exponent <<
2995	GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
2996	(periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
2997	(report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
2998	(ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
2999	GEN7_OACONTROL_ENABLE);
3000	}
3001
3002	static void gen8_oa_enable(struct i915_perf_stream *stream)
3003	{
3004	struct intel_uncore *uncore = stream->uncore;
3005	u32 report_format = stream->oa_buffer.format->format;
3006
3007	/*
3008	* Reset buf pointers so we don't forward reports from before now.
3009	*
3010	* Think carefully if considering trying to avoid this, since it
3011	* also ensures status flags and the buffer itself are cleared
3012	* in error paths, and we have checks for invalid reports based
3013	* on the assumption that certain fields are written to zeroed
3014	* memory which this helps maintains.
3015	*/
3016	gen8_init_oa_buffer(stream);
3017
3018	/*
3019	* Note: we don't rely on the hardware to perform single context
3020	* filtering and instead filter on the cpu based on the context-id
3021	* field of reports
3022	*/
3023	intel_uncore_write(uncore, GEN8_OACONTROL,
3024	val: (report_format << GEN8_OA_REPORT_FORMAT_SHIFT) |
3025	GEN8_OA_COUNTER_ENABLE);
3026	}
3027
3028	static void gen12_oa_enable(struct i915_perf_stream *stream)
3029	{
3030	const struct i915_perf_regs *regs;
3031	u32 val;
3032
3033	/*
3034	* If we don't want OA reports from the OA buffer, then we don't even
3035	* need to program the OAG unit.
3036	*/
3037	if (!(stream->sample_flags & SAMPLE_OA_REPORT))
3038	return;
3039
3040	gen12_init_oa_buffer(stream);
3041
3042	regs = __oa_regs(stream);
3043	val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) |
3044	GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE;
3045
3046	intel_uncore_write(uncore: stream->uncore, reg: regs->oa_ctrl, val);
3047	}
3048
3049	/**
3050	* i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
3051	* @stream: An i915 perf stream opened for OA metrics
3052	*
3053	* [Re]enables hardware periodic sampling according to the period configured
3054	* when opening the stream. This also starts a hrtimer that will periodically
3055	* check for data in the circular OA buffer for notifying userspace (e.g.
3056	* during a read() or poll()).
3057	*/
3058	static void i915_oa_stream_enable(struct i915_perf_stream *stream)
3059	{
3060	stream->pollin = false;
3061
3062	stream->perf->ops.oa_enable(stream);
3063
3064	if (stream->sample_flags & SAMPLE_OA_REPORT)
3065	hrtimer_start(timer: &stream->poll_check_timer,
3066	tim: ns_to_ktime(ns: stream->poll_oa_period),
3067	mode: HRTIMER_MODE_REL_PINNED);
3068	}
3069
3070	static void gen7_oa_disable(struct i915_perf_stream *stream)
3071	{
3072	struct intel_uncore *uncore = stream->uncore;
3073
3074	intel_uncore_write(uncore, GEN7_OACONTROL, val: 0);
3075	if (intel_wait_for_register(uncore,
3076	GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, value: 0,
3077	timeout_ms: 50))
3078	drm_err(&stream->perf->i915->drm,
3079	"wait for OA to be disabled timed out\n");
3080	}
3081
3082	static void gen8_oa_disable(struct i915_perf_stream *stream)
3083	{
3084	struct intel_uncore *uncore = stream->uncore;
3085
3086	intel_uncore_write(uncore, GEN8_OACONTROL, val: 0);
3087	if (intel_wait_for_register(uncore,
3088	GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, value: 0,
3089	timeout_ms: 50))
3090	drm_err(&stream->perf->i915->drm,
3091	"wait for OA to be disabled timed out\n");
3092	}
3093
3094	static void gen12_oa_disable(struct i915_perf_stream *stream)
3095	{
3096	struct intel_uncore *uncore = stream->uncore;
3097
3098	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_ctrl, val: 0);
3099	if (intel_wait_for_register(uncore,
3100	reg: __oa_regs(stream)->oa_ctrl,
3101	GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE, value: 0,
3102	timeout_ms: 50))
3103	drm_err(&stream->perf->i915->drm,
3104	"wait for OA to be disabled timed out\n");
3105
3106	intel_uncore_write(uncore, GEN12_OA_TLB_INV_CR, val: 1);
3107	if (intel_wait_for_register(uncore,
3108	GEN12_OA_TLB_INV_CR,
3109	mask: 1, value: 0,
3110	timeout_ms: 50))
3111	drm_err(&stream->perf->i915->drm,
3112	"wait for OA tlb invalidate timed out\n");
3113	}
3114
3115	/**
3116	* i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
3117	* @stream: An i915 perf stream opened for OA metrics
3118	*
3119	* Stops the OA unit from periodically writing counter reports into the
3120	* circular OA buffer. This also stops the hrtimer that periodically checks for
3121	* data in the circular OA buffer, for notifying userspace.
3122	*/
3123	static void i915_oa_stream_disable(struct i915_perf_stream *stream)
3124	{
3125	stream->perf->ops.oa_disable(stream);
3126
3127	if (stream->sample_flags & SAMPLE_OA_REPORT)
3128	hrtimer_cancel(timer: &stream->poll_check_timer);
3129	}
3130
3131	static const struct i915_perf_stream_ops i915_oa_stream_ops = {
3132	.destroy = i915_oa_stream_destroy,
3133	.enable = i915_oa_stream_enable,
3134	.disable = i915_oa_stream_disable,
3135	.wait_unlocked = i915_oa_wait_unlocked,
3136	.poll_wait = i915_oa_poll_wait,
3137	.read = i915_oa_read,
3138	};
3139
3140	static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
3141	{
3142	struct i915_active *active;
3143	int err;
3144
3145	active = i915_active_create();
3146	if (!active)
3147	return -ENOMEM;
3148
3149	err = stream->perf->ops.enable_metric_set(stream, active);
3150	if (err == 0)
3151	__i915_active_wait(ref: active, TASK_UNINTERRUPTIBLE);
3152
3153	i915_active_put(ref: active);
3154	return err;
3155	}
3156
3157	static void
3158	get_default_sseu_config(struct intel_sseu *out_sseu,
3159	struct intel_engine_cs *engine)
3160	{
3161	const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu;
3162
3163	*out_sseu = intel_sseu_from_device_info(sseu: devinfo_sseu);
3164
3165	if (GRAPHICS_VER(engine->i915) == 11) {
3166	/*
3167	* We only need subslice count so it doesn't matter which ones
3168	* we select - just turn off low bits in the amount of half of
3169	* all available subslices per slice.
3170	*/
3171	out_sseu->subslice_mask =
3172	~(~0 << (hweight8(out_sseu->subslice_mask) / 2));
3173	out_sseu->slice_mask = 0x1;
3174	}
3175	}
3176
3177	static int
3178	get_sseu_config(struct intel_sseu *out_sseu,
3179	struct intel_engine_cs *engine,
3180	const struct drm_i915_gem_context_param_sseu *drm_sseu)
3181	{
3182	if (drm_sseu->engine.engine_class != engine->uabi_class ||
3183	drm_sseu->engine.engine_instance != engine->uabi_instance)
3184	return -EINVAL;
3185
3186	return i915_gem_user_to_context_sseu(gt: engine->gt, user: drm_sseu, context: out_sseu);
3187	}
3188
3189	/*
3190	* OA timestamp frequency = CS timestamp frequency in most platforms. On some
3191	* platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
3192	* cases, return the adjusted CS timestamp frequency to the user.
3193	*/
3194	u32 i915_perf_oa_timestamp_frequency(struct drm_i915_private *i915)
3195	{
3196	struct intel_gt *gt = to_gt(i915);
3197
3198	/* Wa_18013179988 */
3199	if (IS_DG2(i915) || IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 71))) {
3200	intel_wakeref_t wakeref;
3201	u32 reg, shift;
3202
3203	with_intel_runtime_pm(to_gt(i915)->uncore->rpm, wakeref)
3204	reg = intel_uncore_read(uncore: to_gt(i915)->uncore, RPM_CONFIG0);
3205
3206	shift = REG_FIELD_GET(GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK,
3207	reg);
3208
3209	return to_gt(i915)->clock_frequency << (3 - shift);
3210	}
3211
3212	return to_gt(i915)->clock_frequency;
3213	}
3214
3215	/**
3216	* i915_oa_stream_init - validate combined props for OA stream and init
3217	* @stream: An i915 perf stream
3218	* @param: The open parameters passed to `DRM_I915_PERF_OPEN`
3219	* @props: The property state that configures stream (individually validated)
3220	*
3221	* While read_properties_unlocked() validates properties in isolation it
3222	* doesn't ensure that the combination necessarily makes sense.
3223	*
3224	* At this point it has been determined that userspace wants a stream of
3225	* OA metrics, but still we need to further validate the combined
3226	* properties are OK.
3227	*
3228	* If the configuration makes sense then we can allocate memory for
3229	* a circular OA buffer and apply the requested metric set configuration.
3230	*
3231	* Returns: zero on success or a negative error code.
3232	*/
3233	static int i915_oa_stream_init(struct i915_perf_stream *stream,
3234	struct drm_i915_perf_open_param *param,
3235	struct perf_open_properties *props)
3236	{
3237	struct drm_i915_private *i915 = stream->perf->i915;
3238	struct i915_perf *perf = stream->perf;
3239	struct i915_perf_group *g;
3240	int ret;
3241
3242	if (!props->engine) {
3243	drm_dbg(&stream->perf->i915->drm,
3244	"OA engine not specified\n");
3245	return -EINVAL;
3246	}
3247	g = props->engine->oa_group;
3248
3249	/*
3250	* If the sysfs metrics/ directory wasn't registered for some
3251	* reason then don't let userspace try their luck with config
3252	* IDs
3253	*/
3254	if (!perf->metrics_kobj) {
3255	drm_dbg(&stream->perf->i915->drm,
3256	"OA metrics weren't advertised via sysfs\n");
3257	return -EINVAL;
3258	}
3259
3260	if (!(props->sample_flags & SAMPLE_OA_REPORT) &&
3261	(GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) {
3262	drm_dbg(&stream->perf->i915->drm,
3263	"Only OA report sampling supported\n");
3264	return -EINVAL;
3265	}
3266
3267	if (!perf->ops.enable_metric_set) {
3268	drm_dbg(&stream->perf->i915->drm,
3269	"OA unit not supported\n");
3270	return -ENODEV;
3271	}
3272
3273	/*
3274	* To avoid the complexity of having to accurately filter
3275	* counter reports and marshal to the appropriate client
3276	* we currently only allow exclusive access
3277	*/
3278	if (g->exclusive_stream) {
3279	drm_dbg(&stream->perf->i915->drm,
3280	"OA unit already in use\n");
3281	return -EBUSY;
3282	}
3283
3284	if (!props->oa_format) {
3285	drm_dbg(&stream->perf->i915->drm,
3286	"OA report format not specified\n");
3287	return -EINVAL;
3288	}
3289
3290	stream->engine = props->engine;
3291	stream->uncore = stream->engine->gt->uncore;
3292
3293	stream->sample_size = sizeof(struct drm_i915_perf_record_header);
3294
3295	stream->oa_buffer.format = &perf->oa_formats[props->oa_format];
3296	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0))
3297	return -EINVAL;
3298
3299	stream->sample_flags = props->sample_flags;
3300	stream->sample_size += stream->oa_buffer.format->size;
3301
3302	stream->hold_preemption = props->hold_preemption;
3303
3304	stream->periodic = props->oa_periodic;
3305	if (stream->periodic)
3306	stream->period_exponent = props->oa_period_exponent;
3307
3308	if (stream->ctx) {
3309	ret = oa_get_render_ctx_id(stream);
3310	if (ret) {
3311	drm_dbg(&stream->perf->i915->drm,
3312	"Invalid context id to filter with\n");
3313	return ret;
3314	}
3315	}
3316
3317	ret = alloc_noa_wait(stream);
3318	if (ret) {
3319	drm_dbg(&stream->perf->i915->drm,
3320	"Unable to allocate NOA wait batch buffer\n");
3321	goto err_noa_wait_alloc;
3322	}
3323
3324	stream->oa_config = i915_perf_get_oa_config(perf, metrics_set: props->metrics_set);
3325	if (!stream->oa_config) {
3326	drm_dbg(&stream->perf->i915->drm,
3327	"Invalid OA config id=%i\n", props->metrics_set);
3328	ret = -EINVAL;
3329	goto err_config;
3330	}
3331
3332	/* PRM - observability performance counters:
3333	*
3334	* OACONTROL, performance counter enable, note:
3335	*
3336	* "When this bit is set, in order to have coherent counts,
3337	* RC6 power state and trunk clock gating must be disabled.
3338	* This can be achieved by programming MMIO registers as
3339	* 0xA094=0 and 0xA090[31]=1"
3340	*
3341	* In our case we are expecting that taking pm + FORCEWAKE
3342	* references will effectively disable RC6.
3343	*/
3344	intel_engine_pm_get(engine: stream->engine);
3345	intel_uncore_forcewake_get(uncore: stream->uncore, domains: FORCEWAKE_ALL);
3346
3347	ret = alloc_oa_buffer(stream);
3348	if (ret)
3349	goto err_oa_buf_alloc;
3350
3351	stream->ops = &i915_oa_stream_ops;
3352
3353	stream->engine->gt->perf.sseu = props->sseu;
3354	WRITE_ONCE(g->exclusive_stream, stream);
3355
3356	ret = i915_perf_stream_enable_sync(stream);
3357	if (ret) {
3358	drm_dbg(&stream->perf->i915->drm,
3359	"Unable to enable metric set\n");
3360	goto err_enable;
3361	}
3362
3363	drm_dbg(&stream->perf->i915->drm,
3364	"opening stream oa config uuid=%s\n",
3365	stream->oa_config->uuid);
3366
3367	hrtimer_init(timer: &stream->poll_check_timer,
3368	CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
3369	stream->poll_check_timer.function = oa_poll_check_timer_cb;
3370	init_waitqueue_head(&stream->poll_wq);
3371	spin_lock_init(&stream->oa_buffer.ptr_lock);
3372	mutex_init(&stream->lock);
3373
3374	return 0;
3375
3376	err_enable:
3377	WRITE_ONCE(g->exclusive_stream, NULL);
3378	perf->ops.disable_metric_set(stream);
3379
3380	free_oa_buffer(stream);
3381
3382	err_oa_buf_alloc:
3383	intel_uncore_forcewake_put(uncore: stream->uncore, domains: FORCEWAKE_ALL);
3384	intel_engine_pm_put(engine: stream->engine);
3385
3386	free_oa_configs(stream);
3387
3388	err_config:
3389	free_noa_wait(stream);
3390
3391	err_noa_wait_alloc:
3392	if (stream->ctx)
3393	oa_put_render_ctx_id(stream);
3394
3395	return ret;
3396	}
3397
3398	void i915_oa_init_reg_state(const struct intel_context *ce,
3399	const struct intel_engine_cs *engine)
3400	{
3401	struct i915_perf_stream *stream;
3402
3403	if (engine->class != RENDER_CLASS)
3404	return;
3405
3406	/* perf.exclusive_stream serialised by lrc_configure_all_contexts() */
3407	stream = READ_ONCE(engine->oa_group->exclusive_stream);
3408	if (stream && GRAPHICS_VER(stream->perf->i915) < 12)
3409	gen8_update_reg_state_unlocked(ce, stream);
3410	}
3411
3412	/**
3413	* i915_perf_read - handles read() FOP for i915 perf stream FDs
3414	* @file: An i915 perf stream file
3415	* @buf: destination buffer given by userspace
3416	* @count: the number of bytes userspace wants to read
3417	* @ppos: (inout) file seek position (unused)
3418	*
3419	* The entry point for handling a read() on a stream file descriptor from
3420	* userspace. Most of the work is left to the i915_perf_read_locked() and
3421	* &i915_perf_stream_ops->read but to save having stream implementations (of
3422	* which we might have multiple later) we handle blocking read here.
3423	*
3424	* We can also consistently treat trying to read from a disabled stream
3425	* as an IO error so implementations can assume the stream is enabled
3426	* while reading.
3427	*
3428	* Returns: The number of bytes copied or a negative error code on failure.
3429	*/
3430	static ssize_t i915_perf_read(struct file *file,
3431	char __user *buf,
3432	size_t count,
3433	loff_t *ppos)
3434	{
3435	struct i915_perf_stream *stream = file->private_data;
3436	size_t offset = 0;
3437	int ret;
3438
3439	/* To ensure it's handled consistently we simply treat all reads of a
3440	* disabled stream as an error. In particular it might otherwise lead
3441	* to a deadlock for blocking file descriptors...
3442	*/
3443	if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))
3444	return -EIO;
3445
3446	if (!(file->f_flags & O_NONBLOCK)) {
3447	/* There's the small chance of false positives from
3448	* stream->ops->wait_unlocked.
3449	*
3450	* E.g. with single context filtering since we only wait until
3451	* oabuffer has >= 1 report we don't immediately know whether
3452	* any reports really belong to the current context
3453	*/
3454	do {
3455	ret = stream->ops->wait_unlocked(stream);
3456	if (ret)
3457	return ret;
3458
3459	mutex_lock(&stream->lock);
3460	ret = stream->ops->read(stream, buf, count, &offset);
3461	mutex_unlock(lock: &stream->lock);
3462	} while (!offset && !ret);
3463	} else {
3464	mutex_lock(&stream->lock);
3465	ret = stream->ops->read(stream, buf, count, &offset);
3466	mutex_unlock(lock: &stream->lock);
3467	}
3468
3469	/* We allow the poll checking to sometimes report false positive EPOLLIN
3470	* events where we might actually report EAGAIN on read() if there's
3471	* not really any data available. In this situation though we don't
3472	* want to enter a busy loop between poll() reporting a EPOLLIN event
3473	* and read() returning -EAGAIN. Clearing the oa.pollin state here
3474	* effectively ensures we back off until the next hrtimer callback
3475	* before reporting another EPOLLIN event.
3476	* The exception to this is if ops->read() returned -ENOSPC which means
3477	* that more OA data is available than could fit in the user provided
3478	* buffer. In this case we want the next poll() call to not block.
3479	*/
3480	if (ret != -ENOSPC)
3481	stream->pollin = false;
3482
3483	/* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */
3484	return offset ?: (ret ?: -EAGAIN);
3485	}
3486
3487	static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
3488	{
3489	struct i915_perf_stream *stream =
3490	container_of(hrtimer, typeof(*stream), poll_check_timer);
3491
3492	if (oa_buffer_check_unlocked(stream)) {
3493	stream->pollin = true;
3494	wake_up(&stream->poll_wq);
3495	}
3496
3497	hrtimer_forward_now(timer: hrtimer,
3498	interval: ns_to_ktime(ns: stream->poll_oa_period));
3499
3500	return HRTIMER_RESTART;
3501	}
3502
3503	/**
3504	* i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3505	* @stream: An i915 perf stream
3506	* @file: An i915 perf stream file
3507	* @wait: poll() state table
3508	*
3509	* For handling userspace polling on an i915 perf stream, this calls through to
3510	* &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
3511	* will be woken for new stream data.
3512	*
3513	* Returns: any poll events that are ready without sleeping
3514	*/
3515	static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3516	struct file *file,
3517	poll_table *wait)
3518	{
3519	__poll_t events = 0;
3520
3521	stream->ops->poll_wait(stream, file, wait);
3522
3523	/* Note: we don't explicitly check whether there's something to read
3524	* here since this path may be very hot depending on what else
3525	* userspace is polling, or on the timeout in use. We rely solely on
3526	* the hrtimer/oa_poll_check_timer_cb to notify us when there are
3527	* samples to read.
3528	*/
3529	if (stream->pollin)
3530	events |= EPOLLIN;
3531
3532	return events;
3533	}
3534
3535	/**
3536	* i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3537	* @file: An i915 perf stream file
3538	* @wait: poll() state table
3539	*
3540	* For handling userspace polling on an i915 perf stream, this ensures
3541	* poll_wait() gets called with a wait queue that will be woken for new stream
3542	* data.
3543	*
3544	* Note: Implementation deferred to i915_perf_poll_locked()
3545	*
3546	* Returns: any poll events that are ready without sleeping
3547	*/
3548	static __poll_t i915_perf_poll(struct file *file, poll_table *wait)
3549	{
3550	struct i915_perf_stream *stream = file->private_data;
3551	__poll_t ret;
3552
3553	mutex_lock(&stream->lock);
3554	ret = i915_perf_poll_locked(stream, file, wait);
3555	mutex_unlock(lock: &stream->lock);
3556
3557	return ret;
3558	}
3559
3560	/**
3561	* i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
3562	* @stream: A disabled i915 perf stream
3563	*
3564	* [Re]enables the associated capture of data for this stream.
3565	*
3566	* If a stream was previously enabled then there's currently no intention
3567	* to provide userspace any guarantee about the preservation of previously
3568	* buffered data.
3569	*/
3570	static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3571	{
3572	if (stream->enabled)
3573	return;
3574
3575	/* Allow stream->ops->enable() to refer to this */
3576	stream->enabled = true;
3577
3578	if (stream->ops->enable)
3579	stream->ops->enable(stream);
3580
3581	if (stream->hold_preemption)
3582	intel_context_set_nopreempt(ce: stream->pinned_ctx);
3583	}
3584
3585	/**
3586	* i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
3587	* @stream: An enabled i915 perf stream
3588	*
3589	* Disables the associated capture of data for this stream.
3590	*
3591	* The intention is that disabling an re-enabling a stream will ideally be
3592	* cheaper than destroying and re-opening a stream with the same configuration,
3593	* though there are no formal guarantees about what state or buffered data
3594	* must be retained between disabling and re-enabling a stream.
3595	*
3596	* Note: while a stream is disabled it's considered an error for userspace
3597	* to attempt to read from the stream (-EIO).
3598	*/
3599	static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3600	{
3601	if (!stream->enabled)
3602	return;
3603
3604	/* Allow stream->ops->disable() to refer to this */
3605	stream->enabled = false;
3606
3607	if (stream->hold_preemption)
3608	intel_context_clear_nopreempt(ce: stream->pinned_ctx);
3609
3610	if (stream->ops->disable)
3611	stream->ops->disable(stream);
3612	}
3613
3614	static long i915_perf_config_locked(struct i915_perf_stream *stream,
3615	unsigned long metrics_set)
3616	{
3617	struct i915_oa_config *config;
3618	long ret = stream->oa_config->id;
3619
3620	config = i915_perf_get_oa_config(perf: stream->perf, metrics_set);
3621	if (!config)
3622	return -EINVAL;
3623
3624	if (config != stream->oa_config) {
3625	int err;
3626
3627	/*
3628	* If OA is bound to a specific context, emit the
3629	* reconfiguration inline from that context. The update
3630	* will then be ordered with respect to submission on that
3631	* context.
3632	*
3633	* When set globally, we use a low priority kernel context,
3634	* so it will effectively take effect when idle.
3635	*/
3636	err = emit_oa_config(stream, oa_config: config, ce: oa_context(stream), NULL);
3637	if (!err)
3638	config = xchg(&stream->oa_config, config);
3639	else
3640	ret = err;
3641	}
3642
3643	i915_oa_config_put(oa_config: config);
3644
3645	return ret;
3646	}
3647
3648	/**
3649	* i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3650	* @stream: An i915 perf stream
3651	* @cmd: the ioctl request
3652	* @arg: the ioctl data
3653	*
3654	* Returns: zero on success or a negative error code. Returns -EINVAL for
3655	* an unknown ioctl request.
3656	*/
3657	static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3658	unsigned int cmd,
3659	unsigned long arg)
3660	{
3661	switch (cmd) {
3662	case I915_PERF_IOCTL_ENABLE:
3663	i915_perf_enable_locked(stream);
3664	return 0;
3665	case I915_PERF_IOCTL_DISABLE:
3666	i915_perf_disable_locked(stream);
3667	return 0;
3668	case I915_PERF_IOCTL_CONFIG:
3669	return i915_perf_config_locked(stream, metrics_set: arg);
3670	}
3671
3672	return -EINVAL;
3673	}
3674
3675	/**
3676	* i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3677	* @file: An i915 perf stream file
3678	* @cmd: the ioctl request
3679	* @arg: the ioctl data
3680	*
3681	* Implementation deferred to i915_perf_ioctl_locked().
3682	*
3683	* Returns: zero on success or a negative error code. Returns -EINVAL for
3684	* an unknown ioctl request.
3685	*/
3686	static long i915_perf_ioctl(struct file *file,
3687	unsigned int cmd,
3688	unsigned long arg)
3689	{
3690	struct i915_perf_stream *stream = file->private_data;
3691	long ret;
3692
3693	mutex_lock(&stream->lock);
3694	ret = i915_perf_ioctl_locked(stream, cmd, arg);
3695	mutex_unlock(lock: &stream->lock);
3696
3697	return ret;
3698	}
3699
3700	/**
3701	* i915_perf_destroy_locked - destroy an i915 perf stream
3702	* @stream: An i915 perf stream
3703	*
3704	* Frees all resources associated with the given i915 perf @stream, disabling
3705	* any associated data capture in the process.
3706	*
3707	* Note: The &gt->perf.lock mutex has been taken to serialize
3708	* with any non-file-operation driver hooks.
3709	*/
3710	static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3711	{
3712	if (stream->enabled)
3713	i915_perf_disable_locked(stream);
3714
3715	if (stream->ops->destroy)
3716	stream->ops->destroy(stream);
3717
3718	if (stream->ctx)
3719	i915_gem_context_put(ctx: stream->ctx);
3720
3721	kfree(objp: stream);
3722	}
3723
3724	/**
3725	* i915_perf_release - handles userspace close() of a stream file
3726	* @inode: anonymous inode associated with file
3727	* @file: An i915 perf stream file
3728	*
3729	* Cleans up any resources associated with an open i915 perf stream file.
3730	*
3731	* NB: close() can't really fail from the userspace point of view.
3732	*
3733	* Returns: zero on success or a negative error code.
3734	*/
3735	static int i915_perf_release(struct inode *inode, struct file *file)
3736	{
3737	struct i915_perf_stream *stream = file->private_data;
3738	struct i915_perf *perf = stream->perf;
3739	struct intel_gt *gt = stream->engine->gt;
3740
3741	/*
3742	* Within this call, we know that the fd is being closed and we have no
3743	* other user of stream->lock. Use the perf lock to destroy the stream
3744	* here.
3745	*/
3746	mutex_lock(&gt->perf.lock);
3747	i915_perf_destroy_locked(stream);
3748	mutex_unlock(lock: &gt->perf.lock);
3749
3750	/* Release the reference the perf stream kept on the driver. */
3751	drm_dev_put(dev: &perf->i915->drm);
3752
3753	return 0;
3754	}
3755
3756
3757	static const struct file_operations fops = {
3758	.owner = THIS_MODULE,
3759	.llseek = no_llseek,
3760	.release = i915_perf_release,
3761	.poll = i915_perf_poll,
3762	.read = i915_perf_read,
3763	.unlocked_ioctl = i915_perf_ioctl,
3764	/* Our ioctl have no arguments, so it's safe to use the same function
3765	* to handle 32bits compatibility.
3766	*/
3767	.compat_ioctl = i915_perf_ioctl,
3768	};
3769
3770
3771	/**
3772	* i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3773	* @perf: i915 perf instance
3774	* @param: The open parameters passed to 'DRM_I915_PERF_OPEN`
3775	* @props: individually validated u64 property value pairs
3776	* @file: drm file
3777	*
3778	* See i915_perf_ioctl_open() for interface details.
3779	*
3780	* Implements further stream config validation and stream initialization on
3781	* behalf of i915_perf_open_ioctl() with the &gt->perf.lock mutex
3782	* taken to serialize with any non-file-operation driver hooks.
3783	*
3784	* Note: at this point the @props have only been validated in isolation and
3785	* it's still necessary to validate that the combination of properties makes
3786	* sense.
3787	*
3788	* In the case where userspace is interested in OA unit metrics then further
3789	* config validation and stream initialization details will be handled by
3790	* i915_oa_stream_init(). The code here should only validate config state that
3791	* will be relevant to all stream types / backends.
3792	*
3793	* Returns: zero on success or a negative error code.
3794	*/
3795	static int
3796	i915_perf_open_ioctl_locked(struct i915_perf *perf,
3797	struct drm_i915_perf_open_param *param,
3798	struct perf_open_properties *props,
3799	struct drm_file *file)
3800	{
3801	struct i915_gem_context *specific_ctx = NULL;
3802	struct i915_perf_stream *stream = NULL;
3803	unsigned long f_flags = 0;
3804	bool privileged_op = true;
3805	int stream_fd;
3806	int ret;
3807
3808	if (props->single_context) {
3809	u32 ctx_handle = props->ctx_handle;
3810	struct drm_i915_file_private *file_priv = file->driver_priv;
3811
3812	specific_ctx = i915_gem_context_lookup(file_priv, id: ctx_handle);
3813	if (IS_ERR(ptr: specific_ctx)) {
3814	drm_dbg(&perf->i915->drm,
3815	"Failed to look up context with ID %u for opening perf stream\n",
3816	ctx_handle);
3817	ret = PTR_ERR(ptr: specific_ctx);
3818	goto err;
3819	}
3820	}
3821
3822	/*
3823	* On Haswell the OA unit supports clock gating off for a specific
3824	* context and in this mode there's no visibility of metrics for the
3825	* rest of the system, which we consider acceptable for a
3826	* non-privileged client.
3827	*
3828	* For Gen8->11 the OA unit no longer supports clock gating off for a
3829	* specific context and the kernel can't securely stop the counters
3830	* from updating as system-wide / global values. Even though we can
3831	* filter reports based on the included context ID we can't block
3832	* clients from seeing the raw / global counter values via
3833	* MI_REPORT_PERF_COUNT commands and so consider it a privileged op to
3834	* enable the OA unit by default.
3835	*
3836	* For Gen12+ we gain a new OAR unit that only monitors the RCS on a
3837	* per context basis. So we can relax requirements there if the user
3838	* doesn't request global stream access (i.e. query based sampling
3839	* using MI_RECORD_PERF_COUNT.
3840	*/
3841	if (IS_HASWELL(perf->i915) && specific_ctx)
3842	privileged_op = false;
3843	else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx &&
3844	(props->sample_flags & SAMPLE_OA_REPORT) == 0)
3845	privileged_op = false;
3846
3847	if (props->hold_preemption) {
3848	if (!props->single_context) {
3849	drm_dbg(&perf->i915->drm,
3850	"preemption disable with no context\n");
3851	ret = -EINVAL;
3852	goto err;
3853	}
3854	privileged_op = true;
3855	}
3856
3857	/*
3858	* Asking for SSEU configuration is a priviliged operation.
3859	*/
3860	if (props->has_sseu)
3861	privileged_op = true;
3862	else
3863	get_default_sseu_config(out_sseu: &props->sseu, engine: props->engine);
3864
3865	/* Similar to perf's kernel.perf_paranoid_cpu sysctl option
3866	* we check a dev.i915.perf_stream_paranoid sysctl option
3867	* to determine if it's ok to access system wide OA counters
3868	* without CAP_PERFMON or CAP_SYS_ADMIN privileges.
3869	*/
3870	if (privileged_op &&
3871	i915_perf_stream_paranoid && !perfmon_capable()) {
3872	drm_dbg(&perf->i915->drm,
3873	"Insufficient privileges to open i915 perf stream\n");
3874	ret = -EACCES;
3875	goto err_ctx;
3876	}
3877
3878	stream = kzalloc(size: sizeof(*stream), GFP_KERNEL);
3879	if (!stream) {
3880	ret = -ENOMEM;
3881	goto err_ctx;
3882	}
3883
3884	stream->perf = perf;
3885	stream->ctx = specific_ctx;
3886	stream->poll_oa_period = props->poll_oa_period;
3887
3888	ret = i915_oa_stream_init(stream, param, props);
3889	if (ret)
3890	goto err_alloc;
3891
3892	/* we avoid simply assigning stream->sample_flags = props->sample_flags
3893	* to have _stream_init check the combination of sample flags more
3894	* thoroughly, but still this is the expected result at this point.
3895	*/
3896	if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3897	ret = -ENODEV;
3898	goto err_flags;
3899	}
3900
3901	if (param->flags & I915_PERF_FLAG_FD_CLOEXEC)
3902	f_flags |= O_CLOEXEC;
3903	if (param->flags & I915_PERF_FLAG_FD_NONBLOCK)
3904	f_flags |= O_NONBLOCK;
3905
3906	stream_fd = anon_inode_getfd(name: "[i915_perf]", fops: &fops, priv: stream, flags: f_flags);
3907	if (stream_fd < 0) {
3908	ret = stream_fd;
3909	goto err_flags;
3910	}
3911
3912	if (!(param->flags & I915_PERF_FLAG_DISABLED))
3913	i915_perf_enable_locked(stream);
3914
3915	/* Take a reference on the driver that will be kept with stream_fd
3916	* until its release.
3917	*/
3918	drm_dev_get(dev: &perf->i915->drm);
3919
3920	return stream_fd;
3921
3922	err_flags:
3923	if (stream->ops->destroy)
3924	stream->ops->destroy(stream);
3925	err_alloc:
3926	kfree(objp: stream);
3927	err_ctx:
3928	if (specific_ctx)
3929	i915_gem_context_put(ctx: specific_ctx);
3930	err:
3931	return ret;
3932	}
3933
3934	static u64 oa_exponent_to_ns(struct i915_perf *perf, int exponent)
3935	{
3936	u64 nom = (2ULL << exponent) * NSEC_PER_SEC;
3937	u32 den = i915_perf_oa_timestamp_frequency(i915: perf->i915);
3938
3939	return div_u64(dividend: nom + den - 1, divisor: den);
3940	}
3941
3942	static __always_inline bool
3943	oa_format_valid(struct i915_perf *perf, enum drm_i915_oa_format format)
3944	{
3945	return test_bit(format, perf->format_mask);
3946	}
3947
3948	static __always_inline void
3949	oa_format_add(struct i915_perf *perf, enum drm_i915_oa_format format)
3950	{
3951	__set_bit(format, perf->format_mask);
3952	}
3953
3954	/**
3955	* read_properties_unlocked - validate + copy userspace stream open properties
3956	* @perf: i915 perf instance
3957	* @uprops: The array of u64 key value pairs given by userspace
3958	* @n_props: The number of key value pairs expected in @uprops
3959	* @props: The stream configuration built up while validating properties
3960	*
3961	* Note this function only validates properties in isolation it doesn't
3962	* validate that the combination of properties makes sense or that all
3963	* properties necessary for a particular kind of stream have been set.
3964	*
3965	* Note that there currently aren't any ordering requirements for properties so
3966	* we shouldn't validate or assume anything about ordering here. This doesn't
3967	* rule out defining new properties with ordering requirements in the future.
3968	*/
3969	static int read_properties_unlocked(struct i915_perf *perf,
3970	u64 __user *uprops,
3971	u32 n_props,
3972	struct perf_open_properties *props)
3973	{
3974	struct drm_i915_gem_context_param_sseu user_sseu;
3975	const struct i915_oa_format *f;
3976	u64 __user *uprop = uprops;
3977	bool config_instance = false;
3978	bool config_class = false;
3979	bool config_sseu = false;
3980	u8 class, instance;
3981	u32 i;
3982	int ret;
3983
3984	memset(props, 0, sizeof(struct perf_open_properties));
3985	props->poll_oa_period = DEFAULT_POLL_PERIOD_NS;
3986
3987	/* Considering that ID = 0 is reserved and assuming that we don't
3988	* (currently) expect any configurations to ever specify duplicate
3989	* values for a particular property ID then the last _PROP_MAX value is
3990	* one greater than the maximum number of properties we expect to get
3991	* from userspace.
3992	*/
3993	if (!n_props || n_props >= DRM_I915_PERF_PROP_MAX) {
3994	drm_dbg(&perf->i915->drm,
3995	"Invalid number of i915 perf properties given\n");
3996	return -EINVAL;
3997	}
3998
3999	/* Defaults when class:instance is not passed */
4000	class = I915_ENGINE_CLASS_RENDER;
4001	instance = 0;
4002
4003	for (i = 0; i < n_props; i++) {
4004	u64 oa_period, oa_freq_hz;
4005	u64 id, value;
4006
4007	ret = get_user(id, uprop);
4008	if (ret)
4009	return ret;
4010
4011	ret = get_user(value, uprop + 1);
4012	if (ret)
4013	return ret;
4014
4015	if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) {
4016	drm_dbg(&perf->i915->drm,
4017	"Unknown i915 perf property ID\n");
4018	return -EINVAL;
4019	}
4020
4021	switch ((enum drm_i915_perf_property_id)id) {
4022	case DRM_I915_PERF_PROP_CTX_HANDLE:
4023	props->single_context = 1;
4024	props->ctx_handle = value;
4025	break;
4026	case DRM_I915_PERF_PROP_SAMPLE_OA:
4027	if (value)
4028	props->sample_flags |= SAMPLE_OA_REPORT;
4029	break;
4030	case DRM_I915_PERF_PROP_OA_METRICS_SET:
4031	if (value == 0) {
4032	drm_dbg(&perf->i915->drm,
4033	"Unknown OA metric set ID\n");
4034	return -EINVAL;
4035	}
4036	props->metrics_set = value;
4037	break;
4038	case DRM_I915_PERF_PROP_OA_FORMAT:
4039	if (value == 0 || value >= I915_OA_FORMAT_MAX) {
4040	drm_dbg(&perf->i915->drm,
4041	"Out-of-range OA report format %llu\n",
4042	value);
4043	return -EINVAL;
4044	}
4045	if (!oa_format_valid(perf, format: value)) {
4046	drm_dbg(&perf->i915->drm,
4047	"Unsupported OA report format %llu\n",
4048	value);
4049	return -EINVAL;
4050	}
4051	props->oa_format = value;
4052	break;
4053	case DRM_I915_PERF_PROP_OA_EXPONENT:
4054	if (value > OA_EXPONENT_MAX) {
4055	drm_dbg(&perf->i915->drm,
4056	"OA timer exponent too high (> %u)\n",
4057	OA_EXPONENT_MAX);
4058	return -EINVAL;
4059	}
4060
4061	/* Theoretically we can program the OA unit to sample
4062	* e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns
4063	* for BXT. We don't allow such high sampling
4064	* frequencies by default unless root.
4065	*/
4066
4067	BUILD_BUG_ON(sizeof(oa_period) != 8);
4068	oa_period = oa_exponent_to_ns(perf, exponent: value);
4069
4070	/* This check is primarily to ensure that oa_period <=
4071	* UINT32_MAX (before passing to do_div which only
4072	* accepts a u32 denominator), but we can also skip
4073	* checking anything < 1Hz which implicitly can't be
4074	* limited via an integer oa_max_sample_rate.
4075	*/
4076	if (oa_period <= NSEC_PER_SEC) {
4077	u64 tmp = NSEC_PER_SEC;
4078	do_div(tmp, oa_period);
4079	oa_freq_hz = tmp;
4080	} else
4081	oa_freq_hz = 0;
4082
4083	if (oa_freq_hz > i915_oa_max_sample_rate && !perfmon_capable()) {
4084	drm_dbg(&perf->i915->drm,
4085	"OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without CAP_PERFMON or CAP_SYS_ADMIN privileges\n",
4086	i915_oa_max_sample_rate);
4087	return -EACCES;
4088	}
4089
4090	props->oa_periodic = true;
4091	props->oa_period_exponent = value;
4092	break;
4093	case DRM_I915_PERF_PROP_HOLD_PREEMPTION:
4094	props->hold_preemption = !!value;
4095	break;
4096	case DRM_I915_PERF_PROP_GLOBAL_SSEU: {
4097	if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 50)) {
4098	drm_dbg(&perf->i915->drm,
4099	"SSEU config not supported on gfx %x\n",
4100	GRAPHICS_VER_FULL(perf->i915));
4101	return -ENODEV;
4102	}
4103
4104	if (copy_from_user(to: &user_sseu,
4105	u64_to_user_ptr(value),
4106	n: sizeof(user_sseu))) {
4107	drm_dbg(&perf->i915->drm,
4108	"Unable to copy global sseu parameter\n");
4109	return -EFAULT;
4110	}
4111	config_sseu = true;
4112	break;
4113	}
4114	case DRM_I915_PERF_PROP_POLL_OA_PERIOD:
4115	if (value < 100000 /* 100us */) {
4116	drm_dbg(&perf->i915->drm,
4117	"OA availability timer too small (%lluns < 100us)\n",
4118	value);
4119	return -EINVAL;
4120	}
4121	props->poll_oa_period = value;
4122	break;
4123	case DRM_I915_PERF_PROP_OA_ENGINE_CLASS:
4124	class = (u8)value;
4125	config_class = true;
4126	break;
4127	case DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE:
4128	instance = (u8)value;
4129	config_instance = true;
4130	break;
4131	default:
4132	MISSING_CASE(id);
4133	return -EINVAL;
4134	}
4135
4136	uprop += 2;
4137	}
4138
4139	if ((config_class && !config_instance) ||
4140	(config_instance && !config_class)) {
4141	drm_dbg(&perf->i915->drm,
4142	"OA engine-class and engine-instance parameters must be passed together\n");
4143	return -EINVAL;
4144	}
4145
4146	props->engine = intel_engine_lookup_user(i915: perf->i915, class, instance);
4147	if (!props->engine) {
4148	drm_dbg(&perf->i915->drm,
4149	"OA engine class and instance invalid %d:%d\n",
4150	class, instance);
4151	return -EINVAL;
4152	}
4153
4154	if (!engine_supports_oa(engine: props->engine)) {
4155	drm_dbg(&perf->i915->drm,
4156	"Engine not supported by OA %d:%d\n",
4157	class, instance);
4158	return -EINVAL;
4159	}
4160
4161	/*
4162	* Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
4163	* C6 disable in BIOS. Fail if Media C6 is enabled on steppings where OAM
4164	* does not work as expected.
4165	*/
4166	if (IS_MEDIA_GT_IP_STEP(props->engine->gt, IP_VER(13, 0), STEP_A0, STEP_C0) &&
4167	props->engine->oa_group->type == TYPE_OAM &&
4168	intel_check_bios_c6_setup(rc6: &props->engine->gt->rc6)) {
4169	drm_dbg(&perf->i915->drm,
4170	"OAM requires media C6 to be disabled in BIOS\n");
4171	return -EINVAL;
4172	}
4173
4174	i = array_index_nospec(props->oa_format, I915_OA_FORMAT_MAX);
4175	f = &perf->oa_formats[i];
4176	if (!engine_supports_oa_format(engine: props->engine, type: f->type)) {
4177	drm_dbg(&perf->i915->drm,
4178	"Invalid OA format %d for class %d\n",
4179	f->type, props->engine->class);
4180	return -EINVAL;
4181	}
4182
4183	if (config_sseu) {
4184	ret = get_sseu_config(out_sseu: &props->sseu, engine: props->engine, drm_sseu: &user_sseu);
4185	if (ret) {
4186	drm_dbg(&perf->i915->drm,
4187	"Invalid SSEU configuration\n");
4188	return ret;
4189	}
4190	props->has_sseu = true;
4191	}
4192
4193	return 0;
4194	}
4195
4196	/**
4197	* i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
4198	* @dev: drm device
4199	* @data: ioctl data copied from userspace (unvalidated)
4200	* @file: drm file
4201	*
4202	* Validates the stream open parameters given by userspace including flags
4203	* and an array of u64 key, value pair properties.
4204	*
4205	* Very little is assumed up front about the nature of the stream being
4206	* opened (for instance we don't assume it's for periodic OA unit metrics). An
4207	* i915-perf stream is expected to be a suitable interface for other forms of
4208	* buffered data written by the GPU besides periodic OA metrics.
4209	*
4210	* Note we copy the properties from userspace outside of the i915 perf
4211	* mutex to avoid an awkward lockdep with mmap_lock.
4212	*
4213	* Most of the implementation details are handled by
4214	* i915_perf_open_ioctl_locked() after taking the &gt->perf.lock
4215	* mutex for serializing with any non-file-operation driver hooks.
4216	*
4217	* Return: A newly opened i915 Perf stream file descriptor or negative
4218	* error code on failure.
4219	*/
4220	int i915_perf_open_ioctl(struct drm_device *dev, void *data,
4221	struct drm_file *file)
4222	{
4223	struct i915_perf *perf = &to_i915(dev)->perf;
4224	struct drm_i915_perf_open_param *param = data;
4225	struct intel_gt *gt;
4226	struct perf_open_properties props;
4227	u32 known_open_flags;
4228	int ret;
4229
4230	if (!perf->i915) {
4231	drm_dbg(&perf->i915->drm,
4232	"i915 perf interface not available for this system\n");
4233	return -ENOTSUPP;
4234	}
4235
4236	known_open_flags = I915_PERF_FLAG_FD_CLOEXEC |
4237	I915_PERF_FLAG_FD_NONBLOCK |
4238	I915_PERF_FLAG_DISABLED;
4239	if (param->flags & ~known_open_flags) {
4240	drm_dbg(&perf->i915->drm,
4241	"Unknown drm_i915_perf_open_param flag\n");
4242	return -EINVAL;
4243	}
4244
4245	ret = read_properties_unlocked(perf,
4246	u64_to_user_ptr(param->properties_ptr),
4247	n_props: param->num_properties,
4248	props: &props);
4249	if (ret)
4250	return ret;
4251
4252	gt = props.engine->gt;
4253
4254	mutex_lock(&gt->perf.lock);
4255	ret = i915_perf_open_ioctl_locked(perf, param, props: &props, file);
4256	mutex_unlock(lock: &gt->perf.lock);
4257
4258	return ret;
4259	}
4260
4261	/**
4262	* i915_perf_register - exposes i915-perf to userspace
4263	* @i915: i915 device instance
4264	*
4265	* In particular OA metric sets are advertised under a sysfs metrics/
4266	* directory allowing userspace to enumerate valid IDs that can be
4267	* used to open an i915-perf stream.
4268	*/
4269	void i915_perf_register(struct drm_i915_private *i915)
4270	{
4271	struct i915_perf *perf = &i915->perf;
4272	struct intel_gt *gt = to_gt(i915);
4273
4274	if (!perf->i915)
4275	return;
4276
4277	/* To be sure we're synchronized with an attempted
4278	* i915_perf_open_ioctl(); considering that we register after
4279	* being exposed to userspace.
4280	*/
4281	mutex_lock(&gt->perf.lock);
4282
4283	perf->metrics_kobj =
4284	kobject_create_and_add(name: "metrics",
4285	parent: &i915->drm.primary->kdev->kobj);
4286
4287	mutex_unlock(lock: &gt->perf.lock);
4288	}
4289
4290	/**
4291	* i915_perf_unregister - hide i915-perf from userspace
4292	* @i915: i915 device instance
4293	*
4294	* i915-perf state cleanup is split up into an 'unregister' and
4295	* 'deinit' phase where the interface is first hidden from
4296	* userspace by i915_perf_unregister() before cleaning up
4297	* remaining state in i915_perf_fini().
4298	*/
4299	void i915_perf_unregister(struct drm_i915_private *i915)
4300	{
4301	struct i915_perf *perf = &i915->perf;
4302
4303	if (!perf->metrics_kobj)
4304	return;
4305
4306	kobject_put(kobj: perf->metrics_kobj);
4307	perf->metrics_kobj = NULL;
4308	}
4309
4310	static bool gen8_is_valid_flex_addr(struct i915_perf *perf, u32 addr)
4311	{
4312	static const i915_reg_t flex_eu_regs[] = {
4313	EU_PERF_CNTL0,
4314	EU_PERF_CNTL1,
4315	EU_PERF_CNTL2,
4316	EU_PERF_CNTL3,
4317	EU_PERF_CNTL4,
4318	EU_PERF_CNTL5,
4319	EU_PERF_CNTL6,
4320	};
4321	int i;
4322
4323	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
4324	if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr)
4325	return true;
4326	}
4327	return false;
4328	}
4329
4330	static bool reg_in_range_table(u32 addr, const struct i915_range *table)
4331	{
4332	while (table->start || table->end) {
4333	if (addr >= table->start && addr <= table->end)
4334	return true;
4335
4336	table++;
4337	}
4338
4339	return false;
4340	}
4341
4342	#define REG_EQUAL(addr, mmio) \
4343	((addr) == i915_mmio_reg_offset(mmio))
4344
4345	static const struct i915_range gen7_oa_b_counters[] = {
4346	{ .start = 0x2710, .end = 0x272c }, /* OASTARTTRIG[1-8] */
4347	{ .start = 0x2740, .end = 0x275c }, /* OAREPORTTRIG[1-8] */
4348	{ .start = 0x2770, .end = 0x27ac }, /* OACEC[0-7][0-1] */
4349	{}
4350	};
4351
4352	static const struct i915_range gen12_oa_b_counters[] = {
4353	{ .start = 0x2b2c, .end = 0x2b2c }, /* GEN12_OAG_OA_PESS */
4354	{ .start = 0xd900, .end = 0xd91c }, /* GEN12_OAG_OASTARTTRIG[1-8] */
4355	{ .start = 0xd920, .end = 0xd93c }, /* GEN12_OAG_OAREPORTTRIG1[1-8] */
4356	{ .start = 0xd940, .end = 0xd97c }, /* GEN12_OAG_CEC[0-7][0-1] */
4357	{ .start = 0xdc00, .end = 0xdc3c }, /* GEN12_OAG_SCEC[0-7][0-1] */
4358	{ .start = 0xdc40, .end = 0xdc40 }, /* GEN12_OAG_SPCTR_CNF */
4359	{ .start = 0xdc44, .end = 0xdc44 }, /* GEN12_OAA_DBG_REG */
4360	{}
4361	};
4362
4363	static const struct i915_range mtl_oam_b_counters[] = {
4364	{ .start = 0x393000, .end = 0x39301c }, /* GEN12_OAM_STARTTRIG1[1-8] */
4365	{ .start = 0x393020, .end = 0x39303c }, /* GEN12_OAM_REPORTTRIG1[1-8] */
4366	{ .start = 0x393040, .end = 0x39307c }, /* GEN12_OAM_CEC[0-7][0-1] */
4367	{ .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */
4368	{}
4369	};
4370
4371	static const struct i915_range xehp_oa_b_counters[] = {
4372	{ .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */
4373	{ .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */
4374	{}
4375	};
4376
4377	static const struct i915_range gen7_oa_mux_regs[] = {
4378	{ .start = 0x91b8, .end = 0x91cc }, /* OA_PERFCNT[1-2], OA_PERFMATRIX */
4379	{ .start = 0x9800, .end = 0x9888 }, /* MICRO_BP0_0 - NOA_WRITE */
4380	{ .start = 0xe180, .end = 0xe180 }, /* HALF_SLICE_CHICKEN2 */
4381	{}
4382	};
4383
4384	static const struct i915_range hsw_oa_mux_regs[] = {
4385	{ .start = 0x09e80, .end = 0x09ea4 }, /* HSW_MBVID2_NOA[0-9] */
4386	{ .start = 0x09ec0, .end = 0x09ec0 }, /* HSW_MBVID2_MISR0 */
4387	{ .start = 0x25100, .end = 0x2ff90 },
4388	{}
4389	};
4390
4391	static const struct i915_range chv_oa_mux_regs[] = {
4392	{ .start = 0x182300, .end = 0x1823a4 },
4393	{}
4394	};
4395
4396	static const struct i915_range gen8_oa_mux_regs[] = {
4397	{ .start = 0x0d00, .end = 0x0d2c }, /* RPM_CONFIG[0-1], NOA_CONFIG[0-8] */
4398	{ .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */
4399	{}
4400	};
4401
4402	static const struct i915_range gen11_oa_mux_regs[] = {
4403	{ .start = 0x91c8, .end = 0x91dc }, /* OA_PERFCNT[3-4] */
4404	{}
4405	};
4406
4407	static const struct i915_range gen12_oa_mux_regs[] = {
4408	{ .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4409	{ .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4410	{ .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
4411	{ .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
4412	{ .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */
4413	{}
4414	};
4415
4416	/*
4417	* Ref: 14010536224:
4418	* 0x20cc is repurposed on MTL, so use a separate array for MTL.
4419	*/
4420	static const struct i915_range mtl_oa_mux_regs[] = {
4421	{ .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4422	{ .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4423	{ .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
4424	{ .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
4425	{ .start = 0x38d100, .end = 0x38d114}, /* VISACTL */
4426	{}
4427	};
4428
4429	static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4430	{
4431	return reg_in_range_table(addr, table: gen7_oa_b_counters);
4432	}
4433
4434	static bool gen8_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4435	{
4436	return reg_in_range_table(addr, table: gen7_oa_mux_regs) ||
4437	reg_in_range_table(addr, table: gen8_oa_mux_regs);
4438	}
4439
4440	static bool gen11_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4441	{
4442	return reg_in_range_table(addr, table: gen7_oa_mux_regs) ||
4443	reg_in_range_table(addr, table: gen8_oa_mux_regs) ||
4444	reg_in_range_table(addr, table: gen11_oa_mux_regs);
4445	}
4446
4447	static bool hsw_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4448	{
4449	return reg_in_range_table(addr, table: gen7_oa_mux_regs) ||
4450	reg_in_range_table(addr, table: hsw_oa_mux_regs);
4451	}
4452
4453	static bool chv_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4454	{
4455	return reg_in_range_table(addr, table: gen7_oa_mux_regs) ||
4456	reg_in_range_table(addr, table: chv_oa_mux_regs);
4457	}
4458
4459	static bool gen12_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4460	{
4461	return reg_in_range_table(addr, table: gen12_oa_b_counters);
4462	}
4463
4464	static bool mtl_is_valid_oam_b_counter_addr(struct i915_perf *perf, u32 addr)
4465	{
4466	if (HAS_OAM(perf->i915) &&
4467	GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70))
4468	return reg_in_range_table(addr, table: mtl_oam_b_counters);
4469
4470	return false;
4471	}
4472
4473	static bool xehp_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4474	{
4475	return reg_in_range_table(addr, table: xehp_oa_b_counters) ||
4476	reg_in_range_table(addr, table: gen12_oa_b_counters) ||
4477	mtl_is_valid_oam_b_counter_addr(perf, addr);
4478	}
4479
4480	static bool gen12_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4481	{
4482	if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70))
4483	return reg_in_range_table(addr, table: mtl_oa_mux_regs);
4484	else
4485	return reg_in_range_table(addr, table: gen12_oa_mux_regs);
4486	}
4487
4488	static u32 mask_reg_value(u32 reg, u32 val)
4489	{
4490	/* HALF_SLICE_CHICKEN2 is programmed with a the
4491	* WaDisableSTUnitPowerOptimization workaround. Make sure the value
4492	* programmed by userspace doesn't change this.
4493	*/
4494	if (REG_EQUAL(reg, HALF_SLICE_CHICKEN2))
4495	val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
4496
4497	/* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function
4498	* indicated by its name and a bunch of selection fields used by OA
4499	* configs.
4500	*/
4501	if (REG_EQUAL(reg, WAIT_FOR_RC6_EXIT))
4502	val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
4503
4504	return val;
4505	}
4506
4507	static struct i915_oa_reg *alloc_oa_regs(struct i915_perf *perf,
4508	bool (*is_valid)(struct i915_perf *perf, u32 addr),
4509	u32 __user *regs,
4510	u32 n_regs)
4511	{
4512	struct i915_oa_reg *oa_regs;
4513	int err;
4514	u32 i;
4515
4516	if (!n_regs)
4517	return NULL;
4518
4519	/* No is_valid function means we're not allowing any register to be programmed. */
4520	GEM_BUG_ON(!is_valid);
4521	if (!is_valid)
4522	return ERR_PTR(error: -EINVAL);
4523
4524	oa_regs = kmalloc_array(n: n_regs, size: sizeof(*oa_regs), GFP_KERNEL);
4525	if (!oa_regs)
4526	return ERR_PTR(error: -ENOMEM);
4527
4528	for (i = 0; i < n_regs; i++) {
4529	u32 addr, value;
4530
4531	err = get_user(addr, regs);
4532	if (err)
4533	goto addr_err;
4534
4535	if (!is_valid(perf, addr)) {
4536	drm_dbg(&perf->i915->drm,
4537	"Invalid oa_reg address: %X\n", addr);
4538	err = -EINVAL;
4539	goto addr_err;
4540	}
4541
4542	err = get_user(value, regs + 1);
4543	if (err)
4544	goto addr_err;
4545
4546	oa_regs[i].addr = _MMIO(addr);
4547	oa_regs[i].value = mask_reg_value(reg: addr, val: value);
4548
4549	regs += 2;
4550	}
4551
4552	return oa_regs;
4553
4554	addr_err:
4555	kfree(objp: oa_regs);
4556	return ERR_PTR(error: err);
4557	}
4558
4559	static ssize_t show_dynamic_id(struct kobject *kobj,
4560	struct kobj_attribute *attr,
4561	char *buf)
4562	{
4563	struct i915_oa_config *oa_config =
4564	container_of(attr, typeof(*oa_config), sysfs_metric_id);
4565
4566	return sprintf(buf, fmt: "%d\n", oa_config->id);
4567	}
4568
4569	static int create_dynamic_oa_sysfs_entry(struct i915_perf *perf,
4570	struct i915_oa_config *oa_config)
4571	{
4572	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
4573	oa_config->sysfs_metric_id.attr.name = "id";
4574	oa_config->sysfs_metric_id.attr.mode = S_IRUGO;
4575	oa_config->sysfs_metric_id.show = show_dynamic_id;
4576	oa_config->sysfs_metric_id.store = NULL;
4577
4578	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
4579	oa_config->attrs[1] = NULL;
4580
4581	oa_config->sysfs_metric.name = oa_config->uuid;
4582	oa_config->sysfs_metric.attrs = oa_config->attrs;
4583
4584	return sysfs_create_group(kobj: perf->metrics_kobj,
4585	grp: &oa_config->sysfs_metric);
4586	}
4587
4588	/**
4589	* i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
4590	* @dev: drm device
4591	* @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from
4592	* userspace (unvalidated)
4593	* @file: drm file
4594	*
4595	* Validates the submitted OA register to be saved into a new OA config that
4596	* can then be used for programming the OA unit and its NOA network.
4597	*
4598	* Returns: A new allocated config number to be used with the perf open ioctl
4599	* or a negative error code on failure.
4600	*/
4601	int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
4602	struct drm_file *file)
4603	{
4604	struct i915_perf *perf = &to_i915(dev)->perf;
4605	struct drm_i915_perf_oa_config *args = data;
4606	struct i915_oa_config *oa_config, *tmp;
4607	struct i915_oa_reg *regs;
4608	int err, id;
4609
4610	if (!perf->i915) {
4611	drm_dbg(&perf->i915->drm,
4612	"i915 perf interface not available for this system\n");
4613	return -ENOTSUPP;
4614	}
4615
4616	if (!perf->metrics_kobj) {
4617	drm_dbg(&perf->i915->drm,
4618	"OA metrics weren't advertised via sysfs\n");
4619	return -EINVAL;
4620	}
4621
4622	if (i915_perf_stream_paranoid && !perfmon_capable()) {
4623	drm_dbg(&perf->i915->drm,
4624	"Insufficient privileges to add i915 OA config\n");
4625	return -EACCES;
4626	}
4627
4628	if ((!args->mux_regs_ptr || !args->n_mux_regs) &&
4629	(!args->boolean_regs_ptr || !args->n_boolean_regs) &&
4630	(!args->flex_regs_ptr || !args->n_flex_regs)) {
4631	drm_dbg(&perf->i915->drm,
4632	"No OA registers given\n");
4633	return -EINVAL;
4634	}
4635
4636	oa_config = kzalloc(size: sizeof(*oa_config), GFP_KERNEL);
4637	if (!oa_config) {
4638	drm_dbg(&perf->i915->drm,
4639	"Failed to allocate memory for the OA config\n");
4640	return -ENOMEM;
4641	}
4642
4643	oa_config->perf = perf;
4644	kref_init(kref: &oa_config->ref);
4645
4646	if (!uuid_is_valid(uuid: args->uuid)) {
4647	drm_dbg(&perf->i915->drm,
4648	"Invalid uuid format for OA config\n");
4649	err = -EINVAL;
4650	goto reg_err;
4651	}
4652
4653	/* Last character in oa_config->uuid will be 0 because oa_config is
4654	* kzalloc.
4655	*/
4656	memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid));
4657
4658	oa_config->mux_regs_len = args->n_mux_regs;
4659	regs = alloc_oa_regs(perf,
4660	is_valid: perf->ops.is_valid_mux_reg,
4661	u64_to_user_ptr(args->mux_regs_ptr),
4662	n_regs: args->n_mux_regs);
4663
4664	if (IS_ERR(ptr: regs)) {
4665	drm_dbg(&perf->i915->drm,
4666	"Failed to create OA config for mux_regs\n");
4667	err = PTR_ERR(ptr: regs);
4668	goto reg_err;
4669	}
4670	oa_config->mux_regs = regs;
4671
4672	oa_config->b_counter_regs_len = args->n_boolean_regs;
4673	regs = alloc_oa_regs(perf,
4674	is_valid: perf->ops.is_valid_b_counter_reg,
4675	u64_to_user_ptr(args->boolean_regs_ptr),
4676	n_regs: args->n_boolean_regs);
4677
4678	if (IS_ERR(ptr: regs)) {
4679	drm_dbg(&perf->i915->drm,
4680	"Failed to create OA config for b_counter_regs\n");
4681	err = PTR_ERR(ptr: regs);
4682	goto reg_err;
4683	}
4684	oa_config->b_counter_regs = regs;
4685
4686	if (GRAPHICS_VER(perf->i915) < 8) {
4687	if (args->n_flex_regs != 0) {
4688	err = -EINVAL;
4689	goto reg_err;
4690	}
4691	} else {
4692	oa_config->flex_regs_len = args->n_flex_regs;
4693	regs = alloc_oa_regs(perf,
4694	is_valid: perf->ops.is_valid_flex_reg,
4695	u64_to_user_ptr(args->flex_regs_ptr),
4696	n_regs: args->n_flex_regs);
4697
4698	if (IS_ERR(ptr: regs)) {
4699	drm_dbg(&perf->i915->drm,
4700	"Failed to create OA config for flex_regs\n");
4701	err = PTR_ERR(ptr: regs);
4702	goto reg_err;
4703	}
4704	oa_config->flex_regs = regs;
4705	}
4706
4707	err = mutex_lock_interruptible(&perf->metrics_lock);
4708	if (err)
4709	goto reg_err;
4710
4711	/* We shouldn't have too many configs, so this iteration shouldn't be
4712	* too costly.
4713	*/
4714	idr_for_each_entry(&perf->metrics_idr, tmp, id) {
4715	if (!strcmp(tmp->uuid, oa_config->uuid)) {
4716	drm_dbg(&perf->i915->drm,
4717	"OA config already exists with this uuid\n");
4718	err = -EADDRINUSE;
4719	goto sysfs_err;
4720	}
4721	}
4722
4723	err = create_dynamic_oa_sysfs_entry(perf, oa_config);
4724	if (err) {
4725	drm_dbg(&perf->i915->drm,
4726	"Failed to create sysfs entry for OA config\n");
4727	goto sysfs_err;
4728	}
4729
4730	/* Config id 0 is invalid, id 1 for kernel stored test config. */
4731	oa_config->id = idr_alloc(&perf->metrics_idr,
4732	ptr: oa_config, start: 2,
4733	end: 0, GFP_KERNEL);
4734	if (oa_config->id < 0) {
4735	drm_dbg(&perf->i915->drm,
4736	"Failed to create sysfs entry for OA config\n");
4737	err = oa_config->id;
4738	goto sysfs_err;
4739	}
4740	id = oa_config->id;
4741
4742	drm_dbg(&perf->i915->drm,
4743	"Added config %s id=%i\n", oa_config->uuid, oa_config->id);
4744	mutex_unlock(lock: &perf->metrics_lock);
4745
4746	return id;
4747
4748	sysfs_err:
4749	mutex_unlock(lock: &perf->metrics_lock);
4750	reg_err:
4751	i915_oa_config_put(oa_config);
4752	drm_dbg(&perf->i915->drm,
4753	"Failed to add new OA config\n");
4754	return err;
4755	}
4756
4757	/**
4758	* i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
4759	* @dev: drm device
4760	* @data: ioctl data (pointer to u64 integer) copied from userspace
4761	* @file: drm file
4762	*
4763	* Configs can be removed while being used, the will stop appearing in sysfs
4764	* and their content will be freed when the stream using the config is closed.
4765	*
4766	* Returns: 0 on success or a negative error code on failure.
4767	*/
4768	int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
4769	struct drm_file *file)
4770	{
4771	struct i915_perf *perf = &to_i915(dev)->perf;
4772	u64 *arg = data;
4773	struct i915_oa_config *oa_config;
4774	int ret;
4775
4776	if (!perf->i915) {
4777	drm_dbg(&perf->i915->drm,
4778	"i915 perf interface not available for this system\n");
4779	return -ENOTSUPP;
4780	}
4781
4782	if (i915_perf_stream_paranoid && !perfmon_capable()) {
4783	drm_dbg(&perf->i915->drm,
4784	"Insufficient privileges to remove i915 OA config\n");
4785	return -EACCES;
4786	}
4787
4788	ret = mutex_lock_interruptible(&perf->metrics_lock);
4789	if (ret)
4790	return ret;
4791
4792	oa_config = idr_find(&perf->metrics_idr, id: *arg);
4793	if (!oa_config) {
4794	drm_dbg(&perf->i915->drm,
4795	"Failed to remove unknown OA config\n");
4796	ret = -ENOENT;
4797	goto err_unlock;
4798	}
4799
4800	GEM_BUG_ON(*arg != oa_config->id);
4801
4802	sysfs_remove_group(kobj: perf->metrics_kobj, grp: &oa_config->sysfs_metric);
4803
4804	idr_remove(&perf->metrics_idr, id: *arg);
4805
4806	mutex_unlock(lock: &perf->metrics_lock);
4807
4808	drm_dbg(&perf->i915->drm,
4809	"Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
4810
4811	i915_oa_config_put(oa_config);
4812
4813	return 0;
4814
4815	err_unlock:
4816	mutex_unlock(lock: &perf->metrics_lock);
4817	return ret;
4818	}
4819
4820	static struct ctl_table oa_table[] = {
4821	{
4822	.procname = "perf_stream_paranoid",
4823	.data = &i915_perf_stream_paranoid,
4824	.maxlen = sizeof(i915_perf_stream_paranoid),
4825	.mode = 0644,
4826	.proc_handler = proc_dointvec_minmax,
4827	.extra1 = SYSCTL_ZERO,
4828	.extra2 = SYSCTL_ONE,
4829	},
4830	{
4831	.procname = "oa_max_sample_rate",
4832	.data = &i915_oa_max_sample_rate,
4833	.maxlen = sizeof(i915_oa_max_sample_rate),
4834	.mode = 0644,
4835	.proc_handler = proc_dointvec_minmax,
4836	.extra1 = SYSCTL_ZERO,
4837	.extra2 = &oa_sample_rate_hard_limit,
4838	},
4839	};
4840
4841	static u32 num_perf_groups_per_gt(struct intel_gt *gt)
4842	{
4843	return 1;
4844	}
4845
4846	static u32 __oam_engine_group(struct intel_engine_cs *engine)
4847	{
4848	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) {
4849	/*
4850	* There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
4851	* within the gt use the same OAM. All MTL SKUs list 1 SA MEDIA.
4852	*/
4853	drm_WARN_ON(&engine->i915->drm,
4854	engine->gt->type != GT_MEDIA);
4855
4856	return PERF_GROUP_OAM_SAMEDIA_0;
4857	}
4858
4859	return PERF_GROUP_INVALID;
4860	}
4861
4862	static u32 __oa_engine_group(struct intel_engine_cs *engine)
4863	{
4864	switch (engine->class) {
4865	case RENDER_CLASS:
4866	return PERF_GROUP_OAG;
4867
4868	case VIDEO_DECODE_CLASS:
4869	case VIDEO_ENHANCEMENT_CLASS:
4870	return __oam_engine_group(engine);
4871
4872	default:
4873	return PERF_GROUP_INVALID;
4874	}
4875	}
4876
4877	static struct i915_perf_regs __oam_regs(u32 base)
4878	{
4879	return (struct i915_perf_regs) {
4880	base,
4881	GEN12_OAM_HEAD_POINTER(base),
4882	GEN12_OAM_TAIL_POINTER(base),
4883	GEN12_OAM_BUFFER(base),
4884	GEN12_OAM_CONTEXT_CONTROL(base),
4885	GEN12_OAM_CONTROL(base),
4886	GEN12_OAM_DEBUG(base),
4887	GEN12_OAM_STATUS(base),
4888	GEN12_OAM_CONTROL_COUNTER_FORMAT_SHIFT,
4889	};
4890	}
4891
4892	static struct i915_perf_regs __oag_regs(void)
4893	{
4894	return (struct i915_perf_regs) {
4895	0,
4896	GEN12_OAG_OAHEADPTR,
4897	GEN12_OAG_OATAILPTR,
4898	GEN12_OAG_OABUFFER,
4899	GEN12_OAG_OAGLBCTXCTRL,
4900	GEN12_OAG_OACONTROL,
4901	GEN12_OAG_OA_DEBUG,
4902	GEN12_OAG_OASTATUS,
4903	GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT,
4904	};
4905	}
4906
4907	static void oa_init_groups(struct intel_gt *gt)
4908	{
4909	int i, num_groups = gt->perf.num_perf_groups;
4910
4911	for (i = 0; i < num_groups; i++) {
4912	struct i915_perf_group *g = &gt->perf.group[i];
4913
4914	/* Fused off engines can result in a group with num_engines == 0 */
4915	if (g->num_engines == 0)
4916	continue;
4917
4918	if (i == PERF_GROUP_OAG && gt->type != GT_MEDIA) {
4919	g->regs = __oag_regs();
4920	g->type = TYPE_OAG;
4921	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
4922	g->regs = __oam_regs(base: mtl_oa_base[i]);
4923	g->type = TYPE_OAM;
4924	}
4925	}
4926	}
4927
4928	static int oa_init_gt(struct intel_gt *gt)
4929	{
4930	u32 num_groups = num_perf_groups_per_gt(gt);
4931	struct intel_engine_cs *engine;
4932	struct i915_perf_group *g;
4933	intel_engine_mask_t tmp;
4934
4935	g = kcalloc(n: num_groups, size: sizeof(*g), GFP_KERNEL);
4936	if (!g)
4937	return -ENOMEM;
4938
4939	for_each_engine_masked(engine, gt, ALL_ENGINES, tmp) {
4940	u32 index = __oa_engine_group(engine);
4941
4942	engine->oa_group = NULL;
4943	if (index < num_groups) {
4944	g[index].num_engines++;
4945	engine->oa_group = &g[index];
4946	}
4947	}
4948
4949	gt->perf.num_perf_groups = num_groups;
4950	gt->perf.group = g;
4951
4952	oa_init_groups(gt);
4953
4954	return 0;
4955	}
4956
4957	static int oa_init_engine_groups(struct i915_perf *perf)
4958	{
4959	struct intel_gt *gt;
4960	int i, ret;
4961
4962	for_each_gt(gt, perf->i915, i) {
4963	ret = oa_init_gt(gt);
4964	if (ret)
4965	return ret;
4966	}
4967
4968	return 0;
4969	}
4970
4971	static void oa_init_supported_formats(struct i915_perf *perf)
4972	{
4973	struct drm_i915_private *i915 = perf->i915;
4974	enum intel_platform platform = INTEL_INFO(i915)->platform;
4975
4976	switch (platform) {
4977	case INTEL_HASWELL:
4978	oa_format_add(perf, format: I915_OA_FORMAT_A13);
4979	oa_format_add(perf, format: I915_OA_FORMAT_A13);
4980	oa_format_add(perf, format: I915_OA_FORMAT_A29);
4981	oa_format_add(perf, format: I915_OA_FORMAT_A13_B8_C8);
4982	oa_format_add(perf, format: I915_OA_FORMAT_B4_C8);
4983	oa_format_add(perf, format: I915_OA_FORMAT_A45_B8_C8);
4984	oa_format_add(perf, format: I915_OA_FORMAT_B4_C8_A16);
4985	oa_format_add(perf, format: I915_OA_FORMAT_C4_B8);
4986	break;
4987
4988	case INTEL_BROADWELL:
4989	case INTEL_CHERRYVIEW:
4990	case INTEL_SKYLAKE:
4991	case INTEL_BROXTON:
4992	case INTEL_KABYLAKE:
4993	case INTEL_GEMINILAKE:
4994	case INTEL_COFFEELAKE:
4995	case INTEL_COMETLAKE:
4996	case INTEL_ICELAKE:
4997	case INTEL_ELKHARTLAKE:
4998	case INTEL_JASPERLAKE:
4999	case INTEL_TIGERLAKE:
5000	case INTEL_ROCKETLAKE:
5001	case INTEL_DG1:
5002	case INTEL_ALDERLAKE_S:
5003	case INTEL_ALDERLAKE_P:
5004	oa_format_add(perf, format: I915_OA_FORMAT_A12);
5005	oa_format_add(perf, format: I915_OA_FORMAT_A12_B8_C8);
5006	oa_format_add(perf, format: I915_OA_FORMAT_A32u40_A4u32_B8_C8);
5007	oa_format_add(perf, format: I915_OA_FORMAT_C4_B8);
5008	break;
5009
5010	case INTEL_DG2:
5011	oa_format_add(perf, format: I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
5012	oa_format_add(perf, format: I915_OA_FORMAT_A24u40_A14u32_B8_C8);
5013	break;
5014
5015	case INTEL_METEORLAKE:
5016	oa_format_add(perf, format: I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
5017	oa_format_add(perf, format: I915_OA_FORMAT_A24u40_A14u32_B8_C8);
5018	oa_format_add(perf, format: I915_OAM_FORMAT_MPEC8u64_B8_C8);
5019	oa_format_add(perf, format: I915_OAM_FORMAT_MPEC8u32_B8_C8);
5020	break;
5021
5022	default:
5023	MISSING_CASE(platform);
5024	}
5025	}
5026
5027	static void i915_perf_init_info(struct drm_i915_private *i915)
5028	{
5029	struct i915_perf *perf = &i915->perf;
5030
5031	switch (GRAPHICS_VER(i915)) {
5032	case 8:
5033	perf->ctx_oactxctrl_offset = 0x120;
5034	perf->ctx_flexeu0_offset = 0x2ce;
5035	perf->gen8_valid_ctx_bit = BIT(25);
5036	break;
5037	case 9:
5038	perf->ctx_oactxctrl_offset = 0x128;
5039	perf->ctx_flexeu0_offset = 0x3de;
5040	perf->gen8_valid_ctx_bit = BIT(16);
5041	break;
5042	case 11:
5043	perf->ctx_oactxctrl_offset = 0x124;
5044	perf->ctx_flexeu0_offset = 0x78e;
5045	perf->gen8_valid_ctx_bit = BIT(16);
5046	break;
5047	case 12:
5048	perf->gen8_valid_ctx_bit = BIT(16);
5049	/*
5050	* Calculate offset at runtime in oa_pin_context for gen12 and
5051	* cache the value in perf->ctx_oactxctrl_offset.
5052	*/
5053	break;
5054	default:
5055	MISSING_CASE(GRAPHICS_VER(i915));
5056	}
5057	}
5058
5059	/**
5060	* i915_perf_init - initialize i915-perf state on module bind
5061	* @i915: i915 device instance
5062	*
5063	* Initializes i915-perf state without exposing anything to userspace.
5064	*
5065	* Note: i915-perf initialization is split into an 'init' and 'register'
5066	* phase with the i915_perf_register() exposing state to userspace.
5067	*/
5068	int i915_perf_init(struct drm_i915_private *i915)
5069	{
5070	struct i915_perf *perf = &i915->perf;
5071
5072	perf->oa_formats = oa_formats;
5073	if (IS_HASWELL(i915)) {
5074	perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr;
5075	perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr;
5076	perf->ops.is_valid_flex_reg = NULL;
5077	perf->ops.enable_metric_set = hsw_enable_metric_set;
5078	perf->ops.disable_metric_set = hsw_disable_metric_set;
5079	perf->ops.oa_enable = gen7_oa_enable;
5080	perf->ops.oa_disable = gen7_oa_disable;
5081	perf->ops.read = gen7_oa_read;
5082	perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read;
5083	} else if (HAS_LOGICAL_RING_CONTEXTS(i915)) {
5084	/* Note: that although we could theoretically also support the
5085	* legacy ringbuffer mode on BDW (and earlier iterations of
5086	* this driver, before upstreaming did this) it didn't seem
5087	* worth the complexity to maintain now that BDW+ enable
5088	* execlist mode by default.
5089	*/
5090	perf->ops.read = gen8_oa_read;
5091	i915_perf_init_info(i915);
5092
5093	if (IS_GRAPHICS_VER(i915, 8, 9)) {
5094	perf->ops.is_valid_b_counter_reg =
5095	gen7_is_valid_b_counter_addr;
5096	perf->ops.is_valid_mux_reg =
5097	gen8_is_valid_mux_addr;
5098	perf->ops.is_valid_flex_reg =
5099	gen8_is_valid_flex_addr;
5100
5101	if (IS_CHERRYVIEW(i915)) {
5102	perf->ops.is_valid_mux_reg =
5103	chv_is_valid_mux_addr;
5104	}
5105
5106	perf->ops.oa_enable = gen8_oa_enable;
5107	perf->ops.oa_disable = gen8_oa_disable;
5108	perf->ops.enable_metric_set = gen8_enable_metric_set;
5109	perf->ops.disable_metric_set = gen8_disable_metric_set;
5110	perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
5111	} else if (GRAPHICS_VER(i915) == 11) {
5112	perf->ops.is_valid_b_counter_reg =
5113	gen7_is_valid_b_counter_addr;
5114	perf->ops.is_valid_mux_reg =
5115	gen11_is_valid_mux_addr;
5116	perf->ops.is_valid_flex_reg =
5117	gen8_is_valid_flex_addr;
5118
5119	perf->ops.oa_enable = gen8_oa_enable;
5120	perf->ops.oa_disable = gen8_oa_disable;
5121	perf->ops.enable_metric_set = gen8_enable_metric_set;
5122	perf->ops.disable_metric_set = gen11_disable_metric_set;
5123	perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
5124	} else if (GRAPHICS_VER(i915) == 12) {
5125	perf->ops.is_valid_b_counter_reg =
5126	HAS_OA_SLICE_CONTRIB_LIMITS(i915) ?
5127	xehp_is_valid_b_counter_addr :
5128	gen12_is_valid_b_counter_addr;
5129	perf->ops.is_valid_mux_reg =
5130	gen12_is_valid_mux_addr;
5131	perf->ops.is_valid_flex_reg =
5132	gen8_is_valid_flex_addr;
5133
5134	perf->ops.oa_enable = gen12_oa_enable;
5135	perf->ops.oa_disable = gen12_oa_disable;
5136	perf->ops.enable_metric_set = gen12_enable_metric_set;
5137	perf->ops.disable_metric_set = gen12_disable_metric_set;
5138	perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read;
5139	}
5140	}
5141
5142	if (perf->ops.enable_metric_set) {
5143	struct intel_gt *gt;
5144	int i, ret;
5145
5146	for_each_gt(gt, i915, i)
5147	mutex_init(&gt->perf.lock);
5148
5149	/* Choose a representative limit */
5150	oa_sample_rate_hard_limit = to_gt(i915)->clock_frequency / 2;
5151
5152	mutex_init(&perf->metrics_lock);
5153	idr_init_base(idr: &perf->metrics_idr, base: 1);
5154
5155	/* We set up some ratelimit state to potentially throttle any
5156	* _NOTES about spurious, invalid OA reports which we don't
5157	* forward to userspace.
5158	*
5159	* We print a _NOTE about any throttling when closing the
5160	* stream instead of waiting until driver _fini which no one
5161	* would ever see.
5162	*
5163	* Using the same limiting factors as printk_ratelimit()
5164	*/
5165	ratelimit_state_init(rs: &perf->spurious_report_rs, interval: 5 * HZ, burst: 10);
5166	/* Since we use a DRM_NOTE for spurious reports it would be
5167	* inconsistent to let __ratelimit() automatically print a
5168	* warning for throttling.
5169	*/
5170	ratelimit_set_flags(rs: &perf->spurious_report_rs,
5171	RATELIMIT_MSG_ON_RELEASE);
5172
5173	ratelimit_state_init(rs: &perf->tail_pointer_race,
5174	interval: 5 * HZ, burst: 10);
5175	ratelimit_set_flags(rs: &perf->tail_pointer_race,
5176	RATELIMIT_MSG_ON_RELEASE);
5177
5178	atomic64_set(v: &perf->noa_programming_delay,
5179	i: 500 * 1000 /* 500us */);
5180
5181	perf->i915 = i915;
5182
5183	ret = oa_init_engine_groups(perf);
5184	if (ret) {
5185	drm_err(&i915->drm,
5186	"OA initialization failed %d\n", ret);
5187	return ret;
5188	}
5189
5190	oa_init_supported_formats(perf);
5191	}
5192
5193	return 0;
5194	}
5195
5196	static int destroy_config(int id, void *p, void *data)
5197	{
5198	i915_oa_config_put(oa_config: p);
5199	return 0;
5200	}
5201
5202	int i915_perf_sysctl_register(void)
5203	{
5204	sysctl_header = register_sysctl("dev/i915", oa_table);
5205	return 0;
5206	}
5207
5208	void i915_perf_sysctl_unregister(void)
5209	{
5210	unregister_sysctl_table(table: sysctl_header);
5211	}
5212
5213	/**
5214	* i915_perf_fini - Counter part to i915_perf_init()
5215	* @i915: i915 device instance
5216	*/
5217	void i915_perf_fini(struct drm_i915_private *i915)
5218	{
5219	struct i915_perf *perf = &i915->perf;
5220	struct intel_gt *gt;
5221	int i;
5222
5223	if (!perf->i915)
5224	return;
5225
5226	for_each_gt(gt, perf->i915, i)
5227	kfree(objp: gt->perf.group);
5228
5229	idr_for_each(&perf->metrics_idr, fn: destroy_config, data: perf);
5230	idr_destroy(&perf->metrics_idr);
5231
5232	memset(&perf->ops, 0, sizeof(perf->ops));
5233	perf->i915 = NULL;
5234	}
5235
5236	/**
5237	* i915_perf_ioctl_version - Version of the i915-perf subsystem
5238	* @i915: The i915 device
5239	*
5240	* This version number is used by userspace to detect available features.
5241	*/
5242	int i915_perf_ioctl_version(struct drm_i915_private *i915)
5243	{
5244	/*
5245	* 1: Initial version
5246	* I915_PERF_IOCTL_ENABLE
5247	* I915_PERF_IOCTL_DISABLE
5248	*
5249	* 2: Added runtime modification of OA config.
5250	* I915_PERF_IOCTL_CONFIG
5251	*
5252	* 3: Add DRM_I915_PERF_PROP_HOLD_PREEMPTION parameter to hold
5253	* preemption on a particular context so that performance data is
5254	* accessible from a delta of MI_RPC reports without looking at the
5255	* OA buffer.
5256	*
5257	* 4: Add DRM_I915_PERF_PROP_ALLOWED_SSEU to limit what contexts can
5258	* be run for the duration of the performance recording based on
5259	* their SSEU configuration.
5260	*
5261	* 5: Add DRM_I915_PERF_PROP_POLL_OA_PERIOD parameter that controls the
5262	* interval for the hrtimer used to check for OA data.
5263	*
5264	* 6: Add DRM_I915_PERF_PROP_OA_ENGINE_CLASS and
5265	* DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE
5266	*
5267	* 7: Add support for video decode and enhancement classes.
5268	*/
5269
5270	/*
5271	* Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
5272	* C6 disable in BIOS. If Media C6 is enabled in BIOS, return version 6
5273	* to indicate that OA media is not supported.
5274	*/
5275	if (IS_MEDIA_GT_IP_STEP(i915->media_gt, IP_VER(13, 0), STEP_A0, STEP_C0) &&
5276	intel_check_bios_c6_setup(rc6: &i915->media_gt->rc6))
5277	return 6;
5278
5279	return 7;
5280	}
5281
5282	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
5283	#include "selftests/i915_perf.c"
5284	#endif
5285