1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Performance event support for the System z CPU-measurement Sampling Facility
4	*
5	* Copyright IBM Corp. 2013, 2018
6	* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7	*/
8	#define KMSG_COMPONENT "cpum_sf"
9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11	#include <linux/kernel.h>
12	#include <linux/kernel_stat.h>
13	#include <linux/perf_event.h>
14	#include <linux/percpu.h>
15	#include <linux/pid.h>
16	#include <linux/notifier.h>
17	#include <linux/export.h>
18	#include <linux/slab.h>
19	#include <linux/mm.h>
20	#include <linux/moduleparam.h>
21	#include <asm/cpu_mf.h>
22	#include <asm/irq.h>
23	#include <asm/debug.h>
24	#include <asm/timex.h>
25	#include <linux/io.h>
26
27	/* Minimum number of sample-data-block-tables:
28	* At least one table is required for the sampling buffer structure.
29	* A single table contains up to 511 pointers to sample-data-blocks.
30	*/
31	#define CPUM_SF_MIN_SDBT 1
32
33	/* Number of sample-data-blocks per sample-data-block-table (SDBT):
34	* A table contains SDB pointers (8 bytes) and one table-link entry
35	* that points to the origin of the next SDBT.
36	*/
37	#define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
38
39	/* Maximum page offset for an SDBT table-link entry:
40	* If this page offset is reached, a table-link entry to the next SDBT
41	* must be added.
42	*/
43	#define CPUM_SF_SDBT_TL_OFFSET (CPUM_SF_SDB_PER_TABLE * 8)
44	static inline int require_table_link(const void *sdbt)
45	{
46	return ((unsigned long)sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET;
47	}
48
49	/* Minimum and maximum sampling buffer sizes:
50	*
51	* This number represents the maximum size of the sampling buffer taking
52	* the number of sample-data-block-tables into account. Note that these
53	* numbers apply to the basic-sampling function only.
54	* The maximum number of SDBs is increased by CPUM_SF_SDB_DIAG_FACTOR if
55	* the diagnostic-sampling function is active.
56	*
57	* Sampling buffer size Buffer characteristics
58	* ---------------------------------------------------
59	* 64KB == 16 pages (4KB per page)
60	* 1 page for SDB-tables
61	* 15 pages for SDBs
62	*
63	* 32MB == 8192 pages (4KB per page)
64	* 16 pages for SDB-tables
65	* 8176 pages for SDBs
66	*/
67	static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15;
68	static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176;
69	static unsigned long __read_mostly CPUM_SF_SDB_DIAG_FACTOR = 1;
70
71	struct sf_buffer {
72	unsigned long *sdbt; /* Sample-data-block-table origin */
73	/* buffer characteristics (required for buffer increments) */
74	unsigned long num_sdb; /* Number of sample-data-blocks */
75	unsigned long num_sdbt; /* Number of sample-data-block-tables */
76	unsigned long *tail; /* last sample-data-block-table */
77	};
78
79	struct aux_buffer {
80	struct sf_buffer sfb;
81	unsigned long head; /* index of SDB of buffer head */
82	unsigned long alert_mark; /* index of SDB of alert request position */
83	unsigned long empty_mark; /* mark of SDB not marked full */
84	unsigned long *sdb_index; /* SDB address for fast lookup */
85	unsigned long *sdbt_index; /* SDBT address for fast lookup */
86	};
87
88	struct cpu_hw_sf {
89	/* CPU-measurement sampling information block */
90	struct hws_qsi_info_block qsi;
91	/* CPU-measurement sampling control block */
92	struct hws_lsctl_request_block lsctl;
93	struct sf_buffer sfb; /* Sampling buffer */
94	unsigned int flags; /* Status flags */
95	struct perf_event *event; /* Scheduled perf event */
96	struct perf_output_handle handle; /* AUX buffer output handle */
97	};
98	static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf);
99
100	/* Debug feature */
101	static debug_info_t *sfdbg;
102
103	/* Sampling control helper functions */
104	static inline unsigned long freq_to_sample_rate(struct hws_qsi_info_block *qsi,
105	unsigned long freq)
106	{
107	return (USEC_PER_SEC / freq) * qsi->cpu_speed;
108	}
109
110	static inline unsigned long sample_rate_to_freq(struct hws_qsi_info_block *qsi,
111	unsigned long rate)
112	{
113	return USEC_PER_SEC * qsi->cpu_speed / rate;
114	}
115
116	/* Return TOD timestamp contained in an trailer entry */
117	static inline unsigned long long trailer_timestamp(struct hws_trailer_entry *te)
118	{
119	/* TOD in STCKE format */
120	if (te->header.t)
121	return *((unsigned long long *)&te->timestamp[1]);
122
123	/* TOD in STCK format */
124	return *((unsigned long long *)&te->timestamp[0]);
125	}
126
127	/* Return pointer to trailer entry of an sample data block */
128	static inline struct hws_trailer_entry *trailer_entry_ptr(unsigned long v)
129	{
130	void *ret;
131
132	ret = (void *)v;
133	ret += PAGE_SIZE;
134	ret -= sizeof(struct hws_trailer_entry);
135
136	return ret;
137	}
138
139	/*
140	* Return true if the entry in the sample data block table (sdbt)
141	* is a link to the next sdbt
142	*/
143	static inline int is_link_entry(unsigned long *s)
144	{
145	return *s & 0x1UL ? 1 : 0;
146	}
147
148	/* Return pointer to the linked sdbt */
149	static inline unsigned long *get_next_sdbt(unsigned long *s)
150	{
151	return phys_to_virt(address: *s & ~0x1UL);
152	}
153
154	/*
155	* sf_disable() - Switch off sampling facility
156	*/
157	static int sf_disable(void)
158	{
159	struct hws_lsctl_request_block sreq;
160
161	memset(&sreq, 0, sizeof(sreq));
162	return lsctl(&sreq);
163	}
164
165	/*
166	* sf_buffer_available() - Check for an allocated sampling buffer
167	*/
168	static int sf_buffer_available(struct cpu_hw_sf *cpuhw)
169	{
170	return !!cpuhw->sfb.sdbt;
171	}
172
173	/*
174	* deallocate sampling facility buffer
175	*/
176	static void free_sampling_buffer(struct sf_buffer *sfb)
177	{
178	unsigned long *sdbt, *curr;
179
180	if (!sfb->sdbt)
181	return;
182
183	sdbt = sfb->sdbt;
184	curr = sdbt;
185
186	/* Free the SDBT after all SDBs are processed... */
187	while (1) {
188	if (!*curr || !sdbt)
189	break;
190
191	/* Process table-link entries */
192	if (is_link_entry(s: curr)) {
193	curr = get_next_sdbt(s: curr);
194	if (sdbt)
195	free_page((unsigned long)sdbt);
196
197	/* If the origin is reached, sampling buffer is freed */
198	if (curr == sfb->sdbt)
199	break;
200	else
201	sdbt = curr;
202	} else {
203	/* Process SDB pointer */
204	if (*curr) {
205	free_page((unsigned long)phys_to_virt(*curr));
206	curr++;
207	}
208	}
209	}
210
211	debug_sprintf_event(sfdbg, 5, "%s: freed sdbt %#lx\n", __func__,
212	(unsigned long)sfb->sdbt);
213	memset(sfb, 0, sizeof(*sfb));
214	}
215
216	static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags)
217	{
218	struct hws_trailer_entry *te;
219	unsigned long sdb;
220
221	/* Allocate and initialize sample-data-block */
222	sdb = get_zeroed_page(gfp_mask: gfp_flags);
223	if (!sdb)
224	return -ENOMEM;
225	te = trailer_entry_ptr(v: sdb);
226	te->header.a = 1;
227
228	/* Link SDB into the sample-data-block-table */
229	*sdbt = virt_to_phys(address: (void *)sdb);
230
231	return 0;
232	}
233
234	/*
235	* realloc_sampling_buffer() - extend sampler memory
236	*
237	* Allocates new sample-data-blocks and adds them to the specified sampling
238	* buffer memory.
239	*
240	* Important: This modifies the sampling buffer and must be called when the
241	* sampling facility is disabled.
242	*
243	* Returns zero on success, non-zero otherwise.
244	*/
245	static int realloc_sampling_buffer(struct sf_buffer *sfb,
246	unsigned long num_sdb, gfp_t gfp_flags)
247	{
248	int i, rc;
249	unsigned long *new, *tail, *tail_prev = NULL;
250
251	if (!sfb->sdbt || !sfb->tail)
252	return -EINVAL;
253
254	if (!is_link_entry(s: sfb->tail))
255	return -EINVAL;
256
257	/* Append to the existing sampling buffer, overwriting the table-link
258	* register.
259	* The tail variables always points to the "tail" (last and table-link)
260	* entry in an SDB-table.
261	*/
262	tail = sfb->tail;
263
264	/* Do a sanity check whether the table-link entry points to
265	* the sampling buffer origin.
266	*/
267	if (sfb->sdbt != get_next_sdbt(s: tail)) {
268	debug_sprintf_event(sfdbg, 3, "%s: "
269	"sampling buffer is not linked: origin %#lx"
270	" tail %#lx\n", __func__,
271	(unsigned long)sfb->sdbt,
272	(unsigned long)tail);
273	return -EINVAL;
274	}
275
276	/* Allocate remaining SDBs */
277	rc = 0;
278	for (i = 0; i < num_sdb; i++) {
279	/* Allocate a new SDB-table if it is full. */
280	if (require_table_link(sdbt: tail)) {
281	new = (unsigned long *)get_zeroed_page(gfp_mask: gfp_flags);
282	if (!new) {
283	rc = -ENOMEM;
284	break;
285	}
286	sfb->num_sdbt++;
287	/* Link current page to tail of chain */
288	*tail = virt_to_phys(address: (void *)new) + 1;
289	tail_prev = tail;
290	tail = new;
291	}
292
293	/* Allocate a new sample-data-block.
294	* If there is not enough memory, stop the realloc process
295	* and simply use what was allocated. If this is a temporary
296	* issue, a new realloc call (if required) might succeed.
297	*/
298	rc = alloc_sample_data_block(sdbt: tail, gfp_flags);
299	if (rc) {
300	/* Undo last SDBT. An SDBT with no SDB at its first
301	* entry but with an SDBT entry instead can not be
302	* handled by the interrupt handler code.
303	* Avoid this situation.
304	*/
305	if (tail_prev) {
306	sfb->num_sdbt--;
307	free_page((unsigned long)new);
308	tail = tail_prev;
309	}
310	break;
311	}
312	sfb->num_sdb++;
313	tail++;
314	tail_prev = new = NULL; /* Allocated at least one SBD */
315	}
316
317	/* Link sampling buffer to its origin */
318	*tail = virt_to_phys(address: sfb->sdbt) + 1;
319	sfb->tail = tail;
320
321	debug_sprintf_event(sfdbg, 4, "%s: new buffer"
322	" settings: sdbt %lu sdb %lu\n", __func__,
323	sfb->num_sdbt, sfb->num_sdb);
324	return rc;
325	}
326
327	/*
328	* allocate_sampling_buffer() - allocate sampler memory
329	*
330	* Allocates and initializes a sampling buffer structure using the
331	* specified number of sample-data-blocks (SDB). For each allocation,
332	* a 4K page is used. The number of sample-data-block-tables (SDBT)
333	* are calculated from SDBs.
334	* Also set the ALERT_REQ mask in each SDBs trailer.
335	*
336	* Returns zero on success, non-zero otherwise.
337	*/
338	static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)
339	{
340	int rc;
341
342	if (sfb->sdbt)
343	return -EINVAL;
344
345	/* Allocate the sample-data-block-table origin */
346	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
347	if (!sfb->sdbt)
348	return -ENOMEM;
349	sfb->num_sdb = 0;
350	sfb->num_sdbt = 1;
351
352	/* Link the table origin to point to itself to prepare for
353	* realloc_sampling_buffer() invocation.
354	*/
355	sfb->tail = sfb->sdbt;
356	*sfb->tail = virt_to_phys(address: (void *)sfb->sdbt) + 1;
357
358	/* Allocate requested number of sample-data-blocks */
359	rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);
360	if (rc) {
361	free_sampling_buffer(sfb);
362	debug_sprintf_event(sfdbg, 4, "%s: "
363	"realloc_sampling_buffer failed with rc %i\n",
364	__func__, rc);
365	} else
366	debug_sprintf_event(sfdbg, 4,
367	"%s: tear %#lx dear %#lx\n", __func__,
368	(unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt);
369	return rc;
370	}
371
372	static void sfb_set_limits(unsigned long min, unsigned long max)
373	{
374	struct hws_qsi_info_block si;
375
376	CPUM_SF_MIN_SDB = min;
377	CPUM_SF_MAX_SDB = max;
378
379	memset(&si, 0, sizeof(si));
380	if (!qsi(&si))
381	CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes);
382	}
383
384	static unsigned long sfb_max_limit(struct hw_perf_event *hwc)
385	{
386	return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR
387	: CPUM_SF_MAX_SDB;
388	}
389
390	static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,
391	struct hw_perf_event *hwc)
392	{
393	if (!sfb->sdbt)
394	return SFB_ALLOC_REG(hwc);
395	if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)
396	return SFB_ALLOC_REG(hwc) - sfb->num_sdb;
397	return 0;
398	}
399
400	static int sfb_has_pending_allocs(struct sf_buffer *sfb,
401	struct hw_perf_event *hwc)
402	{
403	return sfb_pending_allocs(sfb, hwc) > 0;
404	}
405
406	static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc)
407	{
408	/* Limit the number of SDBs to not exceed the maximum */
409	num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc));
410	if (num)
411	SFB_ALLOC_REG(hwc) += num;
412	}
413
414	static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc)
415	{
416	SFB_ALLOC_REG(hwc) = 0;
417	sfb_account_allocs(num, hwc);
418	}
419
420	static void deallocate_buffers(struct cpu_hw_sf *cpuhw)
421	{
422	if (cpuhw->sfb.sdbt)
423	free_sampling_buffer(sfb: &cpuhw->sfb);
424	}
425
426	static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)
427	{
428	unsigned long n_sdb, freq;
429	size_t sample_size;
430
431	/* Calculate sampling buffers using 4K pages
432	*
433	* 1. The sampling size is 32 bytes for basic sampling. This size
434	* is the same for all machine types. Diagnostic
435	* sampling uses auxlilary data buffer setup which provides the
436	* memory for SDBs using linux common code auxiliary trace
437	* setup.
438	*
439	* 2. Function alloc_sampling_buffer() sets the Alert Request
440	* Control indicator to trigger a measurement-alert to harvest
441	* sample-data-blocks (SDB). This is done per SDB. This
442	* measurement alert interrupt fires quick enough to handle
443	* one SDB, on very high frequency and work loads there might
444	* be 2 to 3 SBDs available for sample processing.
445	* Currently there is no need for setup alert request on every
446	* n-th page. This is counterproductive as one IRQ triggers
447	* a very high number of samples to be processed at one IRQ.
448	*
449	* 3. Use the sampling frequency as input.
450	* Compute the number of SDBs and ensure a minimum
451	* of CPUM_SF_MIN_SDB. Depending on frequency add some more
452	* SDBs to handle a higher sampling rate.
453	* Use a minimum of CPUM_SF_MIN_SDB and allow for 100 samples
454	* (one SDB) for every 10000 HZ frequency increment.
455	*
456	* 4. Compute the number of sample-data-block-tables (SDBT) and
457	* ensure a minimum of CPUM_SF_MIN_SDBT (one table can manage up
458	* to 511 SDBs).
459	*/
460	sample_size = sizeof(struct hws_basic_entry);
461	freq = sample_rate_to_freq(qsi: &cpuhw->qsi, rate: SAMPL_RATE(hwc));
462	n_sdb = CPUM_SF_MIN_SDB + DIV_ROUND_UP(freq, 10000);
463
464	/* If there is already a sampling buffer allocated, it is very likely
465	* that the sampling facility is enabled too. If the event to be
466	* initialized requires a greater sampling buffer, the allocation must
467	* be postponed. Changing the sampling buffer requires the sampling
468	* facility to be in the disabled state. So, account the number of
469	* required SDBs and let cpumsf_pmu_enable() resize the buffer just
470	* before the event is started.
471	*/
472	sfb_init_allocs(num: n_sdb, hwc);
473	if (sf_buffer_available(cpuhw))
474	return 0;
475
476	debug_sprintf_event(sfdbg, 3,
477	"%s: rate %lu f %lu sdb %lu/%lu"
478	" sample_size %lu cpuhw %p\n", __func__,
479	SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc),
480	sample_size, cpuhw);
481
482	return alloc_sampling_buffer(sfb: &cpuhw->sfb,
483	num_sdb: sfb_pending_allocs(sfb: &cpuhw->sfb, hwc));
484	}
485
486	static unsigned long min_percent(unsigned int percent, unsigned long base,
487	unsigned long min)
488	{
489	return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100));
490	}
491
492	static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base)
493	{
494	/* Use a percentage-based approach to extend the sampling facility
495	* buffer. Accept up to 5% sample data loss.
496	* Vary the extents between 1% to 5% of the current number of
497	* sample-data-blocks.
498	*/
499	if (ratio <= 5)
500	return 0;
501	if (ratio <= 25)
502	return min_percent(percent: 1, base, min: 1);
503	if (ratio <= 50)
504	return min_percent(percent: 1, base, min: 1);
505	if (ratio <= 75)
506	return min_percent(percent: 2, base, min: 2);
507	if (ratio <= 100)
508	return min_percent(percent: 3, base, min: 3);
509	if (ratio <= 250)
510	return min_percent(percent: 4, base, min: 4);
511
512	return min_percent(percent: 5, base, min: 8);
513	}
514
515	static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,
516	struct hw_perf_event *hwc)
517	{
518	unsigned long ratio, num;
519
520	if (!OVERFLOW_REG(hwc))
521	return;
522
523	/* The sample_overflow contains the average number of sample data
524	* that has been lost because sample-data-blocks were full.
525	*
526	* Calculate the total number of sample data entries that has been
527	* discarded. Then calculate the ratio of lost samples to total samples
528	* per second in percent.
529	*/
530	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,
531	sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));
532
533	/* Compute number of sample-data-blocks */
534	num = compute_sfb_extent(ratio, base: cpuhw->sfb.num_sdb);
535	if (num)
536	sfb_account_allocs(num, hwc);
537
538	debug_sprintf_event(sfdbg, 5, "%s: overflow %llu ratio %lu num %lu\n",
539	__func__, OVERFLOW_REG(hwc), ratio, num);
540	OVERFLOW_REG(hwc) = 0;
541	}
542
543	/* extend_sampling_buffer() - Extend sampling buffer
544	* @sfb: Sampling buffer structure (for local CPU)
545	* @hwc: Perf event hardware structure
546	*
547	* Use this function to extend the sampling buffer based on the overflow counter
548	* and postponed allocation extents stored in the specified Perf event hardware.
549	*
550	* Important: This function disables the sampling facility in order to safely
551	* change the sampling buffer structure. Do not call this function
552	* when the PMU is active.
553	*/
554	static void extend_sampling_buffer(struct sf_buffer *sfb,
555	struct hw_perf_event *hwc)
556	{
557	unsigned long num, num_old;
558	int rc;
559
560	num = sfb_pending_allocs(sfb, hwc);
561	if (!num)
562	return;
563	num_old = sfb->num_sdb;
564
565	/* Disable the sampling facility to reset any states and also
566	* clear pending measurement alerts.
567	*/
568	sf_disable();
569
570	/* Extend the sampling buffer.
571	* This memory allocation typically happens in an atomic context when
572	* called by perf. Because this is a reallocation, it is fine if the
573	* new SDB-request cannot be satisfied immediately.
574	*/
575	rc = realloc_sampling_buffer(sfb, num_sdb: num, GFP_ATOMIC);
576	if (rc)
577	debug_sprintf_event(sfdbg, 5, "%s: realloc failed with rc %i\n",
578	__func__, rc);
579
580	if (sfb_has_pending_allocs(sfb, hwc))
581	debug_sprintf_event(sfdbg, 5, "%s: "
582	"req %lu alloc %lu remaining %lu\n",
583	__func__, num, sfb->num_sdb - num_old,
584	sfb_pending_allocs(sfb, hwc));
585	}
586
587	/* Number of perf events counting hardware events */
588	static atomic_t num_events;
589	/* Used to avoid races in calling reserve/release_cpumf_hardware */
590	static DEFINE_MUTEX(pmc_reserve_mutex);
591
592	#define PMC_INIT 0
593	#define PMC_RELEASE 1
594	#define PMC_FAILURE 2
595	static void setup_pmc_cpu(void *flags)
596	{
597	struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf);
598	int err = 0;
599
600	switch (*((int *)flags)) {
601	case PMC_INIT:
602	memset(cpusf, 0, sizeof(*cpusf));
603	err = qsi(&cpusf->qsi);
604	if (err)
605	break;
606	cpusf->flags |= PMU_F_RESERVED;
607	err = sf_disable();
608	break;
609	case PMC_RELEASE:
610	cpusf->flags &= ~PMU_F_RESERVED;
611	err = sf_disable();
612	if (!err)
613	deallocate_buffers(cpuhw: cpusf);
614	break;
615	}
616	if (err) {
617	*((int *)flags) |= PMC_FAILURE;
618	pr_err("Switching off the sampling facility failed with rc %i\n", err);
619	}
620	}
621
622	static void release_pmc_hardware(void)
623	{
624	int flags = PMC_RELEASE;
625
626	irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
627	on_each_cpu(func: setup_pmc_cpu, info: &flags, wait: 1);
628	}
629
630	static int reserve_pmc_hardware(void)
631	{
632	int flags = PMC_INIT;
633
634	on_each_cpu(func: setup_pmc_cpu, info: &flags, wait: 1);
635	if (flags & PMC_FAILURE) {
636	release_pmc_hardware();
637	return -ENODEV;
638	}
639	irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
640
641	return 0;
642	}
643
644	static void hw_perf_event_destroy(struct perf_event *event)
645	{
646	/* Release PMC if this is the last perf event */
647	if (!atomic_add_unless(v: &num_events, a: -1, u: 1)) {
648	mutex_lock(&pmc_reserve_mutex);
649	if (atomic_dec_return(v: &num_events) == 0)
650	release_pmc_hardware();
651	mutex_unlock(lock: &pmc_reserve_mutex);
652	}
653	}
654
655	static void hw_init_period(struct hw_perf_event *hwc, u64 period)
656	{
657	hwc->sample_period = period;
658	hwc->last_period = hwc->sample_period;
659	local64_set(&hwc->period_left, hwc->sample_period);
660	}
661
662	static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si,
663	unsigned long rate)
664	{
665	return clamp_t(unsigned long, rate,
666	si->min_sampl_rate, si->max_sampl_rate);
667	}
668
669	static u32 cpumsf_pid_type(struct perf_event *event,
670	u32 pid, enum pid_type type)
671	{
672	struct task_struct *tsk;
673
674	/* Idle process */
675	if (!pid)
676	goto out;
677
678	tsk = find_task_by_pid_ns(nr: pid, ns: &init_pid_ns);
679	pid = -1;
680	if (tsk) {
681	/*
682	* Only top level events contain the pid namespace in which
683	* they are created.
684	*/
685	if (event->parent)
686	event = event->parent;
687	pid = __task_pid_nr_ns(task: tsk, type, ns: event->ns);
688	/*
689	* See also 1d953111b648
690	* "perf/core: Don't report zero PIDs for exiting tasks".
691	*/
692	if (!pid && !pid_alive(p: tsk))
693	pid = -1;
694	}
695	out:
696	return pid;
697	}
698
699	static void cpumsf_output_event_pid(struct perf_event *event,
700	struct perf_sample_data *data,
701	struct pt_regs *regs)
702	{
703	u32 pid;
704	struct perf_event_header header;
705	struct perf_output_handle handle;
706
707	/*
708	* Obtain the PID from the basic-sampling data entry and
709	* correct the data->tid_entry.pid value.
710	*/
711	pid = data->tid_entry.pid;
712
713	/* Protect callchain buffers, tasks */
714	rcu_read_lock();
715
716	perf_prepare_sample(data, event, regs);
717	perf_prepare_header(header: &header, data, event, regs);
718	if (perf_output_begin(handle: &handle, data, event, size: header.size))
719	goto out;
720
721	/* Update the process ID (see also kernel/events/core.c) */
722	data->tid_entry.pid = cpumsf_pid_type(event, pid, type: PIDTYPE_TGID);
723	data->tid_entry.tid = cpumsf_pid_type(event, pid, type: PIDTYPE_PID);
724
725	perf_output_sample(handle: &handle, header: &header, data, event);
726	perf_output_end(handle: &handle);
727	out:
728	rcu_read_unlock();
729	}
730
731	static unsigned long getrate(bool freq, unsigned long sample,
732	struct hws_qsi_info_block *si)
733	{
734	unsigned long rate;
735
736	if (freq) {
737	rate = freq_to_sample_rate(qsi: si, freq: sample);
738	rate = hw_limit_rate(si, rate);
739	} else {
740	/* The min/max sampling rates specifies the valid range
741	* of sample periods. If the specified sample period is
742	* out of range, limit the period to the range boundary.
743	*/
744	rate = hw_limit_rate(si, rate: sample);
745
746	/* The perf core maintains a maximum sample rate that is
747	* configurable through the sysctl interface. Ensure the
748	* sampling rate does not exceed this value. This also helps
749	* to avoid throttling when pushing samples with
750	* perf_event_overflow().
751	*/
752	if (sample_rate_to_freq(qsi: si, rate) >
753	sysctl_perf_event_sample_rate) {
754	debug_sprintf_event(sfdbg, 1, "%s: "
755	"Sampling rate exceeds maximum "
756	"perf sample rate\n", __func__);
757	rate = 0;
758	}
759	}
760	return rate;
761	}
762
763	/* The sampling information (si) contains information about the
764	* min/max sampling intervals and the CPU speed. So calculate the
765	* correct sampling interval and avoid the whole period adjust
766	* feedback loop.
767	*
768	* Since the CPU Measurement sampling facility can not handle frequency
769	* calculate the sampling interval when frequency is specified using
770	* this formula:
771	* interval := cpu_speed * 1000000 / sample_freq
772	*
773	* Returns errno on bad input and zero on success with parameter interval
774	* set to the correct sampling rate.
775	*
776	* Note: This function turns off freq bit to avoid calling function
777	* perf_adjust_period(). This causes frequency adjustment in the common
778	* code part which causes tremendous variations in the counter values.
779	*/
780	static int __hw_perf_event_init_rate(struct perf_event *event,
781	struct hws_qsi_info_block *si)
782	{
783	struct perf_event_attr *attr = &event->attr;
784	struct hw_perf_event *hwc = &event->hw;
785	unsigned long rate;
786
787	if (attr->freq) {
788	if (!attr->sample_freq)
789	return -EINVAL;
790	rate = getrate(freq: attr->freq, sample: attr->sample_freq, si);
791	attr->freq = 0; /* Don't call perf_adjust_period() */
792	SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FREQ_MODE;
793	} else {
794	rate = getrate(freq: attr->freq, sample: attr->sample_period, si);
795	if (!rate)
796	return -EINVAL;
797	}
798	attr->sample_period = rate;
799	SAMPL_RATE(hwc) = rate;
800	hw_init_period(hwc, period: SAMPL_RATE(hwc));
801	debug_sprintf_event(sfdbg, 4, "%s: cpu %d period %#llx freq %d,%#lx\n",
802	__func__, event->cpu, event->attr.sample_period,
803	event->attr.freq, SAMPLE_FREQ_MODE(hwc));
804	return 0;
805	}
806
807	static int __hw_perf_event_init(struct perf_event *event)
808	{
809	struct cpu_hw_sf *cpuhw;
810	struct hws_qsi_info_block si;
811	struct perf_event_attr *attr = &event->attr;
812	struct hw_perf_event *hwc = &event->hw;
813	int cpu, err;
814
815	/* Reserve CPU-measurement sampling facility */
816	err = 0;
817	if (!atomic_inc_not_zero(v: &num_events)) {
818	mutex_lock(&pmc_reserve_mutex);
819	if (atomic_read(v: &num_events) == 0 && reserve_pmc_hardware())
820	err = -EBUSY;
821	else
822	atomic_inc(v: &num_events);
823	mutex_unlock(lock: &pmc_reserve_mutex);
824	}
825	event->destroy = hw_perf_event_destroy;
826
827	if (err)
828	goto out;
829
830	/* Access per-CPU sampling information (query sampling info) */
831	/*
832	* The event->cpu value can be -1 to count on every CPU, for example,
833	* when attaching to a task. If this is specified, use the query
834	* sampling info from the current CPU, otherwise use event->cpu to
835	* retrieve the per-CPU information.
836	* Later, cpuhw indicates whether to allocate sampling buffers for a
837	* particular CPU (cpuhw!=NULL) or each online CPU (cpuw==NULL).
838	*/
839	memset(&si, 0, sizeof(si));
840	cpuhw = NULL;
841	if (event->cpu == -1)
842	qsi(&si);
843	else {
844	/* Event is pinned to a particular CPU, retrieve the per-CPU
845	* sampling structure for accessing the CPU-specific QSI.
846	*/
847	cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
848	si = cpuhw->qsi;
849	}
850
851	/* Check sampling facility authorization and, if not authorized,
852	* fall back to other PMUs. It is safe to check any CPU because
853	* the authorization is identical for all configured CPUs.
854	*/
855	if (!si.as) {
856	err = -ENOENT;
857	goto out;
858	}
859
860	if (si.ribm & CPU_MF_SF_RIBM_NOTAV) {
861	pr_warn("CPU Measurement Facility sampling is temporarily not available\n");
862	err = -EBUSY;
863	goto out;
864	}
865
866	/* Always enable basic sampling */
867	SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE;
868
869	/* Check if diagnostic sampling is requested. Deny if the required
870	* sampling authorization is missing.
871	*/
872	if (attr->config == PERF_EVENT_CPUM_SF_DIAG) {
873	if (!si.ad) {
874	err = -EPERM;
875	goto out;
876	}
877	SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE;
878	}
879
880	err = __hw_perf_event_init_rate(event, si: &si);
881	if (err)
882	goto out;
883
884	/* Initialize sample data overflow accounting */
885	hwc->extra_reg.reg = REG_OVERFLOW;
886	OVERFLOW_REG(hwc) = 0;
887
888	/* Use AUX buffer. No need to allocate it by ourself */
889	if (attr->config == PERF_EVENT_CPUM_SF_DIAG)
890	return 0;
891
892	/* Allocate the per-CPU sampling buffer using the CPU information
893	* from the event. If the event is not pinned to a particular
894	* CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling
895	* buffers for each online CPU.
896	*/
897	if (cpuhw)
898	/* Event is pinned to a particular CPU */
899	err = allocate_buffers(cpuhw, hwc);
900	else {
901	/* Event is not pinned, allocate sampling buffer on
902	* each online CPU
903	*/
904	for_each_online_cpu(cpu) {
905	cpuhw = &per_cpu(cpu_hw_sf, cpu);
906	err = allocate_buffers(cpuhw, hwc);
907	if (err)
908	break;
909	}
910	}
911
912	/* If PID/TID sampling is active, replace the default overflow
913	* handler to extract and resolve the PIDs from the basic-sampling
914	* data entries.
915	*/
916	if (event->attr.sample_type & PERF_SAMPLE_TID)
917	if (is_default_overflow_handler(event))
918	event->overflow_handler = cpumsf_output_event_pid;
919	out:
920	return err;
921	}
922
923	static bool is_callchain_event(struct perf_event *event)
924	{
925	u64 sample_type = event->attr.sample_type;
926
927	return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER |
928	PERF_SAMPLE_STACK_USER);
929	}
930
931	static int cpumsf_pmu_event_init(struct perf_event *event)
932	{
933	int err;
934
935	/* No support for taken branch sampling */
936	/* No support for callchain, stacks and registers */
937	if (has_branch_stack(event) || is_callchain_event(event))
938	return -EOPNOTSUPP;
939
940	switch (event->attr.type) {
941	case PERF_TYPE_RAW:
942	if ((event->attr.config != PERF_EVENT_CPUM_SF) &&
943	(event->attr.config != PERF_EVENT_CPUM_SF_DIAG))
944	return -ENOENT;
945	break;
946	case PERF_TYPE_HARDWARE:
947	/* Support sampling of CPU cycles in addition to the
948	* counter facility. However, the counter facility
949	* is more precise and, hence, restrict this PMU to
950	* sampling events only.
951	*/
952	if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES)
953	return -ENOENT;
954	if (!is_sampling_event(event))
955	return -ENOENT;
956	break;
957	default:
958	return -ENOENT;
959	}
960
961	/* Force reset of idle/hv excludes regardless of what the
962	* user requested.
963	*/
964	if (event->attr.exclude_hv)
965	event->attr.exclude_hv = 0;
966	if (event->attr.exclude_idle)
967	event->attr.exclude_idle = 0;
968
969	err = __hw_perf_event_init(event);
970	if (unlikely(err))
971	if (event->destroy)
972	event->destroy(event);
973	return err;
974	}
975
976	static void cpumsf_pmu_enable(struct pmu *pmu)
977	{
978	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
979	struct hw_perf_event *hwc;
980	int err;
981
982	if (cpuhw->flags & PMU_F_ENABLED)
983	return;
984
985	if (cpuhw->flags & PMU_F_ERR_MASK)
986	return;
987
988	/* Check whether to extent the sampling buffer.
989	*
990	* Two conditions trigger an increase of the sampling buffer for a
991	* perf event:
992	* 1. Postponed buffer allocations from the event initialization.
993	* 2. Sampling overflows that contribute to pending allocations.
994	*
995	* Note that the extend_sampling_buffer() function disables the sampling
996	* facility, but it can be fully re-enabled using sampling controls that
997	* have been saved in cpumsf_pmu_disable().
998	*/
999	if (cpuhw->event) {
1000	hwc = &cpuhw->event->hw;
1001	if (!(SAMPL_DIAG_MODE(hwc))) {
1002	/*
1003	* Account number of overflow-designated
1004	* buffer extents
1005	*/
1006	sfb_account_overflows(cpuhw, hwc);
1007	extend_sampling_buffer(sfb: &cpuhw->sfb, hwc);
1008	}
1009	/* Rate may be adjusted with ioctl() */
1010	cpuhw->lsctl.interval = SAMPL_RATE(&cpuhw->event->hw);
1011	}
1012
1013	/* (Re)enable the PMU and sampling facility */
1014	cpuhw->flags |= PMU_F_ENABLED;
1015	barrier();
1016
1017	err = lsctl(&cpuhw->lsctl);
1018	if (err) {
1019	cpuhw->flags &= ~PMU_F_ENABLED;
1020	pr_err("Loading sampling controls failed: op 1 err %i\n", err);
1021	return;
1022	}
1023
1024	/* Load current program parameter */
1025	lpp(&S390_lowcore.lpp);
1026
1027	debug_sprintf_event(sfdbg, 6, "%s: es %i cs %i ed %i cd %i "
1028	"interval %#lx tear %#lx dear %#lx\n", __func__,
1029	cpuhw->lsctl.es, cpuhw->lsctl.cs, cpuhw->lsctl.ed,
1030	cpuhw->lsctl.cd, cpuhw->lsctl.interval,
1031	cpuhw->lsctl.tear, cpuhw->lsctl.dear);
1032	}
1033
1034	static void cpumsf_pmu_disable(struct pmu *pmu)
1035	{
1036	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1037	struct hws_lsctl_request_block inactive;
1038	struct hws_qsi_info_block si;
1039	int err;
1040
1041	if (!(cpuhw->flags & PMU_F_ENABLED))
1042	return;
1043
1044	if (cpuhw->flags & PMU_F_ERR_MASK)
1045	return;
1046
1047	/* Switch off sampling activation control */
1048	inactive = cpuhw->lsctl;
1049	inactive.cs = 0;
1050	inactive.cd = 0;
1051
1052	err = lsctl(&inactive);
1053	if (err) {
1054	pr_err("Loading sampling controls failed: op 2 err %i\n", err);
1055	return;
1056	}
1057
1058	/* Save state of TEAR and DEAR register contents */
1059	err = qsi(&si);
1060	if (!err) {
1061	/* TEAR/DEAR values are valid only if the sampling facility is
1062	* enabled. Note that cpumsf_pmu_disable() might be called even
1063	* for a disabled sampling facility because cpumsf_pmu_enable()
1064	* controls the enable/disable state.
1065	*/
1066	if (si.es) {
1067	cpuhw->lsctl.tear = si.tear;
1068	cpuhw->lsctl.dear = si.dear;
1069	}
1070	} else
1071	debug_sprintf_event(sfdbg, 3, "%s: qsi() failed with err %i\n",
1072	__func__, err);
1073
1074	cpuhw->flags &= ~PMU_F_ENABLED;
1075	}
1076
1077	/* perf_exclude_event() - Filter event
1078	* @event: The perf event
1079	* @regs: pt_regs structure
1080	* @sde_regs: Sample-data-entry (sde) regs structure
1081	*
1082	* Filter perf events according to their exclude specification.
1083	*
1084	* Return non-zero if the event shall be excluded.
1085	*/
1086	static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs,
1087	struct perf_sf_sde_regs *sde_regs)
1088	{
1089	if (event->attr.exclude_user && user_mode(regs))
1090	return 1;
1091	if (event->attr.exclude_kernel && !user_mode(regs))
1092	return 1;
1093	if (event->attr.exclude_guest && sde_regs->in_guest)
1094	return 1;
1095	if (event->attr.exclude_host && !sde_regs->in_guest)
1096	return 1;
1097	return 0;
1098	}
1099
1100	/* perf_push_sample() - Push samples to perf
1101	* @event: The perf event
1102	* @sample: Hardware sample data
1103	*
1104	* Use the hardware sample data to create perf event sample. The sample
1105	* is the pushed to the event subsystem and the function checks for
1106	* possible event overflows. If an event overflow occurs, the PMU is
1107	* stopped.
1108	*
1109	* Return non-zero if an event overflow occurred.
1110	*/
1111	static int perf_push_sample(struct perf_event *event,
1112	struct hws_basic_entry *basic)
1113	{
1114	int overflow;
1115	struct pt_regs regs;
1116	struct perf_sf_sde_regs *sde_regs;
1117	struct perf_sample_data data;
1118
1119	/* Setup perf sample */
1120	perf_sample_data_init(data: &data, addr: 0, period: event->hw.last_period);
1121
1122	/* Setup pt_regs to look like an CPU-measurement external interrupt
1123	* using the Program Request Alert code. The regs.int_parm_long
1124	* field which is unused contains additional sample-data-entry related
1125	* indicators.
1126	*/
1127	memset(&regs, 0, sizeof(regs));
1128	regs.int_code = 0x1407;
1129	regs.int_parm = CPU_MF_INT_SF_PRA;
1130	sde_regs = (struct perf_sf_sde_regs *) &regs.int_parm_long;
1131
1132	psw_bits(regs.psw).ia = basic->ia;
1133	psw_bits(regs.psw).dat = basic->T;
1134	psw_bits(regs.psw).wait = basic->W;
1135	psw_bits(regs.psw).pstate = basic->P;
1136	psw_bits(regs.psw).as = basic->AS;
1137
1138	/*
1139	* Use the hardware provided configuration level to decide if the
1140	* sample belongs to a guest or host. If that is not available,
1141	* fall back to the following heuristics:
1142	* A non-zero guest program parameter always indicates a guest
1143	* sample. Some early samples or samples from guests without
1144	* lpp usage would be misaccounted to the host. We use the asn
1145	* value as an addon heuristic to detect most of these guest samples.
1146	* If the value differs from 0xffff (the host value), we assume to
1147	* be a KVM guest.
1148	*/
1149	switch (basic->CL) {
1150	case 1: /* logical partition */
1151	sde_regs->in_guest = 0;
1152	break;
1153	case 2: /* virtual machine */
1154	sde_regs->in_guest = 1;
1155	break;
1156	default: /* old machine, use heuristics */
1157	if (basic->gpp || basic->prim_asn != 0xffff)
1158	sde_regs->in_guest = 1;
1159	break;
1160	}
1161
1162	/*
1163	* Store the PID value from the sample-data-entry to be
1164	* processed and resolved by cpumsf_output_event_pid().
1165	*/
1166	data.tid_entry.pid = basic->hpp & LPP_PID_MASK;
1167
1168	overflow = 0;
1169	if (perf_exclude_event(event, regs: &regs, sde_regs))
1170	goto out;
1171	if (perf_event_overflow(event, data: &data, regs: &regs)) {
1172	overflow = 1;
1173	event->pmu->stop(event, 0);
1174	}
1175	perf_event_update_userpage(event);
1176	out:
1177	return overflow;
1178	}
1179
1180	static void perf_event_count_update(struct perf_event *event, u64 count)
1181	{
1182	local64_add(count, &event->count);
1183	}
1184
1185	/* hw_collect_samples() - Walk through a sample-data-block and collect samples
1186	* @event: The perf event
1187	* @sdbt: Sample-data-block table
1188	* @overflow: Event overflow counter
1189	*
1190	* Walks through a sample-data-block and collects sampling data entries that are
1191	* then pushed to the perf event subsystem. Depending on the sampling function,
1192	* there can be either basic-sampling or combined-sampling data entries. A
1193	* combined-sampling data entry consists of a basic- and a diagnostic-sampling
1194	* data entry. The sampling function is determined by the flags in the perf
1195	* event hardware structure. The function always works with a combined-sampling
1196	* data entry but ignores the the diagnostic portion if it is not available.
1197	*
1198	* Note that the implementation focuses on basic-sampling data entries and, if
1199	* such an entry is not valid, the entire combined-sampling data entry is
1200	* ignored.
1201	*
1202	* The overflow variables counts the number of samples that has been discarded
1203	* due to a perf event overflow.
1204	*/
1205	static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt,
1206	unsigned long long *overflow)
1207	{
1208	struct hws_trailer_entry *te;
1209	struct hws_basic_entry *sample;
1210
1211	te = trailer_entry_ptr(v: (unsigned long)sdbt);
1212	sample = (struct hws_basic_entry *)sdbt;
1213	while ((unsigned long *)sample < (unsigned long *)te) {
1214	/* Check for an empty sample */
1215	if (!sample->def || sample->LS)
1216	break;
1217
1218	/* Update perf event period */
1219	perf_event_count_update(event, count: SAMPL_RATE(&event->hw));
1220
1221	/* Check whether sample is valid */
1222	if (sample->def == 0x0001) {
1223	/* If an event overflow occurred, the PMU is stopped to
1224	* throttle event delivery. Remaining sample data is
1225	* discarded.
1226	*/
1227	if (!*overflow) {
1228	/* Check whether sample is consistent */
1229	if (sample->I == 0 && sample->W == 0) {
1230	/* Deliver sample data to perf */
1231	*overflow = perf_push_sample(event,
1232	basic: sample);
1233	}
1234	} else
1235	/* Count discarded samples */
1236	*overflow += 1;
1237	} else {
1238	debug_sprintf_event(sfdbg, 4,
1239	"%s: Found unknown"
1240	" sampling data entry: te->f %i"
1241	" basic.def %#4x (%p)\n", __func__,
1242	te->header.f, sample->def, sample);
1243	/* Sample slot is not yet written or other record.
1244	*
1245	* This condition can occur if the buffer was reused
1246	* from a combined basic- and diagnostic-sampling.
1247	* If only basic-sampling is then active, entries are
1248	* written into the larger diagnostic entries.
1249	* This is typically the case for sample-data-blocks
1250	* that are not full. Stop processing if the first
1251	* invalid format was detected.
1252	*/
1253	if (!te->header.f)
1254	break;
1255	}
1256
1257	/* Reset sample slot and advance to next sample */
1258	sample->def = 0;
1259	sample++;
1260	}
1261	}
1262
1263	/* hw_perf_event_update() - Process sampling buffer
1264	* @event: The perf event
1265	* @flush_all: Flag to also flush partially filled sample-data-blocks
1266	*
1267	* Processes the sampling buffer and create perf event samples.
1268	* The sampling buffer position are retrieved and saved in the TEAR_REG
1269	* register of the specified perf event.
1270	*
1271	* Only full sample-data-blocks are processed. Specify the flush_all flag
1272	* to also walk through partially filled sample-data-blocks.
1273	*/
1274	static void hw_perf_event_update(struct perf_event *event, int flush_all)
1275	{
1276	unsigned long long event_overflow, sampl_overflow, num_sdb;
1277	union hws_trailer_header old, prev, new;
1278	struct hw_perf_event *hwc = &event->hw;
1279	struct hws_trailer_entry *te;
1280	unsigned long *sdbt, sdb;
1281	int done;
1282
1283	/*
1284	* AUX buffer is used when in diagnostic sampling mode.
1285	* No perf events/samples are created.
1286	*/
1287	if (SAMPL_DIAG_MODE(&event->hw))
1288	return;
1289
1290	sdbt = (unsigned long *)TEAR_REG(hwc);
1291	done = event_overflow = sampl_overflow = num_sdb = 0;
1292	while (!done) {
1293	/* Get the trailer entry of the sample-data-block */
1294	sdb = (unsigned long)phys_to_virt(address: *sdbt);
1295	te = trailer_entry_ptr(v: sdb);
1296
1297	/* Leave loop if no more work to do (block full indicator) */
1298	if (!te->header.f) {
1299	done = 1;
1300	if (!flush_all)
1301	break;
1302	}
1303
1304	/* Check the sample overflow count */
1305	if (te->header.overflow)
1306	/* Account sample overflows and, if a particular limit
1307	* is reached, extend the sampling buffer.
1308	* For details, see sfb_account_overflows().
1309	*/
1310	sampl_overflow += te->header.overflow;
1311
1312	/* Timestamps are valid for full sample-data-blocks only */
1313	debug_sprintf_event(sfdbg, 6, "%s: sdbt %#lx/%#lx "
1314	"overflow %llu timestamp %#llx\n",
1315	__func__, sdb, (unsigned long)sdbt,
1316	te->header.overflow,
1317	(te->header.f) ? trailer_timestamp(te) : 0ULL);
1318
1319	/* Collect all samples from a single sample-data-block and
1320	* flag if an (perf) event overflow happened. If so, the PMU
1321	* is stopped and remaining samples will be discarded.
1322	*/
1323	hw_collect_samples(event, sdbt: (unsigned long *)sdb, overflow: &event_overflow);
1324	num_sdb++;
1325
1326	/* Reset trailer (using compare-double-and-swap) */
1327	prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1328	do {
1329	old.val = prev.val;
1330	new.val = prev.val;
1331	new.f = 0;
1332	new.a = 1;
1333	new.overflow = 0;
1334	prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1335	} while (prev.val != old.val);
1336
1337	/* Advance to next sample-data-block */
1338	sdbt++;
1339	if (is_link_entry(s: sdbt))
1340	sdbt = get_next_sdbt(s: sdbt);
1341
1342	/* Update event hardware registers */
1343	TEAR_REG(hwc) = (unsigned long) sdbt;
1344
1345	/* Stop processing sample-data if all samples of the current
1346	* sample-data-block were flushed even if it was not full.
1347	*/
1348	if (flush_all && done)
1349	break;
1350	}
1351
1352	/* Account sample overflows in the event hardware structure */
1353	if (sampl_overflow)
1354	OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) +
1355	sampl_overflow, 1 + num_sdb);
1356
1357	/* Perf_event_overflow() and perf_event_account_interrupt() limit
1358	* the interrupt rate to an upper limit. Roughly 1000 samples per
1359	* task tick.
1360	* Hitting this limit results in a large number
1361	* of throttled REF_REPORT_THROTTLE entries and the samples
1362	* are dropped.
1363	* Slightly increase the interval to avoid hitting this limit.
1364	*/
1365	if (event_overflow) {
1366	SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10);
1367	debug_sprintf_event(sfdbg, 1, "%s: rate adjustment %ld\n",
1368	__func__,
1369	DIV_ROUND_UP(SAMPL_RATE(hwc), 10));
1370	}
1371
1372	if (sampl_overflow || event_overflow)
1373	debug_sprintf_event(sfdbg, 4, "%s: "
1374	"overflows: sample %llu event %llu"
1375	" total %llu num_sdb %llu\n",
1376	__func__, sampl_overflow, event_overflow,
1377	OVERFLOW_REG(hwc), num_sdb);
1378	}
1379
1380	static inline unsigned long aux_sdb_index(struct aux_buffer *aux,
1381	unsigned long i)
1382	{
1383	return i % aux->sfb.num_sdb;
1384	}
1385
1386	static inline unsigned long aux_sdb_num(unsigned long start, unsigned long end)
1387	{
1388	return end >= start ? end - start + 1 : 0;
1389	}
1390
1391	static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)
1392	{
1393	return aux_sdb_num(start: aux->head, end: aux->alert_mark);
1394	}
1395
1396	static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)
1397	{
1398	return aux_sdb_num(start: aux->head, end: aux->empty_mark);
1399	}
1400
1401	/*
1402	* Get trailer entry by index of SDB.
1403	*/
1404	static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,
1405	unsigned long index)
1406	{
1407	unsigned long sdb;
1408
1409	index = aux_sdb_index(aux, i: index);
1410	sdb = aux->sdb_index[index];
1411	return trailer_entry_ptr(v: sdb);
1412	}
1413
1414	/*
1415	* Finish sampling on the cpu. Called by cpumsf_pmu_del() with pmu
1416	* disabled. Collect the full SDBs in AUX buffer which have not reached
1417	* the point of alert indicator. And ignore the SDBs which are not
1418	* full.
1419	*
1420	* 1. Scan SDBs to see how much data is there and consume them.
1421	* 2. Remove alert indicator in the buffer.
1422	*/
1423	static void aux_output_end(struct perf_output_handle *handle)
1424	{
1425	unsigned long i, range_scan, idx;
1426	struct aux_buffer *aux;
1427	struct hws_trailer_entry *te;
1428
1429	aux = perf_get_aux(handle);
1430	if (!aux)
1431	return;
1432
1433	range_scan = aux_sdb_num_alert(aux);
1434	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {
1435	te = aux_sdb_trailer(aux, index: idx);
1436	if (!te->header.f)
1437	break;
1438	}
1439	/* i is num of SDBs which are full */
1440	perf_aux_output_end(handle, size: i << PAGE_SHIFT);
1441
1442	/* Remove alert indicators in the buffer */
1443	te = aux_sdb_trailer(aux, index: aux->alert_mark);
1444	te->header.a = 0;
1445
1446	debug_sprintf_event(sfdbg, 6, "%s: SDBs %ld range %ld head %ld\n",
1447	__func__, i, range_scan, aux->head);
1448	}
1449
1450	/*
1451	* Start sampling on the CPU. Called by cpumsf_pmu_add() when an event
1452	* is first added to the CPU or rescheduled again to the CPU. It is called
1453	* with pmu disabled.
1454	*
1455	* 1. Reset the trailer of SDBs to get ready for new data.
1456	* 2. Tell the hardware where to put the data by reset the SDBs buffer
1457	* head(tear/dear).
1458	*/
1459	static int aux_output_begin(struct perf_output_handle *handle,
1460	struct aux_buffer *aux,
1461	struct cpu_hw_sf *cpuhw)
1462	{
1463	unsigned long range, i, range_scan, idx, head, base, offset;
1464	struct hws_trailer_entry *te;
1465
1466	if (WARN_ON_ONCE(handle->head & ~PAGE_MASK))
1467	return -EINVAL;
1468
1469	aux->head = handle->head >> PAGE_SHIFT;
1470	range = (handle->size + 1) >> PAGE_SHIFT;
1471	if (range <= 1)
1472	return -ENOMEM;
1473
1474	/*
1475	* SDBs between aux->head and aux->empty_mark are already ready
1476	* for new data. range_scan is num of SDBs not within them.
1477	*/
1478	debug_sprintf_event(sfdbg, 6,
1479	"%s: range %ld head %ld alert %ld empty %ld\n",
1480	__func__, range, aux->head, aux->alert_mark,
1481	aux->empty_mark);
1482	if (range > aux_sdb_num_empty(aux)) {
1483	range_scan = range - aux_sdb_num_empty(aux);
1484	idx = aux->empty_mark + 1;
1485	for (i = 0; i < range_scan; i++, idx++) {
1486	te = aux_sdb_trailer(aux, index: idx);
1487	te->header.f = 0;
1488	te->header.a = 0;
1489	te->header.overflow = 0;
1490	}
1491	/* Save the position of empty SDBs */
1492	aux->empty_mark = aux->head + range - 1;
1493	}
1494
1495	/* Set alert indicator */
1496	aux->alert_mark = aux->head + range/2 - 1;
1497	te = aux_sdb_trailer(aux, index: aux->alert_mark);
1498	te->header.a = 1;
1499
1500	/* Reset hardware buffer head */
1501	head = aux_sdb_index(aux, i: aux->head);
1502	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];
1503	offset = head % CPUM_SF_SDB_PER_TABLE;
1504	cpuhw->lsctl.tear = virt_to_phys(address: (void *)base) + offset * sizeof(unsigned long);
1505	cpuhw->lsctl.dear = virt_to_phys(address: (void *)aux->sdb_index[head]);
1506
1507	debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld empty %ld "
1508	"index %ld tear %#lx dear %#lx\n", __func__,
1509	aux->head, aux->alert_mark, aux->empty_mark,
1510	head / CPUM_SF_SDB_PER_TABLE,
1511	cpuhw->lsctl.tear, cpuhw->lsctl.dear);
1512
1513	return 0;
1514	}
1515
1516	/*
1517	* Set alert indicator on SDB at index @alert_index while sampler is running.
1518	*
1519	* Return true if successfully.
1520	* Return false if full indicator is already set by hardware sampler.
1521	*/
1522	static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,
1523	unsigned long long *overflow)
1524	{
1525	union hws_trailer_header old, prev, new;
1526	struct hws_trailer_entry *te;
1527
1528	te = aux_sdb_trailer(aux, index: alert_index);
1529	prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1530	do {
1531	old.val = prev.val;
1532	new.val = prev.val;
1533	*overflow = old.overflow;
1534	if (old.f) {
1535	/*
1536	* SDB is already set by hardware.
1537	* Abort and try to set somewhere
1538	* behind.
1539	*/
1540	return false;
1541	}
1542	new.a = 1;
1543	new.overflow = 0;
1544	prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1545	} while (prev.val != old.val);
1546	return true;
1547	}
1548
1549	/*
1550	* aux_reset_buffer() - Scan and setup SDBs for new samples
1551	* @aux: The AUX buffer to set
1552	* @range: The range of SDBs to scan started from aux->head
1553	* @overflow: Set to overflow count
1554	*
1555	* Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1556	* marked as empty, check if it is already set full by the hardware sampler.
1557	* If yes, that means new data is already there before we can set an alert
1558	* indicator. Caller should try to set alert indicator to some position behind.
1559	*
1560	* Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1561	* previously and have already been consumed by user space. Reset these SDBs
1562	* (clear full indicator and alert indicator) for new data.
1563	* If aux->alert_mark fall in this area, just set it. Overflow count is
1564	* recorded while scanning.
1565	*
1566	* SDBs between aux->head and aux->empty_mark are already reset at last time.
1567	* and ready for new samples. So scanning on this area could be skipped.
1568	*
1569	* Return true if alert indicator is set successfully and false if not.
1570	*/
1571	static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,
1572	unsigned long long *overflow)
1573	{
1574	unsigned long i, range_scan, idx, idx_old;
1575	union hws_trailer_header old, prev, new;
1576	unsigned long long orig_overflow;
1577	struct hws_trailer_entry *te;
1578
1579	debug_sprintf_event(sfdbg, 6, "%s: range %ld head %ld alert %ld "
1580	"empty %ld\n", __func__, range, aux->head,
1581	aux->alert_mark, aux->empty_mark);
1582	if (range <= aux_sdb_num_empty(aux))
1583	/*
1584	* No need to scan. All SDBs in range are marked as empty.
1585	* Just set alert indicator. Should check race with hardware
1586	* sampler.
1587	*/
1588	return aux_set_alert(aux, alert_index: aux->alert_mark, overflow);
1589
1590	if (aux->alert_mark <= aux->empty_mark)
1591	/*
1592	* Set alert indicator on empty SDB. Should check race
1593	* with hardware sampler.
1594	*/
1595	if (!aux_set_alert(aux, alert_index: aux->alert_mark, overflow))
1596	return false;
1597
1598	/*
1599	* Scan the SDBs to clear full and alert indicator used previously.
1600	* Start scanning from one SDB behind empty_mark. If the new alert
1601	* indicator fall into this range, set it.
1602	*/
1603	range_scan = range - aux_sdb_num_empty(aux);
1604	idx_old = idx = aux->empty_mark + 1;
1605	for (i = 0; i < range_scan; i++, idx++) {
1606	te = aux_sdb_trailer(aux, index: idx);
1607	prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1608	do {
1609	old.val = prev.val;
1610	new.val = prev.val;
1611	orig_overflow = old.overflow;
1612	new.f = 0;
1613	new.overflow = 0;
1614	if (idx == aux->alert_mark)
1615	new.a = 1;
1616	else
1617	new.a = 0;
1618	prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1619	} while (prev.val != old.val);
1620	*overflow += orig_overflow;
1621	}
1622
1623	/* Update empty_mark to new position */
1624	aux->empty_mark = aux->head + range - 1;
1625
1626	debug_sprintf_event(sfdbg, 6, "%s: range_scan %ld idx %ld..%ld "
1627	"empty %ld\n", __func__, range_scan, idx_old,
1628	idx - 1, aux->empty_mark);
1629	return true;
1630	}
1631
1632	/*
1633	* Measurement alert handler for diagnostic mode sampling.
1634	*/
1635	static void hw_collect_aux(struct cpu_hw_sf *cpuhw)
1636	{
1637	struct aux_buffer *aux;
1638	int done = 0;
1639	unsigned long range = 0, size;
1640	unsigned long long overflow = 0;
1641	struct perf_output_handle *handle = &cpuhw->handle;
1642	unsigned long num_sdb;
1643
1644	aux = perf_get_aux(handle);
1645	if (WARN_ON_ONCE(!aux))
1646	return;
1647
1648	/* Inform user space new data arrived */
1649	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1650	debug_sprintf_event(sfdbg, 6, "%s: #alert %ld\n", __func__,
1651	size >> PAGE_SHIFT);
1652	perf_aux_output_end(handle, size);
1653
1654	num_sdb = aux->sfb.num_sdb;
1655	while (!done) {
1656	/* Get an output handle */
1657	aux = perf_aux_output_begin(handle, event: cpuhw->event);
1658	if (handle->size == 0) {
1659	pr_err("The AUX buffer with %lu pages for the "
1660	"diagnostic-sampling mode is full\n",
1661	num_sdb);
1662	break;
1663	}
1664	if (WARN_ON_ONCE(!aux))
1665	return;
1666
1667	/* Update head and alert_mark to new position */
1668	aux->head = handle->head >> PAGE_SHIFT;
1669	range = (handle->size + 1) >> PAGE_SHIFT;
1670	if (range == 1)
1671	aux->alert_mark = aux->head;
1672	else
1673	aux->alert_mark = aux->head + range/2 - 1;
1674
1675	if (aux_reset_buffer(aux, range, overflow: &overflow)) {
1676	if (!overflow) {
1677	done = 1;
1678	break;
1679	}
1680	size = range << PAGE_SHIFT;
1681	perf_aux_output_end(handle: &cpuhw->handle, size);
1682	pr_err("Sample data caused the AUX buffer with %lu "
1683	"pages to overflow\n", aux->sfb.num_sdb);
1684	debug_sprintf_event(sfdbg, 1, "%s: head %ld range %ld "
1685	"overflow %lld\n", __func__,
1686	aux->head, range, overflow);
1687	} else {
1688	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1689	perf_aux_output_end(handle: &cpuhw->handle, size);
1690	debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld "
1691	"already full, try another\n",
1692	__func__,
1693	aux->head, aux->alert_mark);
1694	}
1695	}
1696
1697	if (done)
1698	debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld "
1699	"empty %ld\n", __func__, aux->head,
1700	aux->alert_mark, aux->empty_mark);
1701	}
1702
1703	/*
1704	* Callback when freeing AUX buffers.
1705	*/
1706	static void aux_buffer_free(void *data)
1707	{
1708	struct aux_buffer *aux = data;
1709	unsigned long i, num_sdbt;
1710
1711	if (!aux)
1712	return;
1713
1714	/* Free SDBT. SDB is freed by the caller */
1715	num_sdbt = aux->sfb.num_sdbt;
1716	for (i = 0; i < num_sdbt; i++)
1717	free_page(aux->sdbt_index[i]);
1718
1719	kfree(objp: aux->sdbt_index);
1720	kfree(objp: aux->sdb_index);
1721	kfree(objp: aux);
1722
1723	debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu\n", __func__, num_sdbt);
1724	}
1725
1726	static void aux_sdb_init(unsigned long sdb)
1727	{
1728	struct hws_trailer_entry *te;
1729
1730	te = trailer_entry_ptr(v: sdb);
1731
1732	/* Save clock base */
1733	te->clock_base = 1;
1734	te->progusage2 = tod_clock_base.tod;
1735	}
1736
1737	/*
1738	* aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1739	* @event: Event the buffer is setup for, event->cpu == -1 means current
1740	* @pages: Array of pointers to buffer pages passed from perf core
1741	* @nr_pages: Total pages
1742	* @snapshot: Flag for snapshot mode
1743	*
1744	* This is the callback when setup an event using AUX buffer. Perf tool can
1745	* trigger this by an additional mmap() call on the event. Unlike the buffer
1746	* for basic samples, AUX buffer belongs to the event. It is scheduled with
1747	* the task among online cpus when it is a per-thread event.
1748	*
1749	* Return the private AUX buffer structure if success or NULL if fails.
1750	*/
1751	static void *aux_buffer_setup(struct perf_event *event, void **pages,
1752	int nr_pages, bool snapshot)
1753	{
1754	struct sf_buffer *sfb;
1755	struct aux_buffer *aux;
1756	unsigned long *new, *tail;
1757	int i, n_sdbt;
1758
1759	if (!nr_pages || !pages)
1760	return NULL;
1761
1762	if (nr_pages > CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1763	pr_err("AUX buffer size (%i pages) is larger than the "
1764	"maximum sampling buffer limit\n",
1765	nr_pages);
1766	return NULL;
1767	} else if (nr_pages < CPUM_SF_MIN_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1768	pr_err("AUX buffer size (%i pages) is less than the "
1769	"minimum sampling buffer limit\n",
1770	nr_pages);
1771	return NULL;
1772	}
1773
1774	/* Allocate aux_buffer struct for the event */
1775	aux = kzalloc(size: sizeof(struct aux_buffer), GFP_KERNEL);
1776	if (!aux)
1777	goto no_aux;
1778	sfb = &aux->sfb;
1779
1780	/* Allocate sdbt_index for fast reference */
1781	n_sdbt = DIV_ROUND_UP(nr_pages, CPUM_SF_SDB_PER_TABLE);
1782	aux->sdbt_index = kmalloc_array(n: n_sdbt, size: sizeof(void *), GFP_KERNEL);
1783	if (!aux->sdbt_index)
1784	goto no_sdbt_index;
1785
1786	/* Allocate sdb_index for fast reference */
1787	aux->sdb_index = kmalloc_array(n: nr_pages, size: sizeof(void *), GFP_KERNEL);
1788	if (!aux->sdb_index)
1789	goto no_sdb_index;
1790
1791	/* Allocate the first SDBT */
1792	sfb->num_sdbt = 0;
1793	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1794	if (!sfb->sdbt)
1795	goto no_sdbt;
1796	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;
1797	tail = sfb->tail = sfb->sdbt;
1798
1799	/*
1800	* Link the provided pages of AUX buffer to SDBT.
1801	* Allocate SDBT if needed.
1802	*/
1803	for (i = 0; i < nr_pages; i++, tail++) {
1804	if (require_table_link(sdbt: tail)) {
1805	new = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1806	if (!new)
1807	goto no_sdbt;
1808	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;
1809	/* Link current page to tail of chain */
1810	*tail = virt_to_phys(address: new) + 1;
1811	tail = new;
1812	}
1813	/* Tail is the entry in a SDBT */
1814	*tail = virt_to_phys(address: pages[i]);
1815	aux->sdb_index[i] = (unsigned long)pages[i];
1816	aux_sdb_init(sdb: (unsigned long)pages[i]);
1817	}
1818	sfb->num_sdb = nr_pages;
1819
1820	/* Link the last entry in the SDBT to the first SDBT */
1821	*tail = virt_to_phys(address: sfb->sdbt) + 1;
1822	sfb->tail = tail;
1823
1824	/*
1825	* Initial all SDBs are zeroed. Mark it as empty.
1826	* So there is no need to clear the full indicator
1827	* when this event is first added.
1828	*/
1829	aux->empty_mark = sfb->num_sdb - 1;
1830
1831	debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu SDBs %lu\n", __func__,
1832	sfb->num_sdbt, sfb->num_sdb);
1833
1834	return aux;
1835
1836	no_sdbt:
1837	/* SDBs (AUX buffer pages) are freed by caller */
1838	for (i = 0; i < sfb->num_sdbt; i++)
1839	free_page(aux->sdbt_index[i]);
1840	kfree(objp: aux->sdb_index);
1841	no_sdb_index:
1842	kfree(objp: aux->sdbt_index);
1843	no_sdbt_index:
1844	kfree(objp: aux);
1845	no_aux:
1846	return NULL;
1847	}
1848
1849	static void cpumsf_pmu_read(struct perf_event *event)
1850	{
1851	/* Nothing to do ... updates are interrupt-driven */
1852	}
1853
1854	/* Check if the new sampling period/frequency is appropriate.
1855	*
1856	* Return non-zero on error and zero on passed checks.
1857	*/
1858	static int cpumsf_pmu_check_period(struct perf_event *event, u64 value)
1859	{
1860	struct hws_qsi_info_block si;
1861	unsigned long rate;
1862	bool do_freq;
1863
1864	memset(&si, 0, sizeof(si));
1865	if (event->cpu == -1) {
1866	if (qsi(&si))
1867	return -ENODEV;
1868	} else {
1869	/* Event is pinned to a particular CPU, retrieve the per-CPU
1870	* sampling structure for accessing the CPU-specific QSI.
1871	*/
1872	struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
1873
1874	si = cpuhw->qsi;
1875	}
1876
1877	do_freq = !!SAMPLE_FREQ_MODE(&event->hw);
1878	rate = getrate(freq: do_freq, sample: value, si: &si);
1879	if (!rate)
1880	return -EINVAL;
1881
1882	event->attr.sample_period = rate;
1883	SAMPL_RATE(&event->hw) = rate;
1884	hw_init_period(hwc: &event->hw, period: SAMPL_RATE(&event->hw));
1885	debug_sprintf_event(sfdbg, 4, "%s:"
1886	" cpu %d value %#llx period %#llx freq %d\n",
1887	__func__, event->cpu, value,
1888	event->attr.sample_period, do_freq);
1889	return 0;
1890	}
1891
1892	/* Activate sampling control.
1893	* Next call of pmu_enable() starts sampling.
1894	*/
1895	static void cpumsf_pmu_start(struct perf_event *event, int flags)
1896	{
1897	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1898
1899	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1900	return;
1901
1902	if (flags & PERF_EF_RELOAD)
1903	WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1904
1905	perf_pmu_disable(pmu: event->pmu);
1906	event->hw.state = 0;
1907	cpuhw->lsctl.cs = 1;
1908	if (SAMPL_DIAG_MODE(&event->hw))
1909	cpuhw->lsctl.cd = 1;
1910	perf_pmu_enable(pmu: event->pmu);
1911	}
1912
1913	/* Deactivate sampling control.
1914	* Next call of pmu_enable() stops sampling.
1915	*/
1916	static void cpumsf_pmu_stop(struct perf_event *event, int flags)
1917	{
1918	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1919
1920	if (event->hw.state & PERF_HES_STOPPED)
1921	return;
1922
1923	perf_pmu_disable(pmu: event->pmu);
1924	cpuhw->lsctl.cs = 0;
1925	cpuhw->lsctl.cd = 0;
1926	event->hw.state |= PERF_HES_STOPPED;
1927
1928	if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
1929	hw_perf_event_update(event, flush_all: 1);
1930	event->hw.state |= PERF_HES_UPTODATE;
1931	}
1932	perf_pmu_enable(pmu: event->pmu);
1933	}
1934
1935	static int cpumsf_pmu_add(struct perf_event *event, int flags)
1936	{
1937	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1938	struct aux_buffer *aux;
1939	int err;
1940
1941	if (cpuhw->flags & PMU_F_IN_USE)
1942	return -EAGAIN;
1943
1944	if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)
1945	return -EINVAL;
1946
1947	err = 0;
1948	perf_pmu_disable(pmu: event->pmu);
1949
1950	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1951
1952	/* Set up sampling controls. Always program the sampling register
1953	* using the SDB-table start. Reset TEAR_REG event hardware register
1954	* that is used by hw_perf_event_update() to store the sampling buffer
1955	* position after samples have been flushed.
1956	*/
1957	cpuhw->lsctl.s = 0;
1958	cpuhw->lsctl.h = 1;
1959	cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);
1960	if (!SAMPL_DIAG_MODE(&event->hw)) {
1961	cpuhw->lsctl.tear = virt_to_phys(address: cpuhw->sfb.sdbt);
1962	cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt;
1963	TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt;
1964	}
1965
1966	/* Ensure sampling functions are in the disabled state. If disabled,
1967	* switch on sampling enable control. */
1968	if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {
1969	err = -EAGAIN;
1970	goto out;
1971	}
1972	if (SAMPL_DIAG_MODE(&event->hw)) {
1973	aux = perf_aux_output_begin(handle: &cpuhw->handle, event);
1974	if (!aux) {
1975	err = -EINVAL;
1976	goto out;
1977	}
1978	err = aux_output_begin(handle: &cpuhw->handle, aux, cpuhw);
1979	if (err)
1980	goto out;
1981	cpuhw->lsctl.ed = 1;
1982	}
1983	cpuhw->lsctl.es = 1;
1984
1985	/* Set in_use flag and store event */
1986	cpuhw->event = event;
1987	cpuhw->flags |= PMU_F_IN_USE;
1988
1989	if (flags & PERF_EF_START)
1990	cpumsf_pmu_start(event, PERF_EF_RELOAD);
1991	out:
1992	perf_event_update_userpage(event);
1993	perf_pmu_enable(pmu: event->pmu);
1994	return err;
1995	}
1996
1997	static void cpumsf_pmu_del(struct perf_event *event, int flags)
1998	{
1999	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
2000
2001	perf_pmu_disable(pmu: event->pmu);
2002	cpumsf_pmu_stop(event, PERF_EF_UPDATE);
2003
2004	cpuhw->lsctl.es = 0;
2005	cpuhw->lsctl.ed = 0;
2006	cpuhw->flags &= ~PMU_F_IN_USE;
2007	cpuhw->event = NULL;
2008
2009	if (SAMPL_DIAG_MODE(&event->hw))
2010	aux_output_end(handle: &cpuhw->handle);
2011	perf_event_update_userpage(event);
2012	perf_pmu_enable(pmu: event->pmu);
2013	}
2014
2015	CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC, PERF_EVENT_CPUM_SF);
2016	CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC_DIAG, PERF_EVENT_CPUM_SF_DIAG);
2017
2018	/* Attribute list for CPU_SF.
2019	*
2020	* The availablitiy depends on the CPU_MF sampling facility authorization
2021	* for basic + diagnositic samples. This is determined at initialization
2022	* time by the sampling facility device driver.
2023	* If the authorization for basic samples is turned off, it should be
2024	* also turned off for diagnostic sampling.
2025	*
2026	* During initialization of the device driver, check the authorization
2027	* level for diagnostic sampling and installs the attribute
2028	* file for diagnostic sampling if necessary.
2029	*
2030	* For now install a placeholder to reference all possible attributes:
2031	* SF_CYCLES_BASIC and SF_CYCLES_BASIC_DIAG.
2032	* Add another entry for the final NULL pointer.
2033	*/
2034	enum {
2035	SF_CYCLES_BASIC_ATTR_IDX = 0,
2036	SF_CYCLES_BASIC_DIAG_ATTR_IDX,
2037	SF_CYCLES_ATTR_MAX
2038	};
2039
2040	static struct attribute *cpumsf_pmu_events_attr[SF_CYCLES_ATTR_MAX + 1] = {
2041	[SF_CYCLES_BASIC_ATTR_IDX] = CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC)
2042	};
2043
2044	PMU_FORMAT_ATTR(event, "config:0-63");
2045
2046	static struct attribute *cpumsf_pmu_format_attr[] = {
2047	&format_attr_event.attr,
2048	NULL,
2049	};
2050
2051	static struct attribute_group cpumsf_pmu_events_group = {
2052	.name = "events",
2053	.attrs = cpumsf_pmu_events_attr,
2054	};
2055
2056	static struct attribute_group cpumsf_pmu_format_group = {
2057	.name = "format",
2058	.attrs = cpumsf_pmu_format_attr,
2059	};
2060
2061	static const struct attribute_group *cpumsf_pmu_attr_groups[] = {
2062	&cpumsf_pmu_events_group,
2063	&cpumsf_pmu_format_group,
2064	NULL,
2065	};
2066
2067	static struct pmu cpumf_sampling = {
2068	.pmu_enable = cpumsf_pmu_enable,
2069	.pmu_disable = cpumsf_pmu_disable,
2070
2071	.event_init = cpumsf_pmu_event_init,
2072	.add = cpumsf_pmu_add,
2073	.del = cpumsf_pmu_del,
2074
2075	.start = cpumsf_pmu_start,
2076	.stop = cpumsf_pmu_stop,
2077	.read = cpumsf_pmu_read,
2078
2079	.attr_groups = cpumsf_pmu_attr_groups,
2080
2081	.setup_aux = aux_buffer_setup,
2082	.free_aux = aux_buffer_free,
2083
2084	.check_period = cpumsf_pmu_check_period,
2085	};
2086
2087	static void cpumf_measurement_alert(struct ext_code ext_code,
2088	unsigned int alert, unsigned long unused)
2089	{
2090	struct cpu_hw_sf *cpuhw;
2091
2092	if (!(alert & CPU_MF_INT_SF_MASK))
2093	return;
2094	inc_irq_stat(IRQEXT_CMS);
2095	cpuhw = this_cpu_ptr(&cpu_hw_sf);
2096
2097	/* Measurement alerts are shared and might happen when the PMU
2098	* is not reserved. Ignore these alerts in this case. */
2099	if (!(cpuhw->flags & PMU_F_RESERVED))
2100	return;
2101
2102	/* The processing below must take care of multiple alert events that
2103	* might be indicated concurrently. */
2104
2105	/* Program alert request */
2106	if (alert & CPU_MF_INT_SF_PRA) {
2107	if (cpuhw->flags & PMU_F_IN_USE)
2108	if (SAMPL_DIAG_MODE(&cpuhw->event->hw))
2109	hw_collect_aux(cpuhw);
2110	else
2111	hw_perf_event_update(event: cpuhw->event, flush_all: 0);
2112	else
2113	WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE));
2114	}
2115
2116	/* Report measurement alerts only for non-PRA codes */
2117	if (alert != CPU_MF_INT_SF_PRA)
2118	debug_sprintf_event(sfdbg, 6, "%s: alert %#x\n", __func__,
2119	alert);
2120
2121	/* Sampling authorization change request */
2122	if (alert & CPU_MF_INT_SF_SACA)
2123	qsi(&cpuhw->qsi);
2124
2125	/* Loss of sample data due to high-priority machine activities */
2126	if (alert & CPU_MF_INT_SF_LSDA) {
2127	pr_err("Sample data was lost\n");
2128	cpuhw->flags |= PMU_F_ERR_LSDA;
2129	sf_disable();
2130	}
2131
2132	/* Invalid sampling buffer entry */
2133	if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) {
2134	pr_err("A sampling buffer entry is incorrect (alert=0x%x)\n",
2135	alert);
2136	cpuhw->flags |= PMU_F_ERR_IBE;
2137	sf_disable();
2138	}
2139	}
2140
2141	static int cpusf_pmu_setup(unsigned int cpu, int flags)
2142	{
2143	/* Ignore the notification if no events are scheduled on the PMU.
2144	* This might be racy...
2145	*/
2146	if (!atomic_read(v: &num_events))
2147	return 0;
2148
2149	local_irq_disable();
2150	setup_pmc_cpu(&flags);
2151	local_irq_enable();
2152	return 0;
2153	}
2154
2155	static int s390_pmu_sf_online_cpu(unsigned int cpu)
2156	{
2157	return cpusf_pmu_setup(cpu, PMC_INIT);
2158	}
2159
2160	static int s390_pmu_sf_offline_cpu(unsigned int cpu)
2161	{
2162	return cpusf_pmu_setup(cpu, PMC_RELEASE);
2163	}
2164
2165	static int param_get_sfb_size(char *buffer, const struct kernel_param *kp)
2166	{
2167	if (!cpum_sf_avail())
2168	return -ENODEV;
2169	return sprintf(buf: buffer, fmt: "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2170	}
2171
2172	static int param_set_sfb_size(const char *val, const struct kernel_param *kp)
2173	{
2174	int rc;
2175	unsigned long min, max;
2176
2177	if (!cpum_sf_avail())
2178	return -ENODEV;
2179	if (!val || !strlen(val))
2180	return -EINVAL;
2181
2182	/* Valid parameter values: "min,max" or "max" */
2183	min = CPUM_SF_MIN_SDB;
2184	max = CPUM_SF_MAX_SDB;
2185	if (strchr(val, ','))
2186	rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL;
2187	else
2188	rc = kstrtoul(s: val, base: 10, res: &max);
2189
2190	if (min < 2 || min >= max || max > get_num_physpages())
2191	rc = -EINVAL;
2192	if (rc)
2193	return rc;
2194
2195	sfb_set_limits(min, max);
2196	pr_info("The sampling buffer limits have changed to: "
2197	"min %lu max %lu (diag %lu)\n",
2198	CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB, CPUM_SF_SDB_DIAG_FACTOR);
2199	return 0;
2200	}
2201
2202	#define param_check_sfb_size(name, p) __param_check(name, p, void)
2203	static const struct kernel_param_ops param_ops_sfb_size = {
2204	.set = param_set_sfb_size,
2205	.get = param_get_sfb_size,
2206	};
2207
2208	#define RS_INIT_FAILURE_QSI 0x0001
2209	#define RS_INIT_FAILURE_BSDES 0x0002
2210	#define RS_INIT_FAILURE_ALRT 0x0003
2211	#define RS_INIT_FAILURE_PERF 0x0004
2212	static void __init pr_cpumsf_err(unsigned int reason)
2213	{
2214	pr_err("Sampling facility support for perf is not available: "
2215	"reason %#x\n", reason);
2216	}
2217
2218	static int __init init_cpum_sampling_pmu(void)
2219	{
2220	struct hws_qsi_info_block si;
2221	int err;
2222
2223	if (!cpum_sf_avail())
2224	return -ENODEV;
2225
2226	memset(&si, 0, sizeof(si));
2227	if (qsi(&si)) {
2228	pr_cpumsf_err(RS_INIT_FAILURE_QSI);
2229	return -ENODEV;
2230	}
2231
2232	if (!si.as && !si.ad)
2233	return -ENODEV;
2234
2235	if (si.bsdes != sizeof(struct hws_basic_entry)) {
2236	pr_cpumsf_err(RS_INIT_FAILURE_BSDES);
2237	return -EINVAL;
2238	}
2239
2240	if (si.ad) {
2241	sfb_set_limits(min: CPUM_SF_MIN_SDB, max: CPUM_SF_MAX_SDB);
2242	/* Sampling of diagnostic data authorized,
2243	* install event into attribute list of PMU device.
2244	*/
2245	cpumsf_pmu_events_attr[SF_CYCLES_BASIC_DIAG_ATTR_IDX] =
2246	CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC_DIAG);
2247	}
2248
2249	sfdbg = debug_register(KMSG_COMPONENT, 2, 1, 80);
2250	if (!sfdbg) {
2251	pr_err("Registering for s390dbf failed\n");
2252	return -ENOMEM;
2253	}
2254	debug_register_view(sfdbg, &debug_sprintf_view);
2255
2256	err = register_external_irq(EXT_IRQ_MEASURE_ALERT,
2257	cpumf_measurement_alert);
2258	if (err) {
2259	pr_cpumsf_err(RS_INIT_FAILURE_ALRT);
2260	debug_unregister(sfdbg);
2261	goto out;
2262	}
2263
2264	err = perf_pmu_register(pmu: &cpumf_sampling, name: "cpum_sf", type: PERF_TYPE_RAW);
2265	if (err) {
2266	pr_cpumsf_err(RS_INIT_FAILURE_PERF);
2267	unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
2268	cpumf_measurement_alert);
2269	debug_unregister(sfdbg);
2270	goto out;
2271	}
2272
2273	cpuhp_setup_state(state: CPUHP_AP_PERF_S390_SF_ONLINE, name: "perf/s390/sf:online",
2274	startup: s390_pmu_sf_online_cpu, teardown: s390_pmu_sf_offline_cpu);
2275	out:
2276	return err;
2277	}
2278
2279	arch_initcall(init_cpum_sampling_pmu);
2280	core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0644);
2281