1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Performance event support - Processor Activity Instrumentation Facility |
4 | * |
5 | * Copyright IBM Corp. 2022 |
6 | * Author(s): Thomas Richter <tmricht@linux.ibm.com> |
7 | */ |
8 | #define KMSG_COMPONENT "pai_crypto" |
9 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
10 | |
11 | #include <linux/kernel.h> |
12 | #include <linux/kernel_stat.h> |
13 | #include <linux/percpu.h> |
14 | #include <linux/notifier.h> |
15 | #include <linux/init.h> |
16 | #include <linux/export.h> |
17 | #include <linux/io.h> |
18 | #include <linux/perf_event.h> |
19 | #include <asm/ctlreg.h> |
20 | #include <asm/pai.h> |
21 | #include <asm/debug.h> |
22 | |
23 | static debug_info_t *cfm_dbg; |
24 | static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ |
25 | /* extracted with QPACI instruction */ |
26 | |
27 | DEFINE_STATIC_KEY_FALSE(pai_key); |
28 | |
29 | struct pai_userdata { |
30 | u16 num; |
31 | u64 value; |
32 | } __packed; |
33 | |
34 | struct paicrypt_map { |
35 | unsigned long *page; /* Page for CPU to store counters */ |
36 | struct pai_userdata *save; /* Page to store no-zero counters */ |
37 | unsigned int active_events; /* # of PAI crypto users */ |
38 | refcount_t refcnt; /* Reference count mapped buffers */ |
39 | enum paievt_mode mode; /* Type of event */ |
40 | struct perf_event *event; /* Perf event for sampling */ |
41 | }; |
42 | |
43 | struct paicrypt_mapptr { |
44 | struct paicrypt_map *mapptr; |
45 | }; |
46 | |
47 | static struct paicrypt_root { /* Anchor to per CPU data */ |
48 | refcount_t refcnt; /* Overall active events */ |
49 | struct paicrypt_mapptr __percpu *mapptr; |
50 | } paicrypt_root; |
51 | |
52 | /* Free per CPU data when the last event is removed. */ |
53 | static void paicrypt_root_free(void) |
54 | { |
55 | if (refcount_dec_and_test(r: &paicrypt_root.refcnt)) { |
56 | free_percpu(pdata: paicrypt_root.mapptr); |
57 | paicrypt_root.mapptr = NULL; |
58 | } |
59 | debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n" , __func__, |
60 | refcount_read(r: &paicrypt_root.refcnt)); |
61 | } |
62 | |
63 | /* |
64 | * On initialization of first event also allocate per CPU data dynamically. |
65 | * Start with an array of pointers, the array size is the maximum number of |
66 | * CPUs possible, which might be larger than the number of CPUs currently |
67 | * online. |
68 | */ |
69 | static int paicrypt_root_alloc(void) |
70 | { |
71 | if (!refcount_inc_not_zero(r: &paicrypt_root.refcnt)) { |
72 | /* The memory is already zeroed. */ |
73 | paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); |
74 | if (!paicrypt_root.mapptr) |
75 | return -ENOMEM; |
76 | refcount_set(r: &paicrypt_root.refcnt, n: 1); |
77 | } |
78 | return 0; |
79 | } |
80 | |
81 | /* Release the PMU if event is the last perf event */ |
82 | static DEFINE_MUTEX(pai_reserve_mutex); |
83 | |
84 | /* Adjust usage counters and remove allocated memory when all users are |
85 | * gone. |
86 | */ |
87 | static void paicrypt_event_destroy(struct perf_event *event) |
88 | { |
89 | struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, |
90 | event->cpu); |
91 | struct paicrypt_map *cpump = mp->mapptr; |
92 | |
93 | static_branch_dec(&pai_key); |
94 | mutex_lock(&pai_reserve_mutex); |
95 | debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" |
96 | " mode %d refcnt %u\n" , __func__, |
97 | event->attr.config, event->cpu, |
98 | cpump->active_events, cpump->mode, |
99 | refcount_read(r: &cpump->refcnt)); |
100 | free_page(PAI_SAVE_AREA(event)); |
101 | if (refcount_dec_and_test(r: &cpump->refcnt)) { |
102 | debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n" , |
103 | __func__, (unsigned long)cpump->page, |
104 | cpump->save); |
105 | free_page((unsigned long)cpump->page); |
106 | kvfree(addr: cpump->save); |
107 | kfree(objp: cpump); |
108 | mp->mapptr = NULL; |
109 | } |
110 | paicrypt_root_free(); |
111 | mutex_unlock(lock: &pai_reserve_mutex); |
112 | } |
113 | |
114 | static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel) |
115 | { |
116 | if (kernel) |
117 | nr += PAI_CRYPTO_MAXCTR; |
118 | return page[nr]; |
119 | } |
120 | |
121 | /* Read the counter values. Return value from location in CMP. For event |
122 | * CRYPTO_ALL sum up all events. |
123 | */ |
124 | static u64 paicrypt_getdata(struct perf_event *event, bool kernel) |
125 | { |
126 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
127 | struct paicrypt_map *cpump = mp->mapptr; |
128 | u64 sum = 0; |
129 | int i; |
130 | |
131 | if (event->attr.config != PAI_CRYPTO_BASE) { |
132 | return paicrypt_getctr(page: cpump->page, |
133 | nr: event->attr.config - PAI_CRYPTO_BASE, |
134 | kernel); |
135 | } |
136 | |
137 | for (i = 1; i <= paicrypt_cnt; i++) { |
138 | u64 val = paicrypt_getctr(page: cpump->page, nr: i, kernel); |
139 | |
140 | if (!val) |
141 | continue; |
142 | sum += val; |
143 | } |
144 | return sum; |
145 | } |
146 | |
147 | static u64 paicrypt_getall(struct perf_event *event) |
148 | { |
149 | u64 sum = 0; |
150 | |
151 | if (!event->attr.exclude_kernel) |
152 | sum += paicrypt_getdata(event, kernel: true); |
153 | if (!event->attr.exclude_user) |
154 | sum += paicrypt_getdata(event, kernel: false); |
155 | |
156 | return sum; |
157 | } |
158 | |
159 | /* Used to avoid races in checking concurrent access of counting and |
160 | * sampling for crypto events |
161 | * |
162 | * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is |
163 | * allowed and when this event is running, no counting event is allowed. |
164 | * Several counting events are allowed in parallel, but no sampling event |
165 | * is allowed while one (or more) counting events are running. |
166 | * |
167 | * This function is called in process context and it is save to block. |
168 | * When the event initialization functions fails, no other call back will |
169 | * be invoked. |
170 | * |
171 | * Allocate the memory for the event. |
172 | */ |
173 | static struct paicrypt_map *paicrypt_busy(struct perf_event *event) |
174 | { |
175 | struct perf_event_attr *a = &event->attr; |
176 | struct paicrypt_map *cpump = NULL; |
177 | struct paicrypt_mapptr *mp; |
178 | int rc; |
179 | |
180 | mutex_lock(&pai_reserve_mutex); |
181 | |
182 | /* Allocate root node */ |
183 | rc = paicrypt_root_alloc(); |
184 | if (rc) |
185 | goto unlock; |
186 | |
187 | /* Allocate node for this event */ |
188 | mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); |
189 | cpump = mp->mapptr; |
190 | if (!cpump) { /* Paicrypt_map allocated? */ |
191 | cpump = kzalloc(size: sizeof(*cpump), GFP_KERNEL); |
192 | if (!cpump) { |
193 | rc = -ENOMEM; |
194 | goto free_root; |
195 | } |
196 | } |
197 | |
198 | if (a->sample_period) { /* Sampling requested */ |
199 | if (cpump->mode != PAI_MODE_NONE) |
200 | rc = -EBUSY; /* ... sampling/counting active */ |
201 | } else { /* Counting requested */ |
202 | if (cpump->mode == PAI_MODE_SAMPLING) |
203 | rc = -EBUSY; /* ... and sampling active */ |
204 | } |
205 | /* |
206 | * This error case triggers when there is a conflict: |
207 | * Either sampling requested and counting already active, or visa |
208 | * versa. Therefore the struct paicrypto_map for this CPU is |
209 | * needed or the error could not have occurred. Only adjust root |
210 | * node refcount. |
211 | */ |
212 | if (rc) |
213 | goto free_root; |
214 | |
215 | /* Allocate memory for counter page and counter extraction. |
216 | * Only the first counting event has to allocate a page. |
217 | */ |
218 | if (cpump->page) { |
219 | refcount_inc(r: &cpump->refcnt); |
220 | goto unlock; |
221 | } |
222 | |
223 | rc = -ENOMEM; |
224 | cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); |
225 | if (!cpump->page) |
226 | goto free_paicrypt_map; |
227 | cpump->save = kvmalloc_array(n: paicrypt_cnt + 1, |
228 | size: sizeof(struct pai_userdata), GFP_KERNEL); |
229 | if (!cpump->save) { |
230 | free_page((unsigned long)cpump->page); |
231 | cpump->page = NULL; |
232 | goto free_paicrypt_map; |
233 | } |
234 | |
235 | /* Set mode and reference count */ |
236 | rc = 0; |
237 | refcount_set(r: &cpump->refcnt, n: 1); |
238 | cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; |
239 | mp->mapptr = cpump; |
240 | debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" |
241 | " mode %d refcnt %u page %#lx save %p rc %d\n" , |
242 | __func__, a->sample_period, cpump->active_events, |
243 | cpump->mode, refcount_read(r: &cpump->refcnt), |
244 | (unsigned long)cpump->page, cpump->save, rc); |
245 | goto unlock; |
246 | |
247 | free_paicrypt_map: |
248 | kfree(objp: cpump); |
249 | mp->mapptr = NULL; |
250 | free_root: |
251 | paicrypt_root_free(); |
252 | |
253 | unlock: |
254 | mutex_unlock(lock: &pai_reserve_mutex); |
255 | return rc ? ERR_PTR(error: rc) : cpump; |
256 | } |
257 | |
258 | /* Might be called on different CPU than the one the event is intended for. */ |
259 | static int paicrypt_event_init(struct perf_event *event) |
260 | { |
261 | struct perf_event_attr *a = &event->attr; |
262 | struct paicrypt_map *cpump; |
263 | int rc = 0; |
264 | |
265 | /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ |
266 | if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) |
267 | return -ENOENT; |
268 | /* PAI crypto event must be in valid range */ |
269 | if (a->config < PAI_CRYPTO_BASE || |
270 | a->config > PAI_CRYPTO_BASE + paicrypt_cnt) |
271 | return -EINVAL; |
272 | /* Allow only CPU wide operation, no process context for now. */ |
273 | if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) |
274 | return -ENOENT; |
275 | /* Allow only CRYPTO_ALL for sampling. */ |
276 | if (a->sample_period && a->config != PAI_CRYPTO_BASE) |
277 | return -EINVAL; |
278 | /* Get a page to store last counter values for sampling */ |
279 | if (a->sample_period) { |
280 | PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); |
281 | if (!PAI_SAVE_AREA(event)) { |
282 | rc = -ENOMEM; |
283 | goto out; |
284 | } |
285 | } |
286 | |
287 | cpump = paicrypt_busy(event); |
288 | if (IS_ERR(ptr: cpump)) { |
289 | free_page(PAI_SAVE_AREA(event)); |
290 | rc = PTR_ERR(ptr: cpump); |
291 | goto out; |
292 | } |
293 | |
294 | event->destroy = paicrypt_event_destroy; |
295 | |
296 | if (a->sample_period) { |
297 | a->sample_period = 1; |
298 | a->freq = 0; |
299 | /* Register for paicrypt_sched_task() to be called */ |
300 | event->attach_state |= PERF_ATTACH_SCHED_CB; |
301 | /* Add raw data which contain the memory mapped counters */ |
302 | a->sample_type |= PERF_SAMPLE_RAW; |
303 | /* Turn off inheritance */ |
304 | a->inherit = 0; |
305 | } |
306 | |
307 | static_branch_inc(&pai_key); |
308 | out: |
309 | return rc; |
310 | } |
311 | |
312 | static void paicrypt_read(struct perf_event *event) |
313 | { |
314 | u64 prev, new, delta; |
315 | |
316 | prev = local64_read(&event->hw.prev_count); |
317 | new = paicrypt_getall(event); |
318 | local64_set(&event->hw.prev_count, new); |
319 | delta = (prev <= new) ? new - prev |
320 | : (-1ULL - prev) + new + 1; /* overflow */ |
321 | local64_add(delta, &event->count); |
322 | } |
323 | |
324 | static void paicrypt_start(struct perf_event *event, int flags) |
325 | { |
326 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
327 | struct paicrypt_map *cpump = mp->mapptr; |
328 | u64 sum; |
329 | |
330 | if (!event->attr.sample_period) { /* Counting */ |
331 | sum = paicrypt_getall(event); /* Get current value */ |
332 | local64_set(&event->hw.prev_count, sum); |
333 | } else { /* Sampling */ |
334 | cpump->event = event; |
335 | perf_sched_cb_inc(pmu: event->pmu); |
336 | } |
337 | } |
338 | |
339 | static int paicrypt_add(struct perf_event *event, int flags) |
340 | { |
341 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
342 | struct paicrypt_map *cpump = mp->mapptr; |
343 | unsigned long ccd; |
344 | |
345 | if (++cpump->active_events == 1) { |
346 | ccd = virt_to_phys(address: cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; |
347 | WRITE_ONCE(S390_lowcore.ccd, ccd); |
348 | local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); |
349 | } |
350 | if (flags & PERF_EF_START) |
351 | paicrypt_start(event, PERF_EF_RELOAD); |
352 | event->hw.state = 0; |
353 | return 0; |
354 | } |
355 | |
356 | static void paicrypt_stop(struct perf_event *event, int flags) |
357 | { |
358 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
359 | struct paicrypt_map *cpump = mp->mapptr; |
360 | |
361 | if (!event->attr.sample_period) { /* Counting */ |
362 | paicrypt_read(event); |
363 | } else { /* Sampling */ |
364 | perf_sched_cb_dec(pmu: event->pmu); |
365 | cpump->event = NULL; |
366 | } |
367 | event->hw.state = PERF_HES_STOPPED; |
368 | } |
369 | |
370 | static void paicrypt_del(struct perf_event *event, int flags) |
371 | { |
372 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
373 | struct paicrypt_map *cpump = mp->mapptr; |
374 | |
375 | paicrypt_stop(event, PERF_EF_UPDATE); |
376 | if (--cpump->active_events == 0) { |
377 | local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); |
378 | WRITE_ONCE(S390_lowcore.ccd, 0); |
379 | } |
380 | } |
381 | |
382 | /* Create raw data and save it in buffer. Calculate the delta for each |
383 | * counter between this invocation and the last invocation. |
384 | * Returns number of bytes copied. |
385 | * Saves only entries with positive counter difference of the form |
386 | * 2 bytes: Number of counter |
387 | * 8 bytes: Value of counter |
388 | */ |
389 | static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page, |
390 | unsigned long *page_old, bool exclude_user, |
391 | bool exclude_kernel) |
392 | { |
393 | int i, outidx = 0; |
394 | |
395 | for (i = 1; i <= paicrypt_cnt; i++) { |
396 | u64 val = 0, val_old = 0; |
397 | |
398 | if (!exclude_kernel) { |
399 | val += paicrypt_getctr(page, nr: i, kernel: true); |
400 | val_old += paicrypt_getctr(page: page_old, nr: i, kernel: true); |
401 | } |
402 | if (!exclude_user) { |
403 | val += paicrypt_getctr(page, nr: i, kernel: false); |
404 | val_old += paicrypt_getctr(page: page_old, nr: i, kernel: false); |
405 | } |
406 | if (val >= val_old) |
407 | val -= val_old; |
408 | else |
409 | val = (~0ULL - val_old) + val + 1; |
410 | if (val) { |
411 | userdata[outidx].num = i; |
412 | userdata[outidx].value = val; |
413 | outidx++; |
414 | } |
415 | } |
416 | return outidx * sizeof(struct pai_userdata); |
417 | } |
418 | |
419 | static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, |
420 | struct perf_event *event) |
421 | { |
422 | struct perf_sample_data data; |
423 | struct perf_raw_record raw; |
424 | struct pt_regs regs; |
425 | int overflow; |
426 | |
427 | /* Setup perf sample */ |
428 | memset(®s, 0, sizeof(regs)); |
429 | memset(&raw, 0, sizeof(raw)); |
430 | memset(&data, 0, sizeof(data)); |
431 | perf_sample_data_init(data: &data, addr: 0, period: event->hw.last_period); |
432 | if (event->attr.sample_type & PERF_SAMPLE_TID) { |
433 | data.tid_entry.pid = task_tgid_nr(current); |
434 | data.tid_entry.tid = task_pid_nr(current); |
435 | } |
436 | if (event->attr.sample_type & PERF_SAMPLE_TIME) |
437 | data.time = event->clock(); |
438 | if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) |
439 | data.id = event->id; |
440 | if (event->attr.sample_type & PERF_SAMPLE_CPU) { |
441 | data.cpu_entry.cpu = smp_processor_id(); |
442 | data.cpu_entry.reserved = 0; |
443 | } |
444 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { |
445 | raw.frag.size = rawsize; |
446 | raw.frag.data = cpump->save; |
447 | perf_sample_save_raw_data(data: &data, raw: &raw); |
448 | } |
449 | |
450 | overflow = perf_event_overflow(event, data: &data, regs: ®s); |
451 | perf_event_update_userpage(event); |
452 | /* Save crypto counter lowcore page after reading event data. */ |
453 | memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); |
454 | return overflow; |
455 | } |
456 | |
457 | /* Check if there is data to be saved on schedule out of a task. */ |
458 | static int paicrypt_have_sample(void) |
459 | { |
460 | struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); |
461 | struct paicrypt_map *cpump = mp->mapptr; |
462 | struct perf_event *event = cpump->event; |
463 | size_t rawsize; |
464 | int rc = 0; |
465 | |
466 | if (!event) /* No event active */ |
467 | return 0; |
468 | rawsize = paicrypt_copy(userdata: cpump->save, page: cpump->page, |
469 | page_old: (unsigned long *)PAI_SAVE_AREA(event), |
470 | exclude_user: cpump->event->attr.exclude_user, |
471 | exclude_kernel: cpump->event->attr.exclude_kernel); |
472 | if (rawsize) /* No incremented counters */ |
473 | rc = paicrypt_push_sample(rawsize, cpump, event); |
474 | return rc; |
475 | } |
476 | |
477 | /* Called on schedule-in and schedule-out. No access to event structure, |
478 | * but for sampling only event CRYPTO_ALL is allowed. |
479 | */ |
480 | static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) |
481 | { |
482 | /* We started with a clean page on event installation. So read out |
483 | * results on schedule_out and if page was dirty, clear values. |
484 | */ |
485 | if (!sched_in) |
486 | paicrypt_have_sample(); |
487 | } |
488 | |
489 | /* Attribute definitions for paicrypt interface. As with other CPU |
490 | * Measurement Facilities, there is one attribute per mapped counter. |
491 | * The number of mapped counters may vary per machine generation. Use |
492 | * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction |
493 | * to determine the number of mapped counters. The instructions returns |
494 | * a positive number, which is the highest number of supported counters. |
495 | * All counters less than this number are also supported, there are no |
496 | * holes. A returned number of zero means no support for mapped counters. |
497 | * |
498 | * The identification of the counter is a unique number. The chosen range |
499 | * is 0x1000 + offset in mapped kernel page. |
500 | * All CPU Measurement Facility counters identifiers must be unique and |
501 | * the numbers from 0 to 496 are already used for the CPU Measurement |
502 | * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already |
503 | * used for the CPU Measurement Sampling facility. |
504 | */ |
505 | PMU_FORMAT_ATTR(event, "config:0-63" ); |
506 | |
507 | static struct attribute *paicrypt_format_attr[] = { |
508 | &format_attr_event.attr, |
509 | NULL, |
510 | }; |
511 | |
512 | static struct attribute_group paicrypt_events_group = { |
513 | .name = "events" , |
514 | .attrs = NULL /* Filled in attr_event_init() */ |
515 | }; |
516 | |
517 | static struct attribute_group paicrypt_format_group = { |
518 | .name = "format" , |
519 | .attrs = paicrypt_format_attr, |
520 | }; |
521 | |
522 | static const struct attribute_group *paicrypt_attr_groups[] = { |
523 | &paicrypt_events_group, |
524 | &paicrypt_format_group, |
525 | NULL, |
526 | }; |
527 | |
528 | /* Performance monitoring unit for mapped counters */ |
529 | static struct pmu paicrypt = { |
530 | .task_ctx_nr = perf_invalid_context, |
531 | .event_init = paicrypt_event_init, |
532 | .add = paicrypt_add, |
533 | .del = paicrypt_del, |
534 | .start = paicrypt_start, |
535 | .stop = paicrypt_stop, |
536 | .read = paicrypt_read, |
537 | .sched_task = paicrypt_sched_task, |
538 | .attr_groups = paicrypt_attr_groups |
539 | }; |
540 | |
541 | /* List of symbolic PAI counter names. */ |
542 | static const char * const paicrypt_ctrnames[] = { |
543 | [0] = "CRYPTO_ALL" , |
544 | [1] = "KM_DEA" , |
545 | [2] = "KM_TDEA_128" , |
546 | [3] = "KM_TDEA_192" , |
547 | [4] = "KM_ENCRYPTED_DEA" , |
548 | [5] = "KM_ENCRYPTED_TDEA_128" , |
549 | [6] = "KM_ENCRYPTED_TDEA_192" , |
550 | [7] = "KM_AES_128" , |
551 | [8] = "KM_AES_192" , |
552 | [9] = "KM_AES_256" , |
553 | [10] = "KM_ENCRYPTED_AES_128" , |
554 | [11] = "KM_ENCRYPTED_AES_192" , |
555 | [12] = "KM_ENCRYPTED_AES_256" , |
556 | [13] = "KM_XTS_AES_128" , |
557 | [14] = "KM_XTS_AES_256" , |
558 | [15] = "KM_XTS_ENCRYPTED_AES_128" , |
559 | [16] = "KM_XTS_ENCRYPTED_AES_256" , |
560 | [17] = "KMC_DEA" , |
561 | [18] = "KMC_TDEA_128" , |
562 | [19] = "KMC_TDEA_192" , |
563 | [20] = "KMC_ENCRYPTED_DEA" , |
564 | [21] = "KMC_ENCRYPTED_TDEA_128" , |
565 | [22] = "KMC_ENCRYPTED_TDEA_192" , |
566 | [23] = "KMC_AES_128" , |
567 | [24] = "KMC_AES_192" , |
568 | [25] = "KMC_AES_256" , |
569 | [26] = "KMC_ENCRYPTED_AES_128" , |
570 | [27] = "KMC_ENCRYPTED_AES_192" , |
571 | [28] = "KMC_ENCRYPTED_AES_256" , |
572 | [29] = "KMC_PRNG" , |
573 | [30] = "KMA_GCM_AES_128" , |
574 | [31] = "KMA_GCM_AES_192" , |
575 | [32] = "KMA_GCM_AES_256" , |
576 | [33] = "KMA_GCM_ENCRYPTED_AES_128" , |
577 | [34] = "KMA_GCM_ENCRYPTED_AES_192" , |
578 | [35] = "KMA_GCM_ENCRYPTED_AES_256" , |
579 | [36] = "KMF_DEA" , |
580 | [37] = "KMF_TDEA_128" , |
581 | [38] = "KMF_TDEA_192" , |
582 | [39] = "KMF_ENCRYPTED_DEA" , |
583 | [40] = "KMF_ENCRYPTED_TDEA_128" , |
584 | [41] = "KMF_ENCRYPTED_TDEA_192" , |
585 | [42] = "KMF_AES_128" , |
586 | [43] = "KMF_AES_192" , |
587 | [44] = "KMF_AES_256" , |
588 | [45] = "KMF_ENCRYPTED_AES_128" , |
589 | [46] = "KMF_ENCRYPTED_AES_192" , |
590 | [47] = "KMF_ENCRYPTED_AES_256" , |
591 | [48] = "KMCTR_DEA" , |
592 | [49] = "KMCTR_TDEA_128" , |
593 | [50] = "KMCTR_TDEA_192" , |
594 | [51] = "KMCTR_ENCRYPTED_DEA" , |
595 | [52] = "KMCTR_ENCRYPTED_TDEA_128" , |
596 | [53] = "KMCTR_ENCRYPTED_TDEA_192" , |
597 | [54] = "KMCTR_AES_128" , |
598 | [55] = "KMCTR_AES_192" , |
599 | [56] = "KMCTR_AES_256" , |
600 | [57] = "KMCTR_ENCRYPTED_AES_128" , |
601 | [58] = "KMCTR_ENCRYPTED_AES_192" , |
602 | [59] = "KMCTR_ENCRYPTED_AES_256" , |
603 | [60] = "KMO_DEA" , |
604 | [61] = "KMO_TDEA_128" , |
605 | [62] = "KMO_TDEA_192" , |
606 | [63] = "KMO_ENCRYPTED_DEA" , |
607 | [64] = "KMO_ENCRYPTED_TDEA_128" , |
608 | [65] = "KMO_ENCRYPTED_TDEA_192" , |
609 | [66] = "KMO_AES_128" , |
610 | [67] = "KMO_AES_192" , |
611 | [68] = "KMO_AES_256" , |
612 | [69] = "KMO_ENCRYPTED_AES_128" , |
613 | [70] = "KMO_ENCRYPTED_AES_192" , |
614 | [71] = "KMO_ENCRYPTED_AES_256" , |
615 | [72] = "KIMD_SHA_1" , |
616 | [73] = "KIMD_SHA_256" , |
617 | [74] = "KIMD_SHA_512" , |
618 | [75] = "KIMD_SHA3_224" , |
619 | [76] = "KIMD_SHA3_256" , |
620 | [77] = "KIMD_SHA3_384" , |
621 | [78] = "KIMD_SHA3_512" , |
622 | [79] = "KIMD_SHAKE_128" , |
623 | [80] = "KIMD_SHAKE_256" , |
624 | [81] = "KIMD_GHASH" , |
625 | [82] = "KLMD_SHA_1" , |
626 | [83] = "KLMD_SHA_256" , |
627 | [84] = "KLMD_SHA_512" , |
628 | [85] = "KLMD_SHA3_224" , |
629 | [86] = "KLMD_SHA3_256" , |
630 | [87] = "KLMD_SHA3_384" , |
631 | [88] = "KLMD_SHA3_512" , |
632 | [89] = "KLMD_SHAKE_128" , |
633 | [90] = "KLMD_SHAKE_256" , |
634 | [91] = "KMAC_DEA" , |
635 | [92] = "KMAC_TDEA_128" , |
636 | [93] = "KMAC_TDEA_192" , |
637 | [94] = "KMAC_ENCRYPTED_DEA" , |
638 | [95] = "KMAC_ENCRYPTED_TDEA_128" , |
639 | [96] = "KMAC_ENCRYPTED_TDEA_192" , |
640 | [97] = "KMAC_AES_128" , |
641 | [98] = "KMAC_AES_192" , |
642 | [99] = "KMAC_AES_256" , |
643 | [100] = "KMAC_ENCRYPTED_AES_128" , |
644 | [101] = "KMAC_ENCRYPTED_AES_192" , |
645 | [102] = "KMAC_ENCRYPTED_AES_256" , |
646 | [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA" , |
647 | [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128" , |
648 | [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192" , |
649 | [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA" , |
650 | [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128" , |
651 | [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192" , |
652 | [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128" , |
653 | [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192" , |
654 | [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256" , |
655 | [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128" , |
656 | [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192" , |
657 | [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A" , |
658 | [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128" , |
659 | [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256" , |
660 | [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128" , |
661 | [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256" , |
662 | [119] = "PCC_SCALAR_MULTIPLY_P256" , |
663 | [120] = "PCC_SCALAR_MULTIPLY_P384" , |
664 | [121] = "PCC_SCALAR_MULTIPLY_P521" , |
665 | [122] = "PCC_SCALAR_MULTIPLY_ED25519" , |
666 | [123] = "PCC_SCALAR_MULTIPLY_ED448" , |
667 | [124] = "PCC_SCALAR_MULTIPLY_X25519" , |
668 | [125] = "PCC_SCALAR_MULTIPLY_X448" , |
669 | [126] = "PRNO_SHA_512_DRNG" , |
670 | [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO" , |
671 | [128] = "PRNO_TRNG" , |
672 | [129] = "KDSA_ECDSA_VERIFY_P256" , |
673 | [130] = "KDSA_ECDSA_VERIFY_P384" , |
674 | [131] = "KDSA_ECDSA_VERIFY_P521" , |
675 | [132] = "KDSA_ECDSA_SIGN_P256" , |
676 | [133] = "KDSA_ECDSA_SIGN_P384" , |
677 | [134] = "KDSA_ECDSA_SIGN_P521" , |
678 | [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256" , |
679 | [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384" , |
680 | [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521" , |
681 | [138] = "KDSA_EDDSA_VERIFY_ED25519" , |
682 | [139] = "KDSA_EDDSA_VERIFY_ED448" , |
683 | [140] = "KDSA_EDDSA_SIGN_ED25519" , |
684 | [141] = "KDSA_EDDSA_SIGN_ED448" , |
685 | [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519" , |
686 | [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448" , |
687 | [144] = "PCKMO_ENCRYPT_DEA_KEY" , |
688 | [145] = "PCKMO_ENCRYPT_TDEA_128_KEY" , |
689 | [146] = "PCKMO_ENCRYPT_TDEA_192_KEY" , |
690 | [147] = "PCKMO_ENCRYPT_AES_128_KEY" , |
691 | [148] = "PCKMO_ENCRYPT_AES_192_KEY" , |
692 | [149] = "PCKMO_ENCRYPT_AES_256_KEY" , |
693 | [150] = "PCKMO_ENCRYPT_ECC_P256_KEY" , |
694 | [151] = "PCKMO_ENCRYPT_ECC_P384_KEY" , |
695 | [152] = "PCKMO_ENCRYPT_ECC_P521_KEY" , |
696 | [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY" , |
697 | [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY" , |
698 | [155] = "IBM_RESERVED_155" , |
699 | [156] = "IBM_RESERVED_156" , |
700 | }; |
701 | |
702 | static void __init attr_event_free(struct attribute **attrs, int num) |
703 | { |
704 | struct perf_pmu_events_attr *pa; |
705 | int i; |
706 | |
707 | for (i = 0; i < num; i++) { |
708 | struct device_attribute *dap; |
709 | |
710 | dap = container_of(attrs[i], struct device_attribute, attr); |
711 | pa = container_of(dap, struct perf_pmu_events_attr, attr); |
712 | kfree(objp: pa); |
713 | } |
714 | kfree(objp: attrs); |
715 | } |
716 | |
717 | static int __init attr_event_init_one(struct attribute **attrs, int num) |
718 | { |
719 | struct perf_pmu_events_attr *pa; |
720 | |
721 | /* Index larger than array_size, no counter name available */ |
722 | if (num >= ARRAY_SIZE(paicrypt_ctrnames)) { |
723 | attrs[num] = NULL; |
724 | return 0; |
725 | } |
726 | |
727 | pa = kzalloc(size: sizeof(*pa), GFP_KERNEL); |
728 | if (!pa) |
729 | return -ENOMEM; |
730 | |
731 | sysfs_attr_init(&pa->attr.attr); |
732 | pa->id = PAI_CRYPTO_BASE + num; |
733 | pa->attr.attr.name = paicrypt_ctrnames[num]; |
734 | pa->attr.attr.mode = 0444; |
735 | pa->attr.show = cpumf_events_sysfs_show; |
736 | pa->attr.store = NULL; |
737 | attrs[num] = &pa->attr.attr; |
738 | return 0; |
739 | } |
740 | |
741 | /* Create PMU sysfs event attributes on the fly. */ |
742 | static int __init attr_event_init(void) |
743 | { |
744 | struct attribute **attrs; |
745 | int ret, i; |
746 | |
747 | attrs = kmalloc_array(n: paicrypt_cnt + 2, size: sizeof(*attrs), GFP_KERNEL); |
748 | if (!attrs) |
749 | return -ENOMEM; |
750 | for (i = 0; i <= paicrypt_cnt; i++) { |
751 | ret = attr_event_init_one(attrs, num: i); |
752 | if (ret) { |
753 | attr_event_free(attrs, num: i); |
754 | return ret; |
755 | } |
756 | } |
757 | attrs[i] = NULL; |
758 | paicrypt_events_group.attrs = attrs; |
759 | return 0; |
760 | } |
761 | |
762 | static int __init paicrypt_init(void) |
763 | { |
764 | struct qpaci_info_block ib; |
765 | int rc; |
766 | |
767 | if (!test_facility(196)) |
768 | return 0; |
769 | |
770 | qpaci(&ib); |
771 | paicrypt_cnt = ib.num_cc; |
772 | if (paicrypt_cnt == 0) |
773 | return 0; |
774 | if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) { |
775 | pr_err("Too many PMU pai_crypto counters %d\n" , paicrypt_cnt); |
776 | return -E2BIG; |
777 | } |
778 | |
779 | rc = attr_event_init(); /* Export known PAI crypto events */ |
780 | if (rc) { |
781 | pr_err("Creation of PMU pai_crypto /sysfs failed\n" ); |
782 | return rc; |
783 | } |
784 | |
785 | /* Setup s390dbf facility */ |
786 | cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); |
787 | if (!cfm_dbg) { |
788 | pr_err("Registration of s390dbf pai_crypto failed\n" ); |
789 | return -ENOMEM; |
790 | } |
791 | debug_register_view(cfm_dbg, &debug_sprintf_view); |
792 | |
793 | rc = perf_pmu_register(pmu: &paicrypt, name: "pai_crypto" , type: -1); |
794 | if (rc) { |
795 | pr_err("Registering the pai_crypto PMU failed with rc=%i\n" , |
796 | rc); |
797 | debug_unregister_view(cfm_dbg, &debug_sprintf_view); |
798 | debug_unregister(cfm_dbg); |
799 | return rc; |
800 | } |
801 | return 0; |
802 | } |
803 | |
804 | device_initcall(paicrypt_init); |
805 | |