1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* pSeries_lpar.c
4	* Copyright (C) 2001 Todd Inglett, IBM Corporation
5	*
6	* pSeries LPAR support.
7	*/
8
9	/* Enables debugging of low-level hash table routines - careful! */
10	#undef DEBUG
11	#define pr_fmt(fmt) "lpar: " fmt
12
13	#include <linux/kernel.h>
14	#include <linux/dma-mapping.h>
15	#include <linux/console.h>
16	#include <linux/export.h>
17	#include <linux/jump_label.h>
18	#include <linux/delay.h>
19	#include <linux/stop_machine.h>
20	#include <linux/spinlock.h>
21	#include <linux/cpuhotplug.h>
22	#include <linux/workqueue.h>
23	#include <linux/proc_fs.h>
24	#include <linux/pgtable.h>
25	#include <linux/debugfs.h>
26
27	#include <asm/processor.h>
28	#include <asm/mmu.h>
29	#include <asm/page.h>
30	#include <asm/setup.h>
31	#include <asm/mmu_context.h>
32	#include <asm/iommu.h>
33	#include <asm/tlb.h>
34	#include <asm/cputable.h>
35	#include <asm/papr-sysparm.h>
36	#include <asm/udbg.h>
37	#include <asm/smp.h>
38	#include <asm/trace.h>
39	#include <asm/firmware.h>
40	#include <asm/plpar_wrappers.h>
41	#include <asm/kexec.h>
42	#include <asm/fadump.h>
43	#include <asm/dtl.h>
44	#include <asm/vphn.h>
45
46	#include "pseries.h"
47
48	/* Flag bits for H_BULK_REMOVE */
49	#define HBR_REQUEST 0x4000000000000000UL
50	#define HBR_RESPONSE 0x8000000000000000UL
51	#define HBR_END 0xc000000000000000UL
52	#define HBR_AVPN 0x0200000000000000UL
53	#define HBR_ANDCOND 0x0100000000000000UL
54
55
56	/* in hvCall.S */
57	EXPORT_SYMBOL(plpar_hcall);
58	EXPORT_SYMBOL(plpar_hcall9);
59	EXPORT_SYMBOL(plpar_hcall_norets);
60
61	#ifdef CONFIG_PPC_64S_HASH_MMU
62	/*
63	* H_BLOCK_REMOVE supported block size for this page size in segment who's base
64	* page size is that page size.
65	*
66	* The first index is the segment base page size, the second one is the actual
67	* page size.
68	*/
69	static int hblkrm_size[MMU_PAGE_COUNT][MMU_PAGE_COUNT] __ro_after_init;
70	#endif
71
72	/*
73	* Due to the involved complexity, and that the current hypervisor is only
74	* returning this value or 0, we are limiting the support of the H_BLOCK_REMOVE
75	* buffer size to 8 size block.
76	*/
77	#define HBLKRM_SUPPORTED_BLOCK_SIZE 8
78
79	#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
80	static u8 dtl_mask = DTL_LOG_PREEMPT;
81	#else
82	static u8 dtl_mask;
83	#endif
84
85	void alloc_dtl_buffers(unsigned long *time_limit)
86	{
87	int cpu;
88	struct paca_struct *pp;
89	struct dtl_entry *dtl;
90
91	for_each_possible_cpu(cpu) {
92	pp = paca_ptrs[cpu];
93	if (pp->dispatch_log)
94	continue;
95	dtl = kmem_cache_alloc(cachep: dtl_cache, GFP_KERNEL);
96	if (!dtl) {
97	pr_warn("Failed to allocate dispatch trace log for cpu %d\n",
98	cpu);
99	#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
100	pr_warn("Stolen time statistics will be unreliable\n");
101	#endif
102	break;
103	}
104
105	pp->dtl_ridx = 0;
106	pp->dispatch_log = dtl;
107	pp->dispatch_log_end = dtl + N_DISPATCH_LOG;
108	pp->dtl_curr = dtl;
109
110	if (time_limit && time_after(jiffies, *time_limit)) {
111	cond_resched();
112	*time_limit = jiffies + HZ;
113	}
114	}
115	}
116
117	void register_dtl_buffer(int cpu)
118	{
119	long ret;
120	struct paca_struct *pp;
121	struct dtl_entry *dtl;
122	int hwcpu = get_hard_smp_processor_id(cpu);
123
124	pp = paca_ptrs[cpu];
125	dtl = pp->dispatch_log;
126	if (dtl && dtl_mask) {
127	pp->dtl_ridx = 0;
128	pp->dtl_curr = dtl;
129	lppaca_of(cpu).dtl_idx = 0;
130
131	/* hypervisor reads buffer length from this field */
132	dtl->enqueue_to_dispatch_time = cpu_to_be32(DISPATCH_LOG_BYTES);
133	ret = register_dtl(hwcpu, __pa(dtl));
134	if (ret)
135	pr_err("WARNING: DTL registration of cpu %d (hw %d) failed with %ld\n",
136	cpu, hwcpu, ret);
137
138	lppaca_of(cpu).dtl_enable_mask = dtl_mask;
139	}
140	}
141
142	#ifdef CONFIG_PPC_SPLPAR
143	struct dtl_worker {
144	struct delayed_work work;
145	int cpu;
146	};
147
148	struct vcpu_dispatch_data {
149	int last_disp_cpu;
150
151	int total_disp;
152
153	int same_cpu_disp;
154	int same_chip_disp;
155	int diff_chip_disp;
156	int far_chip_disp;
157
158	int numa_home_disp;
159	int numa_remote_disp;
160	int numa_far_disp;
161	};
162
163	/*
164	* This represents the number of cpus in the hypervisor. Since there is no
165	* architected way to discover the number of processors in the host, we
166	* provision for dealing with NR_CPUS. This is currently 2048 by default, and
167	* is sufficient for our purposes. This will need to be tweaked if
168	* CONFIG_NR_CPUS is changed.
169	*/
170	#define NR_CPUS_H NR_CPUS
171
172	DEFINE_RWLOCK(dtl_access_lock);
173	static DEFINE_PER_CPU(struct vcpu_dispatch_data, vcpu_disp_data);
174	static DEFINE_PER_CPU(u64, dtl_entry_ridx);
175	static DEFINE_PER_CPU(struct dtl_worker, dtl_workers);
176	static enum cpuhp_state dtl_worker_state;
177	static DEFINE_MUTEX(dtl_enable_mutex);
178	static int vcpudispatch_stats_on __read_mostly;
179	static int vcpudispatch_stats_freq = 50;
180	static __be32 *vcpu_associativity, *pcpu_associativity;
181
182
183	static void free_dtl_buffers(unsigned long *time_limit)
184	{
185	#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
186	int cpu;
187	struct paca_struct *pp;
188
189	for_each_possible_cpu(cpu) {
190	pp = paca_ptrs[cpu];
191	if (!pp->dispatch_log)
192	continue;
193	kmem_cache_free(dtl_cache, pp->dispatch_log);
194	pp->dtl_ridx = 0;
195	pp->dispatch_log = NULL;
196	pp->dispatch_log_end = NULL;
197	pp->dtl_curr = NULL;
198
199	if (time_limit && time_after(jiffies, *time_limit)) {
200	cond_resched();
201	*time_limit = jiffies + HZ;
202	}
203	}
204	#endif
205	}
206
207	static int init_cpu_associativity(void)
208	{
209	vcpu_associativity = kcalloc(num_possible_cpus() / threads_per_core,
210	VPHN_ASSOC_BUFSIZE * sizeof(__be32), GFP_KERNEL);
211	pcpu_associativity = kcalloc(NR_CPUS_H / threads_per_core,
212	VPHN_ASSOC_BUFSIZE * sizeof(__be32), GFP_KERNEL);
213
214	if (!vcpu_associativity || !pcpu_associativity) {
215	pr_err("error allocating memory for associativity information\n");
216	return -ENOMEM;
217	}
218
219	return 0;
220	}
221
222	static void destroy_cpu_associativity(void)
223	{
224	kfree(vcpu_associativity);
225	kfree(pcpu_associativity);
226	vcpu_associativity = pcpu_associativity = NULL;
227	}
228
229	static __be32 *__get_cpu_associativity(int cpu, __be32 *cpu_assoc, int flag)
230	{
231	__be32 *assoc;
232	int rc = 0;
233
234	assoc = &cpu_assoc[(int)(cpu / threads_per_core) * VPHN_ASSOC_BUFSIZE];
235	if (!assoc[0]) {
236	rc = hcall_vphn(cpu, flag, &assoc[0]);
237	if (rc)
238	return NULL;
239	}
240
241	return assoc;
242	}
243
244	static __be32 *get_pcpu_associativity(int cpu)
245	{
246	return __get_cpu_associativity(cpu, pcpu_associativity, VPHN_FLAG_PCPU);
247	}
248
249	static __be32 *get_vcpu_associativity(int cpu)
250	{
251	return __get_cpu_associativity(cpu, vcpu_associativity, VPHN_FLAG_VCPU);
252	}
253
254	static int cpu_relative_dispatch_distance(int last_disp_cpu, int cur_disp_cpu)
255	{
256	__be32 *last_disp_cpu_assoc, *cur_disp_cpu_assoc;
257
258	if (last_disp_cpu >= NR_CPUS_H || cur_disp_cpu >= NR_CPUS_H)
259	return -EINVAL;
260
261	last_disp_cpu_assoc = get_pcpu_associativity(last_disp_cpu);
262	cur_disp_cpu_assoc = get_pcpu_associativity(cur_disp_cpu);
263
264	if (!last_disp_cpu_assoc || !cur_disp_cpu_assoc)
265	return -EIO;
266
267	return cpu_relative_distance(last_disp_cpu_assoc, cur_disp_cpu_assoc);
268	}
269
270	static int cpu_home_node_dispatch_distance(int disp_cpu)
271	{
272	__be32 *disp_cpu_assoc, *vcpu_assoc;
273	int vcpu_id = smp_processor_id();
274
275	if (disp_cpu >= NR_CPUS_H) {
276	pr_debug_ratelimited("vcpu dispatch cpu %d > %d\n",
277	disp_cpu, NR_CPUS_H);
278	return -EINVAL;
279	}
280
281	disp_cpu_assoc = get_pcpu_associativity(disp_cpu);
282	vcpu_assoc = get_vcpu_associativity(vcpu_id);
283
284	if (!disp_cpu_assoc || !vcpu_assoc)
285	return -EIO;
286
287	return cpu_relative_distance(disp_cpu_assoc, vcpu_assoc);
288	}
289
290	static void update_vcpu_disp_stat(int disp_cpu)
291	{
292	struct vcpu_dispatch_data *disp;
293	int distance;
294
295	disp = this_cpu_ptr(&vcpu_disp_data);
296	if (disp->last_disp_cpu == -1) {
297	disp->last_disp_cpu = disp_cpu;
298	return;
299	}
300
301	disp->total_disp++;
302
303	if (disp->last_disp_cpu == disp_cpu ||
304	(cpu_first_thread_sibling(disp->last_disp_cpu) ==
305	cpu_first_thread_sibling(disp_cpu)))
306	disp->same_cpu_disp++;
307	else {
308	distance = cpu_relative_dispatch_distance(disp->last_disp_cpu,
309	disp_cpu);
310	if (distance < 0)
311	pr_debug_ratelimited("vcpudispatch_stats: cpu %d: error determining associativity\n",
312	smp_processor_id());
313	else {
314	switch (distance) {
315	case 0:
316	disp->same_chip_disp++;
317	break;
318	case 1:
319	disp->diff_chip_disp++;
320	break;
321	case 2:
322	disp->far_chip_disp++;
323	break;
324	default:
325	pr_debug_ratelimited("vcpudispatch_stats: cpu %d (%d -> %d): unexpected relative dispatch distance %d\n",
326	smp_processor_id(),
327	disp->last_disp_cpu,
328	disp_cpu,
329	distance);
330	}
331	}
332	}
333
334	distance = cpu_home_node_dispatch_distance(disp_cpu);
335	if (distance < 0)
336	pr_debug_ratelimited("vcpudispatch_stats: cpu %d: error determining associativity\n",
337	smp_processor_id());
338	else {
339	switch (distance) {
340	case 0:
341	disp->numa_home_disp++;
342	break;
343	case 1:
344	disp->numa_remote_disp++;
345	break;
346	case 2:
347	disp->numa_far_disp++;
348	break;
349	default:
350	pr_debug_ratelimited("vcpudispatch_stats: cpu %d on %d: unexpected numa dispatch distance %d\n",
351	smp_processor_id(),
352	disp_cpu,
353	distance);
354	}
355	}
356
357	disp->last_disp_cpu = disp_cpu;
358	}
359
360	static void process_dtl_buffer(struct work_struct *work)
361	{
362	struct dtl_entry dtle;
363	u64 i = __this_cpu_read(dtl_entry_ridx);
364	struct dtl_entry *dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
365	struct dtl_entry *dtl_end = local_paca->dispatch_log_end;
366	struct lppaca *vpa = local_paca->lppaca_ptr;
367	struct dtl_worker *d = container_of(work, struct dtl_worker, work.work);
368
369	if (!local_paca->dispatch_log)
370	return;
371
372	/* if we have been migrated away, we cancel ourself */
373	if (d->cpu != smp_processor_id()) {
374	pr_debug("vcpudispatch_stats: cpu %d worker migrated -- canceling worker\n",
375	smp_processor_id());
376	return;
377	}
378
379	if (i == be64_to_cpu(vpa->dtl_idx))
380	goto out;
381
382	while (i < be64_to_cpu(vpa->dtl_idx)) {
383	dtle = *dtl;
384	barrier();
385	if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {
386	/* buffer has overflowed */
387	pr_debug_ratelimited("vcpudispatch_stats: cpu %d lost %lld DTL samples\n",
388	d->cpu,
389	be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG - i);
390	i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;
391	dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
392	continue;
393	}
394	update_vcpu_disp_stat(be16_to_cpu(dtle.processor_id));
395	++i;
396	++dtl;
397	if (dtl == dtl_end)
398	dtl = local_paca->dispatch_log;
399	}
400
401	__this_cpu_write(dtl_entry_ridx, i);
402
403	out:
404	schedule_delayed_work_on(d->cpu, to_delayed_work(work),
405	HZ / vcpudispatch_stats_freq);
406	}
407
408	static int dtl_worker_online(unsigned int cpu)
409	{
410	struct dtl_worker *d = &per_cpu(dtl_workers, cpu);
411
412	memset(d, 0, sizeof(*d));
413	INIT_DELAYED_WORK(&d->work, process_dtl_buffer);
414	d->cpu = cpu;
415
416	#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
417	per_cpu(dtl_entry_ridx, cpu) = 0;
418	register_dtl_buffer(cpu);
419	#else
420	per_cpu(dtl_entry_ridx, cpu) = be64_to_cpu(lppaca_of(cpu).dtl_idx);
421	#endif
422
423	schedule_delayed_work_on(cpu, &d->work, HZ / vcpudispatch_stats_freq);
424	return 0;
425	}
426
427	static int dtl_worker_offline(unsigned int cpu)
428	{
429	struct dtl_worker *d = &per_cpu(dtl_workers, cpu);
430
431	cancel_delayed_work_sync(&d->work);
432
433	#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
434	unregister_dtl(get_hard_smp_processor_id(cpu));
435	#endif
436
437	return 0;
438	}
439
440	static void set_global_dtl_mask(u8 mask)
441	{
442	int cpu;
443
444	dtl_mask = mask;
445	for_each_present_cpu(cpu)
446	lppaca_of(cpu).dtl_enable_mask = dtl_mask;
447	}
448
449	static void reset_global_dtl_mask(void)
450	{
451	int cpu;
452
453	#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
454	dtl_mask = DTL_LOG_PREEMPT;
455	#else
456	dtl_mask = 0;
457	#endif
458	for_each_present_cpu(cpu)
459	lppaca_of(cpu).dtl_enable_mask = dtl_mask;
460	}
461
462	static int dtl_worker_enable(unsigned long *time_limit)
463	{
464	int rc = 0, state;
465
466	if (!write_trylock(&dtl_access_lock)) {
467	rc = -EBUSY;
468	goto out;
469	}
470
471	set_global_dtl_mask(DTL_LOG_ALL);
472
473	/* Setup dtl buffers and register those */
474	alloc_dtl_buffers(time_limit);
475
476	state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/dtl:online",
477	dtl_worker_online, dtl_worker_offline);
478	if (state < 0) {
479	pr_err("vcpudispatch_stats: unable to setup workqueue for DTL processing\n");
480	free_dtl_buffers(time_limit);
481	reset_global_dtl_mask();
482	write_unlock(&dtl_access_lock);
483	rc = -EINVAL;
484	goto out;
485	}
486	dtl_worker_state = state;
487
488	out:
489	return rc;
490	}
491
492	static void dtl_worker_disable(unsigned long *time_limit)
493	{
494	cpuhp_remove_state(dtl_worker_state);
495	free_dtl_buffers(time_limit);
496	reset_global_dtl_mask();
497	write_unlock(&dtl_access_lock);
498	}
499
500	static ssize_t vcpudispatch_stats_write(struct file *file, const char __user *p,
501	size_t count, loff_t *ppos)
502	{
503	unsigned long time_limit = jiffies + HZ;
504	struct vcpu_dispatch_data *disp;
505	int rc, cmd, cpu;
506	char buf[16];
507
508	if (count > 15)
509	return -EINVAL;
510
511	if (copy_from_user(buf, p, count))
512	return -EFAULT;
513
514	buf[count] = 0;
515	rc = kstrtoint(buf, 0, &cmd);
516	if (rc || cmd < 0 || cmd > 1) {
517	pr_err("vcpudispatch_stats: please use 0 to disable or 1 to enable dispatch statistics\n");
518	return rc ? rc : -EINVAL;
519	}
520
521	mutex_lock(&dtl_enable_mutex);
522
523	if ((cmd == 0 && !vcpudispatch_stats_on) ||
524	(cmd == 1 && vcpudispatch_stats_on))
525	goto out;
526
527	if (cmd) {
528	rc = init_cpu_associativity();
529	if (rc) {
530	destroy_cpu_associativity();
531	goto out;
532	}
533
534	for_each_possible_cpu(cpu) {
535	disp = per_cpu_ptr(&vcpu_disp_data, cpu);
536	memset(disp, 0, sizeof(*disp));
537	disp->last_disp_cpu = -1;
538	}
539
540	rc = dtl_worker_enable(&time_limit);
541	if (rc) {
542	destroy_cpu_associativity();
543	goto out;
544	}
545	} else {
546	dtl_worker_disable(&time_limit);
547	destroy_cpu_associativity();
548	}
549
550	vcpudispatch_stats_on = cmd;
551
552	out:
553	mutex_unlock(&dtl_enable_mutex);
554	if (rc)
555	return rc;
556	return count;
557	}
558
559	static int vcpudispatch_stats_display(struct seq_file *p, void *v)
560	{
561	int cpu;
562	struct vcpu_dispatch_data *disp;
563
564	if (!vcpudispatch_stats_on) {
565	seq_puts(p, "off\n");
566	return 0;
567	}
568
569	for_each_online_cpu(cpu) {
570	disp = per_cpu_ptr(&vcpu_disp_data, cpu);
571	seq_printf(p, "cpu%d", cpu);
572	seq_put_decimal_ull(p, " ", disp->total_disp);
573	seq_put_decimal_ull(p, " ", disp->same_cpu_disp);
574	seq_put_decimal_ull(p, " ", disp->same_chip_disp);
575	seq_put_decimal_ull(p, " ", disp->diff_chip_disp);
576	seq_put_decimal_ull(p, " ", disp->far_chip_disp);
577	seq_put_decimal_ull(p, " ", disp->numa_home_disp);
578	seq_put_decimal_ull(p, " ", disp->numa_remote_disp);
579	seq_put_decimal_ull(p, " ", disp->numa_far_disp);
580	seq_puts(p, "\n");
581	}
582
583	return 0;
584	}
585
586	static int vcpudispatch_stats_open(struct inode *inode, struct file *file)
587	{
588	return single_open(file, vcpudispatch_stats_display, NULL);
589	}
590
591	static const struct proc_ops vcpudispatch_stats_proc_ops = {
592	.proc_open = vcpudispatch_stats_open,
593	.proc_read = seq_read,
594	.proc_write = vcpudispatch_stats_write,
595	.proc_lseek = seq_lseek,
596	.proc_release = single_release,
597	};
598
599	static ssize_t vcpudispatch_stats_freq_write(struct file *file,
600	const char __user *p, size_t count, loff_t *ppos)
601	{
602	int rc, freq;
603	char buf[16];
604
605	if (count > 15)
606	return -EINVAL;
607
608	if (copy_from_user(buf, p, count))
609	return -EFAULT;
610
611	buf[count] = 0;
612	rc = kstrtoint(buf, 0, &freq);
613	if (rc || freq < 1 || freq > HZ) {
614	pr_err("vcpudispatch_stats_freq: please specify a frequency between 1 and %d\n",
615	HZ);
616	return rc ? rc : -EINVAL;
617	}
618
619	vcpudispatch_stats_freq = freq;
620
621	return count;
622	}
623
624	static int vcpudispatch_stats_freq_display(struct seq_file *p, void *v)
625	{
626	seq_printf(p, "%d\n", vcpudispatch_stats_freq);
627	return 0;
628	}
629
630	static int vcpudispatch_stats_freq_open(struct inode *inode, struct file *file)
631	{
632	return single_open(file, vcpudispatch_stats_freq_display, NULL);
633	}
634
635	static const struct proc_ops vcpudispatch_stats_freq_proc_ops = {
636	.proc_open = vcpudispatch_stats_freq_open,
637	.proc_read = seq_read,
638	.proc_write = vcpudispatch_stats_freq_write,
639	.proc_lseek = seq_lseek,
640	.proc_release = single_release,
641	};
642
643	static int __init vcpudispatch_stats_procfs_init(void)
644	{
645	if (!lppaca_shared_proc())
646	return 0;
647
648	if (!proc_create("powerpc/vcpudispatch_stats", 0600, NULL,
649	&vcpudispatch_stats_proc_ops))
650	pr_err("vcpudispatch_stats: error creating procfs file\n");
651	else if (!proc_create("powerpc/vcpudispatch_stats_freq", 0600, NULL,
652	&vcpudispatch_stats_freq_proc_ops))
653	pr_err("vcpudispatch_stats_freq: error creating procfs file\n");
654
655	return 0;
656	}
657
658	machine_device_initcall(pseries, vcpudispatch_stats_procfs_init);
659
660	#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
661	u64 pseries_paravirt_steal_clock(int cpu)
662	{
663	struct lppaca *lppaca = &lppaca_of(cpu);
664
665	/*
666	* VPA steal time counters are reported at TB frequency. Hence do a
667	* conversion to ns before returning
668	*/
669	return tb_to_ns(be64_to_cpu(READ_ONCE(lppaca->enqueue_dispatch_tb)) +
670	be64_to_cpu(READ_ONCE(lppaca->ready_enqueue_tb)));
671	}
672	#endif
673
674	#endif /* CONFIG_PPC_SPLPAR */
675
676	void vpa_init(int cpu)
677	{
678	int hwcpu = get_hard_smp_processor_id(cpu);
679	unsigned long addr;
680	long ret;
681
682	/*
683	* The spec says it "may be problematic" if CPU x registers the VPA of
684	* CPU y. We should never do that, but wail if we ever do.
685	*/
686	WARN_ON(cpu != smp_processor_id());
687
688	if (cpu_has_feature(CPU_FTR_ALTIVEC))
689	lppaca_of(cpu).vmxregs_in_use = 1;
690
691	if (cpu_has_feature(CPU_FTR_ARCH_207S))
692	lppaca_of(cpu).ebb_regs_in_use = 1;
693
694	addr = __pa(&lppaca_of(cpu));
695	ret = register_vpa(hwcpu, addr);
696
697	if (ret) {
698	pr_err("WARNING: VPA registration for cpu %d (hw %d) of area "
699	"%lx failed with %ld\n", cpu, hwcpu, addr, ret);
700	return;
701	}
702
703	#ifdef CONFIG_PPC_64S_HASH_MMU
704	/*
705	* PAPR says this feature is SLB-Buffer but firmware never
706	* reports that. All SPLPAR support SLB shadow buffer.
707	*/
708	if (!radix_enabled() && firmware_has_feature(FW_FEATURE_SPLPAR)) {
709	addr = __pa(paca_ptrs[cpu]->slb_shadow_ptr);
710	ret = register_slb_shadow(hwcpu, addr);
711	if (ret)
712	pr_err("WARNING: SLB shadow buffer registration for "
713	"cpu %d (hw %d) of area %lx failed with %ld\n",
714	cpu, hwcpu, addr, ret);
715	}
716	#endif /* CONFIG_PPC_64S_HASH_MMU */
717
718	/*
719	* Register dispatch trace log, if one has been allocated.
720	*/
721	register_dtl_buffer(cpu);
722	}
723
724	#ifdef CONFIG_PPC_BOOK3S_64
725
726	static int __init pseries_lpar_register_process_table(unsigned long base,
727	unsigned long page_size, unsigned long table_size)
728	{
729	long rc;
730	unsigned long flags = 0;
731
732	if (table_size)
733	flags |= PROC_TABLE_NEW;
734	if (radix_enabled()) {
735	flags |= PROC_TABLE_RADIX;
736	if (mmu_has_feature(MMU_FTR_GTSE))
737	flags |= PROC_TABLE_GTSE;
738	} else
739	flags |= PROC_TABLE_HPT_SLB;
740	for (;;) {
741	rc = plpar_hcall_norets(H_REGISTER_PROC_TBL, flags, base,
742	page_size, table_size);
743	if (!H_IS_LONG_BUSY(rc))
744	break;
745	mdelay(get_longbusy_msecs(rc));
746	}
747	if (rc != H_SUCCESS) {
748	pr_err("Failed to register process table (rc=%ld)\n", rc);
749	BUG();
750	}
751	return rc;
752	}
753
754	#ifdef CONFIG_PPC_64S_HASH_MMU
755
756	static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
757	unsigned long vpn, unsigned long pa,
758	unsigned long rflags, unsigned long vflags,
759	int psize, int apsize, int ssize)
760	{
761	unsigned long lpar_rc;
762	unsigned long flags;
763	unsigned long slot;
764	unsigned long hpte_v, hpte_r;
765
766	if (!(vflags & HPTE_V_BOLTED))
767	pr_devel("hpte_insert(group=%lx, vpn=%016lx, "
768	"pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n",
769	hpte_group, vpn, pa, rflags, vflags, psize);
770
771	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
772	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
773
774	if (!(vflags & HPTE_V_BOLTED))
775	pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
776
777	/* Now fill in the actual HPTE */
778	/* Set CEC cookie to 0 */
779	/* Zero page = 0 */
780	/* I-cache Invalidate = 0 */
781	/* I-cache synchronize = 0 */
782	/* Exact = 0 */
783	flags = 0;
784
785	if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N))
786	flags |= H_COALESCE_CAND;
787
788	lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot);
789	if (unlikely(lpar_rc == H_PTEG_FULL)) {
790	pr_devel("Hash table group is full\n");
791	return -1;
792	}
793
794	/*
795	* Since we try and ioremap PHBs we don't own, the pte insert
796	* will fail. However we must catch the failure in hash_page
797	* or we will loop forever, so return -2 in this case.
798	*/
799	if (unlikely(lpar_rc != H_SUCCESS)) {
800	pr_err("Failed hash pte insert with error %ld\n", lpar_rc);
801	return -2;
802	}
803	if (!(vflags & HPTE_V_BOLTED))
804	pr_devel(" -> slot: %lu\n", slot & 7);
805
806	/* Because of iSeries, we have to pass down the secondary
807	* bucket bit here as well
808	*/
809	return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3);
810	}
811
812	static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock);
813
814	static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
815	{
816	unsigned long slot_offset;
817	unsigned long lpar_rc;
818	int i;
819	unsigned long dummy1, dummy2;
820
821	/* pick a random slot to start at */
822	slot_offset = mftb() & 0x7;
823
824	for (i = 0; i < HPTES_PER_GROUP; i++) {
825
826	/* don't remove a bolted entry */
827	lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset,
828	HPTE_V_BOLTED, &dummy1, &dummy2);
829	if (lpar_rc == H_SUCCESS)
830	return i;
831
832	/*
833	* The test for adjunct partition is performed before the
834	* ANDCOND test. H_RESOURCE may be returned, so we need to
835	* check for that as well.
836	*/
837	BUG_ON(lpar_rc != H_NOT_FOUND && lpar_rc != H_RESOURCE);
838
839	slot_offset++;
840	slot_offset &= 0x7;
841	}
842
843	return -1;
844	}
845
846	/* Called during kexec sequence with MMU off */
847	static notrace void manual_hpte_clear_all(void)
848	{
849	unsigned long size_bytes = 1UL << ppc64_pft_size;
850	unsigned long hpte_count = size_bytes >> 4;
851	struct {
852	unsigned long pteh;
853	unsigned long ptel;
854	} ptes[4];
855	long lpar_rc;
856	unsigned long i, j;
857
858	/* Read in batches of 4,
859	* invalidate only valid entries not in the VRMA
860	* hpte_count will be a multiple of 4
861	*/
862	for (i = 0; i < hpte_count; i += 4) {
863	lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes);
864	if (lpar_rc != H_SUCCESS) {
865	pr_info("Failed to read hash page table at %ld err %ld\n",
866	i, lpar_rc);
867	continue;
868	}
869	for (j = 0; j < 4; j++){
870	if ((ptes[j].pteh & HPTE_V_VRMA_MASK) ==
871	HPTE_V_VRMA_MASK)
872	continue;
873	if (ptes[j].pteh & HPTE_V_VALID)
874	plpar_pte_remove_raw(0, i + j, 0,
875	&(ptes[j].pteh), &(ptes[j].ptel));
876	}
877	}
878	}
879
880	/* Called during kexec sequence with MMU off */
881	static notrace int hcall_hpte_clear_all(void)
882	{
883	int rc;
884
885	do {
886	rc = plpar_hcall_norets(H_CLEAR_HPT);
887	} while (rc == H_CONTINUE);
888
889	return rc;
890	}
891
892	/* Called during kexec sequence with MMU off */
893	static notrace void pseries_hpte_clear_all(void)
894	{
895	int rc;
896
897	rc = hcall_hpte_clear_all();
898	if (rc != H_SUCCESS)
899	manual_hpte_clear_all();
900
901	#ifdef __LITTLE_ENDIAN__
902	/*
903	* Reset exceptions to big endian.
904	*
905	* FIXME this is a hack for kexec, we need to reset the exception
906	* endian before starting the new kernel and this is a convenient place
907	* to do it.
908	*
909	* This is also called on boot when a fadump happens. In that case we
910	* must not change the exception endian mode.
911	*/
912	if (firmware_has_feature(FW_FEATURE_SET_MODE) && !is_fadump_active())
913	pseries_big_endian_exceptions();
914	#endif
915	}
916
917	/*
918	* NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
919	* the low 3 bits of flags happen to line up. So no transform is needed.
920	* We can probably optimize here and assume the high bits of newpp are
921	* already zero. For now I am paranoid.
922	*/
923	static long pSeries_lpar_hpte_updatepp(unsigned long slot,
924	unsigned long newpp,
925	unsigned long vpn,
926	int psize, int apsize,
927	int ssize, unsigned long inv_flags)
928	{
929	unsigned long lpar_rc;
930	unsigned long flags;
931	unsigned long want_v;
932
933	want_v = hpte_encode_avpn(vpn, psize, ssize);
934
935	flags = (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO)) | H_AVPN;
936	flags |= (newpp & HPTE_R_KEY_HI) >> 48;
937	if (mmu_has_feature(MMU_FTR_KERNEL_RO))
938	/* Move pp0 into bit 8 (IBM 55) */
939	flags |= (newpp & HPTE_R_PP0) >> 55;
940
941	pr_devel(" update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...",
942	want_v, slot, flags, psize);
943
944	lpar_rc = plpar_pte_protect(flags, slot, want_v);
945
946	if (lpar_rc == H_NOT_FOUND) {
947	pr_devel("not found !\n");
948	return -1;
949	}
950
951	pr_devel("ok\n");
952
953	BUG_ON(lpar_rc != H_SUCCESS);
954
955	return 0;
956	}
957
958	static long __pSeries_lpar_hpte_find(unsigned long want_v, unsigned long hpte_group)
959	{
960	long lpar_rc;
961	unsigned long i, j;
962	struct {
963	unsigned long pteh;
964	unsigned long ptel;
965	} ptes[4];
966
967	for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) {
968
969	lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes);
970	if (lpar_rc != H_SUCCESS) {
971	pr_info("Failed to read hash page table at %ld err %ld\n",
972	hpte_group, lpar_rc);
973	continue;
974	}
975
976	for (j = 0; j < 4; j++) {
977	if (HPTE_V_COMPARE(ptes[j].pteh, want_v) &&
978	(ptes[j].pteh & HPTE_V_VALID))
979	return i + j;
980	}
981	}
982
983	return -1;
984	}
985
986	static long pSeries_lpar_hpte_find(unsigned long vpn, int psize, int ssize)
987	{
988	long slot;
989	unsigned long hash;
990	unsigned long want_v;
991	unsigned long hpte_group;
992
993	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
994	want_v = hpte_encode_avpn(vpn, psize, ssize);
995
996	/*
997	* We try to keep bolted entries always in primary hash
998	* But in some case we can find them in secondary too.
999	*/
1000	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1001	slot = __pSeries_lpar_hpte_find(want_v, hpte_group);
1002	if (slot < 0) {
1003	/* Try in secondary */
1004	hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
1005	slot = __pSeries_lpar_hpte_find(want_v, hpte_group);
1006	if (slot < 0)
1007	return -1;
1008	}
1009	return hpte_group + slot;
1010	}
1011
1012	static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp,
1013	unsigned long ea,
1014	int psize, int ssize)
1015	{
1016	unsigned long vpn;
1017	unsigned long lpar_rc, slot, vsid, flags;
1018
1019	vsid = get_kernel_vsid(ea, ssize);
1020	vpn = hpt_vpn(ea, vsid, ssize);
1021
1022	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
1023	BUG_ON(slot == -1);
1024
1025	flags = newpp & (HPTE_R_PP | HPTE_R_N);
1026	if (mmu_has_feature(MMU_FTR_KERNEL_RO))
1027	/* Move pp0 into bit 8 (IBM 55) */
1028	flags |= (newpp & HPTE_R_PP0) >> 55;
1029
1030	flags |= ((newpp & HPTE_R_KEY_HI) >> 48) | (newpp & HPTE_R_KEY_LO);
1031
1032	lpar_rc = plpar_pte_protect(flags, slot, 0);
1033
1034	BUG_ON(lpar_rc != H_SUCCESS);
1035	}
1036
1037	static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
1038	int psize, int apsize,
1039	int ssize, int local)
1040	{
1041	unsigned long want_v;
1042	unsigned long lpar_rc;
1043	unsigned long dummy1, dummy2;
1044
1045	pr_devel(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
1046	slot, vpn, psize, local);
1047
1048	want_v = hpte_encode_avpn(vpn, psize, ssize);
1049	lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2);
1050	if (lpar_rc == H_NOT_FOUND)
1051	return;
1052
1053	BUG_ON(lpar_rc != H_SUCCESS);
1054	}
1055
1056
1057	/*
1058	* As defined in the PAPR's section 14.5.4.1.8
1059	* The control mask doesn't include the returned reference and change bit from
1060	* the processed PTE.
1061	*/
1062	#define HBLKR_AVPN 0x0100000000000000UL
1063	#define HBLKR_CTRL_MASK 0xf800000000000000UL
1064	#define HBLKR_CTRL_SUCCESS 0x8000000000000000UL
1065	#define HBLKR_CTRL_ERRNOTFOUND 0x8800000000000000UL
1066	#define HBLKR_CTRL_ERRBUSY 0xa000000000000000UL
1067
1068	/*
1069	* Returned true if we are supporting this block size for the specified segment
1070	* base page size and actual page size.
1071	*
1072	* Currently, we only support 8 size block.
1073	*/
1074	static inline bool is_supported_hlbkrm(int bpsize, int psize)
1075	{
1076	return (hblkrm_size[bpsize][psize] == HBLKRM_SUPPORTED_BLOCK_SIZE);
1077	}
1078
1079	/**
1080	* H_BLOCK_REMOVE caller.
1081	* @idx should point to the latest @param entry set with a PTEX.
1082	* If PTE cannot be processed because another CPUs has already locked that
1083	* group, those entries are put back in @param starting at index 1.
1084	* If entries has to be retried and @retry_busy is set to true, these entries
1085	* are retried until success. If @retry_busy is set to false, the returned
1086	* is the number of entries yet to process.
1087	*/
1088	static unsigned long call_block_remove(unsigned long idx, unsigned long *param,
1089	bool retry_busy)
1090	{
1091	unsigned long i, rc, new_idx;
1092	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];
1093
1094	if (idx < 2) {
1095	pr_warn("Unexpected empty call to H_BLOCK_REMOVE");
1096	return 0;
1097	}
1098	again:
1099	new_idx = 0;
1100	if (idx > PLPAR_HCALL9_BUFSIZE) {
1101	pr_err("Too many PTEs (%lu) for H_BLOCK_REMOVE", idx);
1102	idx = PLPAR_HCALL9_BUFSIZE;
1103	} else if (idx < PLPAR_HCALL9_BUFSIZE)
1104	param[idx] = HBR_END;
1105
1106	rc = plpar_hcall9(H_BLOCK_REMOVE, retbuf,
1107	param[0], /* AVA */
1108	param[1], param[2], param[3], param[4], /* TS0-7 */
1109	param[5], param[6], param[7], param[8]);
1110	if (rc == H_SUCCESS)
1111	return 0;
1112
1113	BUG_ON(rc != H_PARTIAL);
1114
1115	/* Check that the unprocessed entries were 'not found' or 'busy' */
1116	for (i = 0; i < idx-1; i++) {
1117	unsigned long ctrl = retbuf[i] & HBLKR_CTRL_MASK;
1118
1119	if (ctrl == HBLKR_CTRL_ERRBUSY) {
1120	param[++new_idx] = param[i+1];
1121	continue;
1122	}
1123
1124	BUG_ON(ctrl != HBLKR_CTRL_SUCCESS
1125	&& ctrl != HBLKR_CTRL_ERRNOTFOUND);
1126	}
1127
1128	/*
1129	* If there were entries found busy, retry these entries if requested,
1130	* of if all the entries have to be retried.
1131	*/
1132	if (new_idx && (retry_busy || new_idx == (PLPAR_HCALL9_BUFSIZE-1))) {
1133	idx = new_idx + 1;
1134	goto again;
1135	}
1136
1137	return new_idx;
1138	}
1139
1140	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1141	/*
1142	* Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
1143	* to make sure that we avoid bouncing the hypervisor tlbie lock.
1144	*/
1145	#define PPC64_HUGE_HPTE_BATCH 12
1146
1147	static void hugepage_block_invalidate(unsigned long *slot, unsigned long *vpn,
1148	int count, int psize, int ssize)
1149	{
1150	unsigned long param[PLPAR_HCALL9_BUFSIZE];
1151	unsigned long shift, current_vpgb, vpgb;
1152	int i, pix = 0;
1153
1154	shift = mmu_psize_defs[psize].shift;
1155
1156	for (i = 0; i < count; i++) {
1157	/*
1158	* Shifting 3 bits more on the right to get a
1159	* 8 pages aligned virtual addresse.
1160	*/
1161	vpgb = (vpn[i] >> (shift - VPN_SHIFT + 3));
1162	if (!pix || vpgb != current_vpgb) {
1163	/*
1164	* Need to start a new 8 pages block, flush
1165	* the current one if needed.
1166	*/
1167	if (pix)
1168	(void)call_block_remove(pix, param, true);
1169	current_vpgb = vpgb;
1170	param[0] = hpte_encode_avpn(vpn[i], psize, ssize);
1171	pix = 1;
1172	}
1173
1174	param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot[i];
1175	if (pix == PLPAR_HCALL9_BUFSIZE) {
1176	pix = call_block_remove(pix, param, false);
1177	/*
1178	* pix = 0 means that all the entries were
1179	* removed, we can start a new block.
1180	* Otherwise, this means that there are entries
1181	* to retry, and pix points to latest one, so
1182	* we should increment it and try to continue
1183	* the same block.
1184	*/
1185	if (pix)
1186	pix++;
1187	}
1188	}
1189	if (pix)
1190	(void)call_block_remove(pix, param, true);
1191	}
1192
1193	static void hugepage_bulk_invalidate(unsigned long *slot, unsigned long *vpn,
1194	int count, int psize, int ssize)
1195	{
1196	unsigned long param[PLPAR_HCALL9_BUFSIZE];
1197	int i = 0, pix = 0, rc;
1198
1199	for (i = 0; i < count; i++) {
1200
1201	if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
1202	pSeries_lpar_hpte_invalidate(slot[i], vpn[i], psize, 0,
1203	ssize, 0);
1204	} else {
1205	param[pix] = HBR_REQUEST | HBR_AVPN | slot[i];
1206	param[pix+1] = hpte_encode_avpn(vpn[i], psize, ssize);
1207	pix += 2;
1208	if (pix == 8) {
1209	rc = plpar_hcall9(H_BULK_REMOVE, param,
1210	param[0], param[1], param[2],
1211	param[3], param[4], param[5],
1212	param[6], param[7]);
1213	BUG_ON(rc != H_SUCCESS);
1214	pix = 0;
1215	}
1216	}
1217	}
1218	if (pix) {
1219	param[pix] = HBR_END;
1220	rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
1221	param[2], param[3], param[4], param[5],
1222	param[6], param[7]);
1223	BUG_ON(rc != H_SUCCESS);
1224	}
1225	}
1226
1227	static inline void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
1228	unsigned long *vpn,
1229	int count, int psize,
1230	int ssize)
1231	{
1232	unsigned long flags = 0;
1233	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
1234
1235	if (lock_tlbie)
1236	spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
1237
1238	/* Assuming THP size is 16M */
1239	if (is_supported_hlbkrm(psize, MMU_PAGE_16M))
1240	hugepage_block_invalidate(slot, vpn, count, psize, ssize);
1241	else
1242	hugepage_bulk_invalidate(slot, vpn, count, psize, ssize);
1243
1244	if (lock_tlbie)
1245	spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
1246	}
1247
1248	static void pSeries_lpar_hugepage_invalidate(unsigned long vsid,
1249	unsigned long addr,
1250	unsigned char *hpte_slot_array,
1251	int psize, int ssize, int local)
1252	{
1253	int i, index = 0;
1254	unsigned long s_addr = addr;
1255	unsigned int max_hpte_count, valid;
1256	unsigned long vpn_array[PPC64_HUGE_HPTE_BATCH];
1257	unsigned long slot_array[PPC64_HUGE_HPTE_BATCH];
1258	unsigned long shift, hidx, vpn = 0, hash, slot;
1259
1260	shift = mmu_psize_defs[psize].shift;
1261	max_hpte_count = 1U << (PMD_SHIFT - shift);
1262
1263	for (i = 0; i < max_hpte_count; i++) {
1264	valid = hpte_valid(hpte_slot_array, i);
1265	if (!valid)
1266	continue;
1267	hidx = hpte_hash_index(hpte_slot_array, i);
1268
1269	/* get the vpn */
1270	addr = s_addr + (i * (1ul << shift));
1271	vpn = hpt_vpn(addr, vsid, ssize);
1272	hash = hpt_hash(vpn, shift, ssize);
1273	if (hidx & _PTEIDX_SECONDARY)
1274	hash = ~hash;
1275
1276	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1277	slot += hidx & _PTEIDX_GROUP_IX;
1278
1279	slot_array[index] = slot;
1280	vpn_array[index] = vpn;
1281	if (index == PPC64_HUGE_HPTE_BATCH - 1) {
1282	/*
1283	* Now do a bluk invalidate
1284	*/
1285	__pSeries_lpar_hugepage_invalidate(slot_array,
1286	vpn_array,
1287	PPC64_HUGE_HPTE_BATCH,
1288	psize, ssize);
1289	index = 0;
1290	} else
1291	index++;
1292	}
1293	if (index)
1294	__pSeries_lpar_hugepage_invalidate(slot_array, vpn_array,
1295	index, psize, ssize);
1296	}
1297	#else
1298	static void pSeries_lpar_hugepage_invalidate(unsigned long vsid,
1299	unsigned long addr,
1300	unsigned char *hpte_slot_array,
1301	int psize, int ssize, int local)
1302	{
1303	WARN(1, "%s called without THP support\n", __func__);
1304	}
1305	#endif
1306
1307	static int pSeries_lpar_hpte_removebolted(unsigned long ea,
1308	int psize, int ssize)
1309	{
1310	unsigned long vpn;
1311	unsigned long slot, vsid;
1312
1313	vsid = get_kernel_vsid(ea, ssize);
1314	vpn = hpt_vpn(ea, vsid, ssize);
1315
1316	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
1317	if (slot == -1)
1318	return -ENOENT;
1319
1320	/*
1321	* lpar doesn't use the passed actual page size
1322	*/
1323	pSeries_lpar_hpte_invalidate(slot, vpn, psize, 0, ssize, 0);
1324	return 0;
1325	}
1326
1327
1328	static inline unsigned long compute_slot(real_pte_t pte,
1329	unsigned long vpn,
1330	unsigned long index,
1331	unsigned long shift,
1332	int ssize)
1333	{
1334	unsigned long slot, hash, hidx;
1335
1336	hash = hpt_hash(vpn, shift, ssize);
1337	hidx = __rpte_to_hidx(pte, index);
1338	if (hidx & _PTEIDX_SECONDARY)
1339	hash = ~hash;
1340	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1341	slot += hidx & _PTEIDX_GROUP_IX;
1342	return slot;
1343	}
1344
1345	/**
1346	* The hcall H_BLOCK_REMOVE implies that the virtual pages to processed are
1347	* "all within the same naturally aligned 8 page virtual address block".
1348	*/
1349	static void do_block_remove(unsigned long number, struct ppc64_tlb_batch *batch,
1350	unsigned long *param)
1351	{
1352	unsigned long vpn;
1353	unsigned long i, pix = 0;
1354	unsigned long index, shift, slot, current_vpgb, vpgb;
1355	real_pte_t pte;
1356	int psize, ssize;
1357
1358	psize = batch->psize;
1359	ssize = batch->ssize;
1360
1361	for (i = 0; i < number; i++) {
1362	vpn = batch->vpn[i];
1363	pte = batch->pte[i];
1364	pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
1365	/*
1366	* Shifting 3 bits more on the right to get a
1367	* 8 pages aligned virtual addresse.
1368	*/
1369	vpgb = (vpn >> (shift - VPN_SHIFT + 3));
1370	if (!pix || vpgb != current_vpgb) {
1371	/*
1372	* Need to start a new 8 pages block, flush
1373	* the current one if needed.
1374	*/
1375	if (pix)
1376	(void)call_block_remove(pix, param,
1377	true);
1378	current_vpgb = vpgb;
1379	param[0] = hpte_encode_avpn(vpn, psize,
1380	ssize);
1381	pix = 1;
1382	}
1383
1384	slot = compute_slot(pte, vpn, index, shift, ssize);
1385	param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot;
1386
1387	if (pix == PLPAR_HCALL9_BUFSIZE) {
1388	pix = call_block_remove(pix, param, false);
1389	/*
1390	* pix = 0 means that all the entries were
1391	* removed, we can start a new block.
1392	* Otherwise, this means that there are entries
1393	* to retry, and pix points to latest one, so
1394	* we should increment it and try to continue
1395	* the same block.
1396	*/
1397	if (pix)
1398	pix++;
1399	}
1400	} pte_iterate_hashed_end();
1401	}
1402
1403	if (pix)
1404	(void)call_block_remove(pix, param, true);
1405	}
1406
1407	/*
1408	* TLB Block Invalidate Characteristics
1409	*
1410	* These characteristics define the size of the block the hcall H_BLOCK_REMOVE
1411	* is able to process for each couple segment base page size, actual page size.
1412	*
1413	* The ibm,get-system-parameter properties is returning a buffer with the
1414	* following layout:
1415	*
1416	* [ 2 bytes size of the RTAS buffer (excluding these 2 bytes) ]
1417	* -----------------
1418	* TLB Block Invalidate Specifiers:
1419	* [ 1 byte LOG base 2 of the TLB invalidate block size being specified ]
1420	* [ 1 byte Number of page sizes (N) that are supported for the specified
1421	* TLB invalidate block size ]
1422	* [ 1 byte Encoded segment base page size and actual page size
1423	* MSB=0 means 4k segment base page size and actual page size
1424	* MSB=1 the penc value in mmu_psize_def ]
1425	* ...
1426	* -----------------
1427	* Next TLB Block Invalidate Specifiers...
1428	* -----------------
1429	* [ 0 ]
1430	*/
1431	static inline void set_hblkrm_bloc_size(int bpsize, int psize,
1432	unsigned int block_size)
1433	{
1434	if (block_size > hblkrm_size[bpsize][psize])
1435	hblkrm_size[bpsize][psize] = block_size;
1436	}
1437
1438	/*
1439	* Decode the Encoded segment base page size and actual page size.
1440	* PAPR specifies:
1441	* - bit 7 is the L bit
1442	* - bits 0-5 are the penc value
1443	* If the L bit is 0, this means 4K segment base page size and actual page size
1444	* otherwise the penc value should be read.
1445	*/
1446	#define HBLKRM_L_MASK 0x80
1447	#define HBLKRM_PENC_MASK 0x3f
1448	static inline void __init check_lp_set_hblkrm(unsigned int lp,
1449	unsigned int block_size)
1450	{
1451	unsigned int bpsize, psize;
1452
1453	/* First, check the L bit, if not set, this means 4K */
1454	if ((lp & HBLKRM_L_MASK) == 0) {
1455	set_hblkrm_bloc_size(MMU_PAGE_4K, MMU_PAGE_4K, block_size);
1456	return;
1457	}
1458
1459	lp &= HBLKRM_PENC_MASK;
1460	for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++) {
1461	struct mmu_psize_def *def = &mmu_psize_defs[bpsize];
1462
1463	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
1464	if (def->penc[psize] == lp) {
1465	set_hblkrm_bloc_size(bpsize, psize, block_size);
1466	return;
1467	}
1468	}
1469	}
1470	}
1471
1472	/*
1473	* The size of the TLB Block Invalidate Characteristics is variable. But at the
1474	* maximum it will be the number of possible page sizes *2 + 10 bytes.
1475	* Currently MMU_PAGE_COUNT is 16, which means 42 bytes. Use a cache line size
1476	* (128 bytes) for the buffer to get plenty of space.
1477	*/
1478	#define SPLPAR_TLB_BIC_MAXLENGTH 128
1479
1480	void __init pseries_lpar_read_hblkrm_characteristics(void)
1481	{
1482	static struct papr_sysparm_buf buf __initdata;
1483	int len, idx, bpsize;
1484
1485	if (!firmware_has_feature(FW_FEATURE_BLOCK_REMOVE))
1486	return;
1487
1488	if (papr_sysparm_get(PAPR_SYSPARM_TLB_BLOCK_INVALIDATE_ATTRS, &buf))
1489	return;
1490
1491	len = be16_to_cpu(buf.len);
1492	if (len > SPLPAR_TLB_BIC_MAXLENGTH) {
1493	pr_warn("%s too large returned buffer %d", __func__, len);
1494	return;
1495	}
1496
1497	idx = 0;
1498	while (idx < len) {
1499	u8 block_shift = buf.val[idx++];
1500	u32 block_size;
1501	unsigned int npsize;
1502
1503	if (!block_shift)
1504	break;
1505
1506	block_size = 1 << block_shift;
1507
1508	for (npsize = buf.val[idx++];
1509	npsize > 0 && idx < len; npsize--)
1510	check_lp_set_hblkrm((unsigned int)buf.val[idx++],
1511	block_size);
1512	}
1513
1514	for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++)
1515	for (idx = 0; idx < MMU_PAGE_COUNT; idx++)
1516	if (hblkrm_size[bpsize][idx])
1517	pr_info("H_BLOCK_REMOVE supports base psize:%d psize:%d block size:%d",
1518	bpsize, idx, hblkrm_size[bpsize][idx]);
1519	}
1520
1521	/*
1522	* Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
1523	* lock.
1524	*/
1525	static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
1526	{
1527	unsigned long vpn;
1528	unsigned long i, pix, rc;
1529	unsigned long flags = 0;
1530	struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
1531	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
1532	unsigned long param[PLPAR_HCALL9_BUFSIZE];
1533	unsigned long index, shift, slot;
1534	real_pte_t pte;
1535	int psize, ssize;
1536
1537	if (lock_tlbie)
1538	spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
1539
1540	if (is_supported_hlbkrm(batch->psize, batch->psize)) {
1541	do_block_remove(number, batch, param);
1542	goto out;
1543	}
1544
1545	psize = batch->psize;
1546	ssize = batch->ssize;
1547	pix = 0;
1548	for (i = 0; i < number; i++) {
1549	vpn = batch->vpn[i];
1550	pte = batch->pte[i];
1551	pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
1552	slot = compute_slot(pte, vpn, index, shift, ssize);
1553	if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
1554	/*
1555	* lpar doesn't use the passed actual page size
1556	*/
1557	pSeries_lpar_hpte_invalidate(slot, vpn, psize,
1558	0, ssize, local);
1559	} else {
1560	param[pix] = HBR_REQUEST | HBR_AVPN | slot;
1561	param[pix+1] = hpte_encode_avpn(vpn, psize,
1562	ssize);
1563	pix += 2;
1564	if (pix == 8) {
1565	rc = plpar_hcall9(H_BULK_REMOVE, param,
1566	param[0], param[1], param[2],
1567	param[3], param[4], param[5],
1568	param[6], param[7]);
1569	BUG_ON(rc != H_SUCCESS);
1570	pix = 0;
1571	}
1572	}
1573	} pte_iterate_hashed_end();
1574	}
1575	if (pix) {
1576	param[pix] = HBR_END;
1577	rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
1578	param[2], param[3], param[4], param[5],
1579	param[6], param[7]);
1580	BUG_ON(rc != H_SUCCESS);
1581	}
1582
1583	out:
1584	if (lock_tlbie)
1585	spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
1586	}
1587
1588	static int __init disable_bulk_remove(char *str)
1589	{
1590	if (strcmp(str, "off") == 0 &&
1591	firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
1592	pr_info("Disabling BULK_REMOVE firmware feature");
1593	powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE;
1594	}
1595	return 1;
1596	}
1597
1598	__setup("bulk_remove=", disable_bulk_remove);
1599
1600	#define HPT_RESIZE_TIMEOUT 10000 /* ms */
1601
1602	struct hpt_resize_state {
1603	unsigned long shift;
1604	int commit_rc;
1605	};
1606
1607	static int pseries_lpar_resize_hpt_commit(void *data)
1608	{
1609	struct hpt_resize_state *state = data;
1610
1611	state->commit_rc = plpar_resize_hpt_commit(0, state->shift);
1612	if (state->commit_rc != H_SUCCESS)
1613	return -EIO;
1614
1615	/* Hypervisor has transitioned the HTAB, update our globals */
1616	ppc64_pft_size = state->shift;
1617	htab_size_bytes = 1UL << ppc64_pft_size;
1618	htab_hash_mask = (htab_size_bytes >> 7) - 1;
1619
1620	return 0;
1621	}
1622
1623	/*
1624	* Must be called in process context. The caller must hold the
1625	* cpus_lock.
1626	*/
1627	static int pseries_lpar_resize_hpt(unsigned long shift)
1628	{
1629	struct hpt_resize_state state = {
1630	.shift = shift,
1631	.commit_rc = H_FUNCTION,
1632	};
1633	unsigned int delay, total_delay = 0;
1634	int rc;
1635	ktime_t t0, t1, t2;
1636
1637	might_sleep();
1638
1639	if (!firmware_has_feature(FW_FEATURE_HPT_RESIZE))
1640	return -ENODEV;
1641
1642	pr_info("Attempting to resize HPT to shift %lu\n", shift);
1643
1644	t0 = ktime_get();
1645
1646	rc = plpar_resize_hpt_prepare(0, shift);
1647	while (H_IS_LONG_BUSY(rc)) {
1648	delay = get_longbusy_msecs(rc);
1649	total_delay += delay;
1650	if (total_delay > HPT_RESIZE_TIMEOUT) {
1651	/* prepare with shift==0 cancels an in-progress resize */
1652	rc = plpar_resize_hpt_prepare(0, 0);
1653	if (rc != H_SUCCESS)
1654	pr_warn("Unexpected error %d cancelling timed out HPT resize\n",
1655	rc);
1656	return -ETIMEDOUT;
1657	}
1658	msleep(delay);
1659	rc = plpar_resize_hpt_prepare(0, shift);
1660	}
1661
1662	switch (rc) {
1663	case H_SUCCESS:
1664	/* Continue on */
1665	break;
1666
1667	case H_PARAMETER:
1668	pr_warn("Invalid argument from H_RESIZE_HPT_PREPARE\n");
1669	return -EINVAL;
1670	case H_RESOURCE:
1671	pr_warn("Operation not permitted from H_RESIZE_HPT_PREPARE\n");
1672	return -EPERM;
1673	default:
1674	pr_warn("Unexpected error %d from H_RESIZE_HPT_PREPARE\n", rc);
1675	return -EIO;
1676	}
1677
1678	t1 = ktime_get();
1679
1680	rc = stop_machine_cpuslocked(pseries_lpar_resize_hpt_commit,
1681	&state, NULL);
1682
1683	t2 = ktime_get();
1684
1685	if (rc != 0) {
1686	switch (state.commit_rc) {
1687	case H_PTEG_FULL:
1688	return -ENOSPC;
1689
1690	default:
1691	pr_warn("Unexpected error %d from H_RESIZE_HPT_COMMIT\n",
1692	state.commit_rc);
1693	return -EIO;
1694	};
1695	}
1696
1697	pr_info("HPT resize to shift %lu complete (%lld ms / %lld ms)\n",
1698	shift, (long long) ktime_ms_delta(t1, t0),
1699	(long long) ktime_ms_delta(t2, t1));
1700
1701	return 0;
1702	}
1703
1704	void __init hpte_init_pseries(void)
1705	{
1706	mmu_hash_ops.hpte_invalidate = pSeries_lpar_hpte_invalidate;
1707	mmu_hash_ops.hpte_updatepp = pSeries_lpar_hpte_updatepp;
1708	mmu_hash_ops.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
1709	mmu_hash_ops.hpte_insert = pSeries_lpar_hpte_insert;
1710	mmu_hash_ops.hpte_remove = pSeries_lpar_hpte_remove;
1711	mmu_hash_ops.hpte_removebolted = pSeries_lpar_hpte_removebolted;
1712	mmu_hash_ops.flush_hash_range = pSeries_lpar_flush_hash_range;
1713	mmu_hash_ops.hpte_clear_all = pseries_hpte_clear_all;
1714	mmu_hash_ops.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
1715
1716	if (firmware_has_feature(FW_FEATURE_HPT_RESIZE))
1717	mmu_hash_ops.resize_hpt = pseries_lpar_resize_hpt;
1718
1719	/*
1720	* On POWER9, we need to do a H_REGISTER_PROC_TBL hcall
1721	* to inform the hypervisor that we wish to use the HPT.
1722	*/
1723	if (cpu_has_feature(CPU_FTR_ARCH_300))
1724	pseries_lpar_register_process_table(0, 0, 0);
1725	}
1726	#endif /* CONFIG_PPC_64S_HASH_MMU */
1727
1728	#ifdef CONFIG_PPC_RADIX_MMU
1729	void __init radix_init_pseries(void)
1730	{
1731	pr_info("Using radix MMU under hypervisor\n");
1732
1733	pseries_lpar_register_process_table(__pa(process_tb),
1734	0, PRTB_SIZE_SHIFT - 12);
1735	}
1736	#endif
1737
1738	#ifdef CONFIG_PPC_SMLPAR
1739	#define CMO_FREE_HINT_DEFAULT 1
1740	static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT;
1741
1742	static int __init cmo_free_hint(char *str)
1743	{
1744	char *parm;
1745	parm = strstrip(str);
1746
1747	if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) {
1748	pr_info("%s: CMO free page hinting is not active.\n", __func__);
1749	cmo_free_hint_flag = 0;
1750	return 1;
1751	}
1752
1753	cmo_free_hint_flag = 1;
1754	pr_info("%s: CMO free page hinting is active.\n", __func__);
1755
1756	if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0)
1757	return 1;
1758
1759	return 0;
1760	}
1761
1762	__setup("cmo_free_hint=", cmo_free_hint);
1763
1764	static void pSeries_set_page_state(struct page *page, int order,
1765	unsigned long state)
1766	{
1767	int i, j;
1768	unsigned long cmo_page_sz, addr;
1769
1770	cmo_page_sz = cmo_get_page_size();
1771	addr = __pa((unsigned long)page_address(page));
1772
1773	for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) {
1774	for (j = 0; j < PAGE_SIZE; j += cmo_page_sz)
1775	plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0);
1776	}
1777	}
1778
1779	void arch_free_page(struct page *page, int order)
1780	{
1781	if (radix_enabled())
1782	return;
1783	if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO))
1784	return;
1785
1786	pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED);
1787	}
1788	EXPORT_SYMBOL(arch_free_page);
1789
1790	#endif /* CONFIG_PPC_SMLPAR */
1791	#endif /* CONFIG_PPC_BOOK3S_64 */
1792
1793	#ifdef CONFIG_TRACEPOINTS
1794	#ifdef CONFIG_JUMP_LABEL
1795	struct static_key hcall_tracepoint_key = STATIC_KEY_INIT;
1796
1797	int hcall_tracepoint_regfunc(void)
1798	{
1799	static_key_slow_inc(key: &hcall_tracepoint_key);
1800	return 0;
1801	}
1802
1803	void hcall_tracepoint_unregfunc(void)
1804	{
1805	static_key_slow_dec(key: &hcall_tracepoint_key);
1806	}
1807	#else
1808	/*
1809	* We optimise our hcall path by placing hcall_tracepoint_refcount
1810	* directly in the TOC so we can check if the hcall tracepoints are
1811	* enabled via a single load.
1812	*/
1813
1814	/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
1815	extern long hcall_tracepoint_refcount;
1816
1817	int hcall_tracepoint_regfunc(void)
1818	{
1819	hcall_tracepoint_refcount++;
1820	return 0;
1821	}
1822
1823	void hcall_tracepoint_unregfunc(void)
1824	{
1825	hcall_tracepoint_refcount--;
1826	}
1827	#endif
1828
1829	/*
1830	* Keep track of hcall tracing depth and prevent recursion. Warn if any is
1831	* detected because it may indicate a problem. This will not catch all
1832	* problems with tracing code making hcalls, because the tracing might have
1833	* been invoked from a non-hcall, so the first hcall could recurse into it
1834	* without warning here, but this better than nothing.
1835	*
1836	* Hcalls with specific problems being traced should use the _notrace
1837	* plpar_hcall variants.
1838	*/
1839	static DEFINE_PER_CPU(unsigned int, hcall_trace_depth);
1840
1841
1842	notrace void __trace_hcall_entry(unsigned long opcode, unsigned long *args)
1843	{
1844	unsigned long flags;
1845	unsigned int *depth;
1846
1847	local_irq_save(flags);
1848
1849	depth = this_cpu_ptr(&hcall_trace_depth);
1850
1851	if (WARN_ON_ONCE(*depth))
1852	goto out;
1853
1854	(*depth)++;
1855	preempt_disable();
1856	trace_hcall_entry(opcode, args);
1857	(*depth)--;
1858
1859	out:
1860	local_irq_restore(flags);
1861	}
1862
1863	notrace void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf)
1864	{
1865	unsigned long flags;
1866	unsigned int *depth;
1867
1868	local_irq_save(flags);
1869
1870	depth = this_cpu_ptr(&hcall_trace_depth);
1871
1872	if (*depth) /* Don't warn again on the way out */
1873	goto out;
1874
1875	(*depth)++;
1876	trace_hcall_exit(opcode, retval, retbuf);
1877	preempt_enable();
1878	(*depth)--;
1879
1880	out:
1881	local_irq_restore(flags);
1882	}
1883	#endif
1884
1885	/**
1886	* h_get_mpp
1887	* H_GET_MPP hcall returns info in 7 parms
1888	*/
1889	int h_get_mpp(struct hvcall_mpp_data *mpp_data)
1890	{
1891	int rc;
1892	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];
1893
1894	rc = plpar_hcall9(H_GET_MPP, retbuf);
1895
1896	mpp_data->entitled_mem = retbuf[0];
1897	mpp_data->mapped_mem = retbuf[1];
1898
1899	mpp_data->group_num = (retbuf[2] >> 2 * 8) & 0xffff;
1900	mpp_data->pool_num = retbuf[2] & 0xffff;
1901
1902	mpp_data->mem_weight = (retbuf[3] >> 7 * 8) & 0xff;
1903	mpp_data->unallocated_mem_weight = (retbuf[3] >> 6 * 8) & 0xff;
1904	mpp_data->unallocated_entitlement = retbuf[3] & 0xffffffffffffUL;
1905
1906	mpp_data->pool_size = retbuf[4];
1907	mpp_data->loan_request = retbuf[5];
1908	mpp_data->backing_mem = retbuf[6];
1909
1910	return rc;
1911	}
1912	EXPORT_SYMBOL(h_get_mpp);
1913
1914	int h_get_mpp_x(struct hvcall_mpp_x_data *mpp_x_data)
1915	{
1916	int rc;
1917	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 };
1918
1919	rc = plpar_hcall9(H_GET_MPP_X, retbuf);
1920
1921	mpp_x_data->coalesced_bytes = retbuf[0];
1922	mpp_x_data->pool_coalesced_bytes = retbuf[1];
1923	mpp_x_data->pool_purr_cycles = retbuf[2];
1924	mpp_x_data->pool_spurr_cycles = retbuf[3];
1925
1926	return rc;
1927	}
1928
1929	#ifdef CONFIG_PPC_64S_HASH_MMU
1930	static unsigned long __init vsid_unscramble(unsigned long vsid, int ssize)
1931	{
1932	unsigned long protovsid;
1933	unsigned long va_bits = VA_BITS;
1934	unsigned long modinv, vsid_modulus;
1935	unsigned long max_mod_inv, tmp_modinv;
1936
1937	if (!mmu_has_feature(MMU_FTR_68_BIT_VA))
1938	va_bits = 65;
1939
1940	if (ssize == MMU_SEGSIZE_256M) {
1941	modinv = VSID_MULINV_256M;
1942	vsid_modulus = ((1UL << (va_bits - SID_SHIFT)) - 1);
1943	} else {
1944	modinv = VSID_MULINV_1T;
1945	vsid_modulus = ((1UL << (va_bits - SID_SHIFT_1T)) - 1);
1946	}
1947
1948	/*
1949	* vsid outside our range.
1950	*/
1951	if (vsid >= vsid_modulus)
1952	return 0;
1953
1954	/*
1955	* If modinv is the modular multiplicate inverse of (x % vsid_modulus)
1956	* and vsid = (protovsid * x) % vsid_modulus, then we say:
1957	* protovsid = (vsid * modinv) % vsid_modulus
1958	*/
1959
1960	/* Check if (vsid * modinv) overflow (63 bits) */
1961	max_mod_inv = 0x7fffffffffffffffull / vsid;
1962	if (modinv < max_mod_inv)
1963	return (vsid * modinv) % vsid_modulus;
1964
1965	tmp_modinv = modinv/max_mod_inv;
1966	modinv %= max_mod_inv;
1967
1968	protovsid = (((vsid * max_mod_inv) % vsid_modulus) * tmp_modinv) % vsid_modulus;
1969	protovsid = (protovsid + vsid * modinv) % vsid_modulus;
1970
1971	return protovsid;
1972	}
1973
1974	static int __init reserve_vrma_context_id(void)
1975	{
1976	unsigned long protovsid;
1977
1978	/*
1979	* Reserve context ids which map to reserved virtual addresses. For now
1980	* we only reserve the context id which maps to the VRMA VSID. We ignore
1981	* the addresses in "ibm,adjunct-virtual-addresses" because we don't
1982	* enable adjunct support via the "ibm,client-architecture-support"
1983	* interface.
1984	*/
1985	protovsid = vsid_unscramble(VRMA_VSID, MMU_SEGSIZE_1T);
1986	hash__reserve_context_id(protovsid >> ESID_BITS_1T);
1987	return 0;
1988	}
1989	machine_device_initcall(pseries, reserve_vrma_context_id);
1990	#endif
1991
1992	#ifdef CONFIG_DEBUG_FS
1993	/* debugfs file interface for vpa data */
1994	static ssize_t vpa_file_read(struct file *filp, char __user *buf, size_t len,
1995	loff_t *pos)
1996	{
1997	int cpu = (long)filp->private_data;
1998	struct lppaca *lppaca = &lppaca_of(cpu);
1999
2000	return simple_read_from_buffer(buf, len, pos, lppaca,
2001	sizeof(struct lppaca));
2002	}
2003
2004	static const struct file_operations vpa_fops = {
2005	.open = simple_open,
2006	.read = vpa_file_read,
2007	.llseek = default_llseek,
2008	};
2009
2010	static int __init vpa_debugfs_init(void)
2011	{
2012	char name[16];
2013	long i;
2014	struct dentry *vpa_dir;
2015
2016	if (!firmware_has_feature(FW_FEATURE_SPLPAR))
2017	return 0;
2018
2019	vpa_dir = debugfs_create_dir(name: "vpa", parent: arch_debugfs_dir);
2020
2021	/* set up the per-cpu vpa file*/
2022	for_each_possible_cpu(i) {
2023	sprintf(buf: name, fmt: "cpu-%ld", i);
2024	debugfs_create_file(name, mode: 0400, parent: vpa_dir, data: (void *)i, fops: &vpa_fops);
2025	}
2026
2027	return 0;
2028	}
2029	machine_arch_initcall(pseries, vpa_debugfs_init);
2030	#endif /* CONFIG_DEBUG_FS */
2031