1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Firmware replacement code. |
4 | * |
5 | * Work around broken BIOSes that don't set an aperture, only set the |
6 | * aperture in the AGP bridge, or set too small aperture. |
7 | * |
8 | * If all fails map the aperture over some low memory. This is cheaper than |
9 | * doing bounce buffering. The memory is lost. This is done at early boot |
10 | * because only the bootmem allocator can allocate 32+MB. |
11 | * |
12 | * Copyright 2002 Andi Kleen, SuSE Labs. |
13 | */ |
14 | #define pr_fmt(fmt) "AGP: " fmt |
15 | |
16 | #include <linux/kernel.h> |
17 | #include <linux/kcore.h> |
18 | #include <linux/types.h> |
19 | #include <linux/init.h> |
20 | #include <linux/memblock.h> |
21 | #include <linux/mmzone.h> |
22 | #include <linux/pci_ids.h> |
23 | #include <linux/pci.h> |
24 | #include <linux/bitops.h> |
25 | #include <linux/suspend.h> |
26 | #include <asm/e820/api.h> |
27 | #include <asm/io.h> |
28 | #include <asm/iommu.h> |
29 | #include <asm/gart.h> |
30 | #include <asm/pci-direct.h> |
31 | #include <asm/dma.h> |
32 | #include <asm/amd_nb.h> |
33 | #include <asm/x86_init.h> |
34 | #include <linux/crash_dump.h> |
35 | |
36 | /* |
37 | * Using 512M as goal, in case kexec will load kernel_big |
38 | * that will do the on-position decompress, and could overlap with |
39 | * the gart aperture that is used. |
40 | * Sequence: |
41 | * kernel_small |
42 | * ==> kexec (with kdump trigger path or gart still enabled) |
43 | * ==> kernel_small (gart area become e820_reserved) |
44 | * ==> kexec (with kdump trigger path or gart still enabled) |
45 | * ==> kerne_big (uncompressed size will be big than 64M or 128M) |
46 | * So don't use 512M below as gart iommu, leave the space for kernel |
47 | * code for safe. |
48 | */ |
49 | #define GART_MIN_ADDR (512ULL << 20) |
50 | #define GART_MAX_ADDR (1ULL << 32) |
51 | |
52 | int gart_iommu_aperture; |
53 | int gart_iommu_aperture_disabled __initdata; |
54 | int gart_iommu_aperture_allowed __initdata; |
55 | |
56 | int fallback_aper_order __initdata = 1; /* 64MB */ |
57 | int fallback_aper_force __initdata; |
58 | |
59 | int fix_aperture __initdata = 1; |
60 | |
61 | #if defined(CONFIG_PROC_VMCORE) || defined(CONFIG_PROC_KCORE) |
62 | /* |
63 | * If the first kernel maps the aperture over e820 RAM, the kdump kernel will |
64 | * use the same range because it will remain configured in the northbridge. |
65 | * Trying to dump this area via /proc/vmcore may crash the machine, so exclude |
66 | * it from vmcore. |
67 | */ |
68 | static unsigned long aperture_pfn_start, aperture_page_count; |
69 | |
70 | static int gart_mem_pfn_is_ram(unsigned long pfn) |
71 | { |
72 | return likely((pfn < aperture_pfn_start) || |
73 | (pfn >= aperture_pfn_start + aperture_page_count)); |
74 | } |
75 | |
76 | #ifdef CONFIG_PROC_VMCORE |
77 | static bool gart_oldmem_pfn_is_ram(struct vmcore_cb *cb, unsigned long pfn) |
78 | { |
79 | return !!gart_mem_pfn_is_ram(pfn); |
80 | } |
81 | |
82 | static struct vmcore_cb gart_vmcore_cb = { |
83 | .pfn_is_ram = gart_oldmem_pfn_is_ram, |
84 | }; |
85 | #endif |
86 | |
87 | static void __init exclude_from_core(u64 aper_base, u32 aper_order) |
88 | { |
89 | aperture_pfn_start = aper_base >> PAGE_SHIFT; |
90 | aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; |
91 | #ifdef CONFIG_PROC_VMCORE |
92 | register_vmcore_cb(cb: &gart_vmcore_cb); |
93 | #endif |
94 | #ifdef CONFIG_PROC_KCORE |
95 | WARN_ON(register_mem_pfn_is_ram(&gart_mem_pfn_is_ram)); |
96 | #endif |
97 | } |
98 | #else |
99 | static void exclude_from_core(u64 aper_base, u32 aper_order) |
100 | { |
101 | } |
102 | #endif |
103 | |
104 | /* This code runs before the PCI subsystem is initialized, so just |
105 | access the northbridge directly. */ |
106 | |
107 | static u32 __init allocate_aperture(void) |
108 | { |
109 | u32 aper_size; |
110 | unsigned long addr; |
111 | |
112 | /* aper_size should <= 1G */ |
113 | if (fallback_aper_order > 5) |
114 | fallback_aper_order = 5; |
115 | aper_size = (32 * 1024 * 1024) << fallback_aper_order; |
116 | |
117 | /* |
118 | * Aperture has to be naturally aligned. This means a 2GB aperture |
119 | * won't have much chance of finding a place in the lower 4GB of |
120 | * memory. Unfortunately we cannot move it up because that would |
121 | * make the IOMMU useless. |
122 | */ |
123 | addr = memblock_phys_alloc_range(size: aper_size, align: aper_size, |
124 | GART_MIN_ADDR, GART_MAX_ADDR); |
125 | if (!addr) { |
126 | pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n" , |
127 | addr, addr + aper_size - 1, aper_size >> 10); |
128 | return 0; |
129 | } |
130 | pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n" , |
131 | addr, addr + aper_size - 1, aper_size >> 10); |
132 | register_nosave_region(b: addr >> PAGE_SHIFT, |
133 | e: (addr+aper_size) >> PAGE_SHIFT); |
134 | |
135 | return (u32)addr; |
136 | } |
137 | |
138 | |
139 | /* Find a PCI capability */ |
140 | static u32 __init find_cap(int bus, int slot, int func, int cap) |
141 | { |
142 | int bytes; |
143 | u8 pos; |
144 | |
145 | if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & |
146 | PCI_STATUS_CAP_LIST)) |
147 | return 0; |
148 | |
149 | pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); |
150 | for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { |
151 | u8 id; |
152 | |
153 | pos &= ~3; |
154 | id = read_pci_config_byte(bus, slot, func, offset: pos+PCI_CAP_LIST_ID); |
155 | if (id == 0xff) |
156 | break; |
157 | if (id == cap) |
158 | return pos; |
159 | pos = read_pci_config_byte(bus, slot, func, |
160 | offset: pos+PCI_CAP_LIST_NEXT); |
161 | } |
162 | return 0; |
163 | } |
164 | |
165 | /* Read a standard AGPv3 bridge header */ |
166 | static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) |
167 | { |
168 | u32 apsize; |
169 | u32 apsizereg; |
170 | int nbits; |
171 | u32 aper_low, aper_hi; |
172 | u64 aper; |
173 | u32 old_order; |
174 | |
175 | pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n" , bus, slot, func); |
176 | apsizereg = read_pci_config_16(bus, slot, func, offset: cap + 0x14); |
177 | if (apsizereg == 0xffffffff) { |
178 | pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n" , |
179 | bus, slot, func); |
180 | return 0; |
181 | } |
182 | |
183 | /* old_order could be the value from NB gart setting */ |
184 | old_order = *order; |
185 | |
186 | apsize = apsizereg & 0xfff; |
187 | /* Some BIOS use weird encodings not in the AGPv3 table. */ |
188 | if (apsize & 0xff) |
189 | apsize |= 0xf00; |
190 | nbits = hweight16(apsize); |
191 | *order = 7 - nbits; |
192 | if ((int)*order < 0) /* < 32MB */ |
193 | *order = 0; |
194 | |
195 | aper_low = read_pci_config(bus, slot, func, offset: 0x10); |
196 | aper_hi = read_pci_config(bus, slot, func, offset: 0x14); |
197 | aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); |
198 | |
199 | /* |
200 | * On some sick chips, APSIZE is 0. It means it wants 4G |
201 | * so let double check that order, and lets trust AMD NB settings: |
202 | */ |
203 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n" , |
204 | bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, |
205 | 32 << old_order); |
206 | if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { |
207 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n" , |
208 | bus, slot, func, 32 << *order, apsizereg); |
209 | *order = old_order; |
210 | } |
211 | |
212 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n" , |
213 | bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, |
214 | 32 << *order, apsizereg); |
215 | |
216 | if (!aperture_valid(aper_base: aper, aper_size: (32*1024*1024) << *order, min_size: 32<<20)) |
217 | return 0; |
218 | return (u32)aper; |
219 | } |
220 | |
221 | /* |
222 | * Look for an AGP bridge. Windows only expects the aperture in the |
223 | * AGP bridge and some BIOS forget to initialize the Northbridge too. |
224 | * Work around this here. |
225 | * |
226 | * Do an PCI bus scan by hand because we're running before the PCI |
227 | * subsystem. |
228 | * |
229 | * All AMD AGP bridges are AGPv3 compliant, so we can do this scan |
230 | * generically. It's probably overkill to always scan all slots because |
231 | * the AGP bridges should be always an own bus on the HT hierarchy, |
232 | * but do it here for future safety. |
233 | */ |
234 | static u32 __init search_agp_bridge(u32 *order, int *valid_agp) |
235 | { |
236 | int bus, slot, func; |
237 | |
238 | /* Poor man's PCI discovery */ |
239 | for (bus = 0; bus < 256; bus++) { |
240 | for (slot = 0; slot < 32; slot++) { |
241 | for (func = 0; func < 8; func++) { |
242 | u32 class, cap; |
243 | u8 type; |
244 | class = read_pci_config(bus, slot, func, |
245 | PCI_CLASS_REVISION); |
246 | if (class == 0xffffffff) |
247 | break; |
248 | |
249 | switch (class >> 16) { |
250 | case PCI_CLASS_BRIDGE_HOST: |
251 | case PCI_CLASS_BRIDGE_OTHER: /* needed? */ |
252 | /* AGP bridge? */ |
253 | cap = find_cap(bus, slot, func, |
254 | PCI_CAP_ID_AGP); |
255 | if (!cap) |
256 | break; |
257 | *valid_agp = 1; |
258 | return read_agp(bus, slot, func, cap, |
259 | order); |
260 | } |
261 | |
262 | /* No multi-function device? */ |
263 | type = read_pci_config_byte(bus, slot, func, |
264 | PCI_HEADER_TYPE); |
265 | if (!(type & 0x80)) |
266 | break; |
267 | } |
268 | } |
269 | } |
270 | pr_info("No AGP bridge found\n" ); |
271 | |
272 | return 0; |
273 | } |
274 | |
275 | static bool gart_fix_e820 __initdata = true; |
276 | |
277 | static int __init parse_gart_mem(char *p) |
278 | { |
279 | return kstrtobool(s: p, res: &gart_fix_e820); |
280 | } |
281 | early_param("gart_fix_e820" , parse_gart_mem); |
282 | |
283 | /* |
284 | * With kexec/kdump, if the first kernel doesn't shut down the GART and the |
285 | * second kernel allocates a different GART region, there might be two |
286 | * overlapping GART regions present: |
287 | * |
288 | * - the first still used by the GART initialized in the first kernel. |
289 | * - (sub-)set of it used as normal RAM by the second kernel. |
290 | * |
291 | * which leads to memory corruptions and a kernel panic eventually. |
292 | * |
293 | * This can also happen if the BIOS has forgotten to mark the GART region |
294 | * as reserved. |
295 | * |
296 | * Try to update the e820 map to mark that new region as reserved. |
297 | */ |
298 | void __init early_gart_iommu_check(void) |
299 | { |
300 | u32 agp_aper_order = 0; |
301 | int i, fix, slot, valid_agp = 0; |
302 | u32 ctl; |
303 | u32 aper_size = 0, aper_order = 0, last_aper_order = 0; |
304 | u64 aper_base = 0, last_aper_base = 0; |
305 | int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; |
306 | |
307 | if (!amd_gart_present()) |
308 | return; |
309 | |
310 | if (!early_pci_allowed()) |
311 | return; |
312 | |
313 | /* This is mostly duplicate of iommu_hole_init */ |
314 | search_agp_bridge(order: &agp_aper_order, valid_agp: &valid_agp); |
315 | |
316 | fix = 0; |
317 | for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
318 | int bus; |
319 | int dev_base, dev_limit; |
320 | |
321 | bus = amd_nb_bus_dev_ranges[i].bus; |
322 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
323 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
324 | |
325 | for (slot = dev_base; slot < dev_limit; slot++) { |
326 | if (!early_is_amd_nb(value: read_pci_config(bus, slot, func: 3, offset: 0x00))) |
327 | continue; |
328 | |
329 | ctl = read_pci_config(bus, slot, func: 3, AMD64_GARTAPERTURECTL); |
330 | aper_enabled = ctl & GARTEN; |
331 | aper_order = (ctl >> 1) & 7; |
332 | aper_size = (32 * 1024 * 1024) << aper_order; |
333 | aper_base = read_pci_config(bus, slot, func: 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
334 | aper_base <<= 25; |
335 | |
336 | if (last_valid) { |
337 | if ((aper_order != last_aper_order) || |
338 | (aper_base != last_aper_base) || |
339 | (aper_enabled != last_aper_enabled)) { |
340 | fix = 1; |
341 | break; |
342 | } |
343 | } |
344 | |
345 | last_aper_order = aper_order; |
346 | last_aper_base = aper_base; |
347 | last_aper_enabled = aper_enabled; |
348 | last_valid = 1; |
349 | } |
350 | } |
351 | |
352 | if (!fix && !aper_enabled) |
353 | return; |
354 | |
355 | if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) |
356 | fix = 1; |
357 | |
358 | if (gart_fix_e820 && !fix && aper_enabled) { |
359 | if (e820__mapped_any(start: aper_base, end: aper_base + aper_size, |
360 | type: E820_TYPE_RAM)) { |
361 | /* reserve it, so we can reuse it in second kernel */ |
362 | pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n" , |
363 | aper_base, aper_base + aper_size - 1); |
364 | e820__range_add(start: aper_base, size: aper_size, type: E820_TYPE_RESERVED); |
365 | e820__update_table_print(); |
366 | } |
367 | } |
368 | |
369 | if (valid_agp) |
370 | return; |
371 | |
372 | /* disable them all at first */ |
373 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
374 | int bus; |
375 | int dev_base, dev_limit; |
376 | |
377 | bus = amd_nb_bus_dev_ranges[i].bus; |
378 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
379 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
380 | |
381 | for (slot = dev_base; slot < dev_limit; slot++) { |
382 | if (!early_is_amd_nb(value: read_pci_config(bus, slot, func: 3, offset: 0x00))) |
383 | continue; |
384 | |
385 | ctl = read_pci_config(bus, slot, func: 3, AMD64_GARTAPERTURECTL); |
386 | ctl &= ~GARTEN; |
387 | write_pci_config(bus, slot, func: 3, AMD64_GARTAPERTURECTL, val: ctl); |
388 | } |
389 | } |
390 | |
391 | } |
392 | |
393 | static int __initdata printed_gart_size_msg; |
394 | |
395 | void __init gart_iommu_hole_init(void) |
396 | { |
397 | u32 agp_aper_base = 0, agp_aper_order = 0; |
398 | u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; |
399 | u64 aper_base, last_aper_base = 0; |
400 | int fix, slot, valid_agp = 0; |
401 | int i, node; |
402 | |
403 | if (!amd_gart_present()) |
404 | return; |
405 | |
406 | if (gart_iommu_aperture_disabled || !fix_aperture || |
407 | !early_pci_allowed()) |
408 | return; |
409 | |
410 | pr_info("Checking aperture...\n" ); |
411 | |
412 | if (!fallback_aper_force) |
413 | agp_aper_base = search_agp_bridge(order: &agp_aper_order, valid_agp: &valid_agp); |
414 | |
415 | fix = 0; |
416 | node = 0; |
417 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
418 | int bus; |
419 | int dev_base, dev_limit; |
420 | u32 ctl; |
421 | |
422 | bus = amd_nb_bus_dev_ranges[i].bus; |
423 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
424 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
425 | |
426 | for (slot = dev_base; slot < dev_limit; slot++) { |
427 | if (!early_is_amd_nb(value: read_pci_config(bus, slot, func: 3, offset: 0x00))) |
428 | continue; |
429 | |
430 | iommu_detected = 1; |
431 | gart_iommu_aperture = 1; |
432 | x86_init.iommu.iommu_init = gart_iommu_init; |
433 | |
434 | ctl = read_pci_config(bus, slot, func: 3, |
435 | AMD64_GARTAPERTURECTL); |
436 | |
437 | /* |
438 | * Before we do anything else disable the GART. It may |
439 | * still be enabled if we boot into a crash-kernel here. |
440 | * Reconfiguring the GART while it is enabled could have |
441 | * unknown side-effects. |
442 | */ |
443 | ctl &= ~GARTEN; |
444 | write_pci_config(bus, slot, func: 3, AMD64_GARTAPERTURECTL, val: ctl); |
445 | |
446 | aper_order = (ctl >> 1) & 7; |
447 | aper_size = (32 * 1024 * 1024) << aper_order; |
448 | aper_base = read_pci_config(bus, slot, func: 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
449 | aper_base <<= 25; |
450 | |
451 | pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n" , |
452 | node, aper_base, aper_base + aper_size - 1, |
453 | aper_size >> 20); |
454 | node++; |
455 | |
456 | if (!aperture_valid(aper_base, aper_size, min_size: 64<<20)) { |
457 | if (valid_agp && agp_aper_base && |
458 | agp_aper_base == aper_base && |
459 | agp_aper_order == aper_order) { |
460 | /* the same between two setting from NB and agp */ |
461 | if (!no_iommu && |
462 | max_pfn > MAX_DMA32_PFN && |
463 | !printed_gart_size_msg) { |
464 | pr_err("you are using iommu with agp, but GART size is less than 64MB\n" ); |
465 | pr_err("please increase GART size in your BIOS setup\n" ); |
466 | pr_err("if BIOS doesn't have that option, contact your HW vendor!\n" ); |
467 | printed_gart_size_msg = 1; |
468 | } |
469 | } else { |
470 | fix = 1; |
471 | goto out; |
472 | } |
473 | } |
474 | |
475 | if ((last_aper_order && aper_order != last_aper_order) || |
476 | (last_aper_base && aper_base != last_aper_base)) { |
477 | fix = 1; |
478 | goto out; |
479 | } |
480 | last_aper_order = aper_order; |
481 | last_aper_base = aper_base; |
482 | } |
483 | } |
484 | |
485 | out: |
486 | if (!fix && !fallback_aper_force) { |
487 | if (last_aper_base) { |
488 | /* |
489 | * If this is the kdump kernel, the first kernel |
490 | * may have allocated the range over its e820 RAM |
491 | * and fixed up the northbridge |
492 | */ |
493 | exclude_from_core(aper_base: last_aper_base, aper_order: last_aper_order); |
494 | } |
495 | return; |
496 | } |
497 | |
498 | if (!fallback_aper_force) { |
499 | aper_alloc = agp_aper_base; |
500 | aper_order = agp_aper_order; |
501 | } |
502 | |
503 | if (aper_alloc) { |
504 | /* Got the aperture from the AGP bridge */ |
505 | } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || |
506 | force_iommu || |
507 | valid_agp || |
508 | fallback_aper_force) { |
509 | pr_info("Your BIOS doesn't leave an aperture memory hole\n" ); |
510 | pr_info("Please enable the IOMMU option in the BIOS setup\n" ); |
511 | pr_info("This costs you %dMB of RAM\n" , |
512 | 32 << fallback_aper_order); |
513 | |
514 | aper_order = fallback_aper_order; |
515 | aper_alloc = allocate_aperture(); |
516 | if (!aper_alloc) { |
517 | /* |
518 | * Could disable AGP and IOMMU here, but it's |
519 | * probably not worth it. But the later users |
520 | * cannot deal with bad apertures and turning |
521 | * on the aperture over memory causes very |
522 | * strange problems, so it's better to panic |
523 | * early. |
524 | */ |
525 | panic(fmt: "Not enough memory for aperture" ); |
526 | } |
527 | } else { |
528 | return; |
529 | } |
530 | |
531 | /* |
532 | * If this is the kdump kernel _and_ the first kernel did not |
533 | * configure the aperture in the northbridge, this range may |
534 | * overlap with the first kernel's memory. We can't access the |
535 | * range through vmcore even though it should be part of the dump. |
536 | */ |
537 | exclude_from_core(aper_base: aper_alloc, aper_order); |
538 | |
539 | /* Fix up the north bridges */ |
540 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
541 | int bus, dev_base, dev_limit; |
542 | |
543 | /* |
544 | * Don't enable translation yet but enable GART IO and CPU |
545 | * accesses and set DISTLBWALKPRB since GART table memory is UC. |
546 | */ |
547 | u32 ctl = aper_order << 1; |
548 | |
549 | bus = amd_nb_bus_dev_ranges[i].bus; |
550 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
551 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
552 | for (slot = dev_base; slot < dev_limit; slot++) { |
553 | if (!early_is_amd_nb(value: read_pci_config(bus, slot, func: 3, offset: 0x00))) |
554 | continue; |
555 | |
556 | write_pci_config(bus, slot, func: 3, AMD64_GARTAPERTURECTL, val: ctl); |
557 | write_pci_config(bus, slot, func: 3, AMD64_GARTAPERTUREBASE, val: aper_alloc >> 25); |
558 | } |
559 | } |
560 | |
561 | set_up_gart_resume(aper_order, aper_alloc); |
562 | } |
563 | |