1 diff -ru usr/src/nv/Makefile.kbuild usr/src/nv.2286310/Makefile.kbuild
2 --- usr/src/nv/Makefile.kbuild 2008-03-16 14:13:10.000000000 -0700
3 +++ usr/src/nv.2286310/Makefile.kbuild 2008-03-16 14:37:47.204131496 -0700
11 sysctl_max_map_count \
12 diff -ru usr/src/nv/conftest.sh usr/src/nv.2286310/conftest.sh
13 --- usr/src/nv/conftest.sh 2008-03-16 14:13:10.000000000 -0700
14 +++ usr/src/nv.2286310/conftest.sh 2008-03-16 14:37:47.204131496 -0700
21 + # Determine if the set_pages_uc() function is present.
23 + echo "#include <linux/autoconf.h>
24 + #include <asm/cacheflush.h>
25 + void conftest_set_pages_uc(void) {
29 + $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
32 + if [ -f conftest$$.o ]; then
34 + echo "#undef NV_SET_PAGES_UC_PRESENT" >> conftest.h
37 + echo "#ifdef NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h
38 + echo "#undef NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h
39 + echo "#endif" >> conftest.h
40 + echo "#define NV_SET_PAGES_UC_PRESENT" >> conftest.h
47 # Determine if the change_page_attr() function is
52 + echo "#ifndef NV_SET_PAGES_UC_PRESENT" >> conftest.h
53 echo "#define NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h
54 + echo "#endif" >> conftest.h
64 echo "#include <linux/autoconf.h>
65 #include <linux/interrupt.h>
66 irq_handler_t conftest_isr;
67 diff -ru usr/src/nv/nv-linux.h usr/src/nv.2286310/nv-linux.h
68 --- usr/src/nv/nv-linux.h 2008-03-16 14:13:10.000000000 -0700
69 +++ usr/src/nv.2286310/nv-linux.h 2008-03-16 14:37:47.204131496 -0700
72 #define NV_PGD_OFFSET(address, kernel, mm) \
74 + struct mm_struct *__mm = (mm); \
77 - __pgd = pgd_offset(mm, address); \
78 + __pgd = pgd_offset(__mm, address); \
80 __pgd = pgd_offset_k(address); \
82 @@ -1208,21 +1209,24 @@
83 nv_check_pci_config_space(nv, cb); \
86 +extern int nv_update_memory_types;
89 - * a BUG() is triggered on early 2.6 x86_64 kernels. the underlying
90 - * problem actually exists on many architectures and kernels, but
91 - * these are the only kernels that check the condition and trigger
92 - * a BUG(). note that this is a problem of the core kernel, not an
93 - * nvidia bug (and can still be triggered by agpgart). let's avoid
94 - * change_page_attr on those kernels.
95 + * Using change_page_attr() on early Linux/x86-64 2.6 kernels may
96 + * result in a BUG() being triggered. The underlying problem
97 + * actually exists on multiple architectures and kernels, but only
98 + * the above check for the condition and trigger a BUG().
100 + * Note that this is a due to a bug in the Linux kernel, not an
101 + * NVIDIA driver bug (it can also be triggered by AGPGART).
103 + * We therefore need to determine at runtime if change_page_attr()
104 + * can be used safely on these kernels.
106 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
107 -extern int nv_use_cpa;
109 -#if defined(NVCPU_X86_64) && !defined(KERNEL_2_4) && \
110 - (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
111 -#define NV_CHANGE_PAGE_ATTR_BUG_PRESENT 1
113 +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) && defined(NVCPU_X86_64) && \
114 + !defined(KERNEL_2_4) && \
115 + (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
116 +#define NV_CHANGE_PAGE_ATTR_BUG_PRESENT
119 #if defined(NVCPU_X86) || defined(NVCPU_X86_64)
120 @@ -1234,7 +1238,7 @@
122 * We need to be careful to mask out _PAGE_NX when the host system
123 * doesn't support this feature or when it's disabled: the kernel
124 - * may not do this in its implementation of the change_page_attr()
125 + * may not do this in its implementation of the change_page_attr()
128 #ifndef X86_FEATURE_NX
129 diff -ru usr/src/nv/nv-reg.h usr/src/nv.2286310/nv-reg.h
130 --- usr/src/nv/nv-reg.h 2008-03-16 14:13:10.000000000 -0700
131 +++ usr/src/nv.2286310/nv-reg.h 2008-03-16 14:37:47.204131496 -0700
132 @@ -391,34 +391,36 @@
133 #define NV_REG_REMAP_LIMIT NV_REG_STRING(__NV_REMAP_LIMIT)
137 + * Option: UpdateMemoryTypes
141 - * Many kernels have a broken implementation of change_page_attr that leads
142 - * to cache aliasing problems. x86_64 kernels between 2.6.0 and 2.6.10 will
143 - * force a kernel BUG_ON() when this condition is encountered. For this
144 - * reason, the NVIDIA driver is very careful about not using the CPA kernel
145 - * interface on these kernels.
147 - * Some distributions have backported this fix to kernel versions that fall
148 - * within this version range. The NVIDIA driver attempts to automatically
149 - * detect these fixes and reenable usage of the change_page_attr interface.
151 - * Due to the serious nature of the problems that can arise from this, the
152 - * NVIDIA driver implements a manual registry key to force usage of this API
153 - * to be enabled or disabled. This registry key can be used to force usage
154 - * of the API on a known fixed kernel if the NVIDIA driver fails to detect
155 - * the kernel as fixed. This registry key can also be used to disable usage
156 - * of the API on a bad kernel that is misdetected as a fixed kernel.
158 - * The default value is '-1' (use NVIDIA driver default logic)
159 - * A value of '0' will forcibly disable change_page_attr calls.
160 - * A value of '1' will forcibly enable change_page_attr calls.
161 + * Many kernels have broken implementations of the change_page_attr()
162 + * kernel interface that may cause cache aliasing problems. Linux/x86-64
163 + * kernels between 2.6.0 and 2.6.10 may prompt kernel BUG()s due to
164 + * improper accounting in the interface's large page management code, for
165 + * example. For this reason, the NVIDIA Linux driver is very careful about
166 + * not using the change_page_attr() kernel interface on these kernels.
168 + * Due to the serious nature of the problems that can arise from bugs in
169 + * the change_page_attr(), set_pages_{uc,wb}() and other kernel interfaces
170 + * used to modify memory types, the NVIDIA driver implements a manual
171 + * registry key override to allow forcibly enabling or disabling use of
176 + * ~0 = use the NVIDIA driver's default logic (default)
177 + * 0 = enable use of change_page_attr(), etc.
178 + * 1 = disable use of change_page_attr(), etc.
180 + * By default, the NVIDIA driver will attempt to auto-detect if it can
181 + * safely use the change_page_attr() and other kernel interfaces to modify
182 + * the memory types of kernel mappings.
185 -#define __NV_USE_CPA UseCPA
186 -#define NV_REG_USE_CPA NV_REG_STRING(__NV_USE_CPA)
187 +#define __NV_UPDATE_MEMORY_TYPES UpdateMemoryTypes
188 +#define NV_REG_UPDATE_MEMORY_TYPES NV_REG_STRING(__NV_UPDATE_MEMORY_TYPES)
191 * Option: RegistryDwords
193 NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_GID, 0);
194 NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_MODE, 0666);
195 NV_DEFINE_REG_ENTRY(__NV_REMAP_LIMIT, 0);
196 -NV_DEFINE_REG_ENTRY(__NV_USE_CPA, -1);
197 +NV_DEFINE_REG_ENTRY(__NV_UPDATE_MEMORY_TYPES, ~0);
198 NV_DEFINE_REG_ENTRY(__NV_USE_VBIOS, 1);
199 NV_DEFINE_REG_ENTRY(__NV_RM_EDGE_INTR_CHECK, 1);
202 NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_GID),
203 NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_MODE),
204 NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_REMAP_LIMIT),
205 - NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_CPA),
206 + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_UPDATE_MEMORY_TYPES),
207 NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_VBIOS),
208 NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_EDGE_INTR_CHECK),
210 diff -ru usr/src/nv/nv-vm.c usr/src/nv.2286310/nv-vm.c
211 --- usr/src/nv/nv-vm.c 2008-03-16 14:13:09.000000000 -0700
212 +++ usr/src/nv.2286310/nv-vm.c 2008-03-16 14:37:47.204131496 -0700
218 - * AMD Athlon processors expose a subtle bug in the Linux
219 - * kernel, that may lead to AGP memory corruption. Recent
220 - * kernel versions had a workaround for this problem, but
221 - * 2.4.20 is the first kernel to address it properly. The
222 - * page_attr API provides the means to solve the problem.
225 static inline void nv_set_page_attrib_uncached(nv_pte_t *page_ptr)
227 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
229 + if (nv_update_memory_types)
231 - struct page *page = virt_to_page(__va(page_ptr->phys_addr));
232 +#if defined(NV_SET_PAGES_UC_PRESENT)
233 + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr);
234 + set_pages_uc(page, 1);
235 +#elif defined(NV_CHANGE_PAGE_ATTR_PRESENT)
236 + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr);
237 pgprot_t prot = PAGE_KERNEL_NOCACHE;
238 #if defined(NVCPU_X86) || defined(NVCPU_X86_64)
239 pgprot_val(prot) &= __nv_supported_pte_mask;
241 change_page_attr(page, 1, prot);
247 static inline void nv_set_page_attrib_cached(nv_pte_t *page_ptr)
249 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
251 + if (nv_update_memory_types)
253 - struct page *page = virt_to_page(__va(page_ptr->phys_addr));
254 +#if defined(NV_SET_PAGES_UC_PRESENT)
255 + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr);
256 + set_pages_wb(page, 1);
257 +#elif defined(NV_CHANGE_PAGE_ATTR_PRESENT)
258 + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr);
259 pgprot_t prot = PAGE_KERNEL;
260 #if defined(NVCPU_X86) || defined(NVCPU_X86_64)
261 pgprot_val(prot) &= __nv_supported_pte_mask;
263 change_page_attr(page, 1, prot);
266 -#endif /* NV_CHANGE_PAGE_ATTR_PRESENT */
269 static inline void nv_lock_page(nv_pte_t *page_ptr)
271 #if defined(NV_CPA_NEEDS_FLUSHING)
272 nv_execute_on_all_cpus(cache_flush, NULL);
274 -#if defined (NVCPU_X86) || defined (NVCPU_X86_64)
275 +#if (defined(NVCPU_X86) || defined(NVCPU_X86_64)) && \
276 + defined(NV_CHANGE_PAGE_ATTR_PRESENT)
279 nv_ext_flush_caches(); // handle other platform flushes if present
282 address = (unsigned long)virt_addr + i * PAGE_SIZE;
284 - pgd = NV_PGD_OFFSET(address, 1, &init_mm);
285 + pgd = NV_PGD_OFFSET(address, 1, NULL);
286 if (!NV_PGD_PRESENT(pgd))
289 diff -ru usr/src/nv/nv.c usr/src/nv.2286310/nv.c
290 --- usr/src/nv/nv.c 2008-03-16 14:13:09.000000000 -0700
291 +++ usr/src/nv.2286310/nv.c 2008-03-16 14:37:47.208131723 -0700
293 #include "nv_compiler.h"
298 #ifdef MODULE_ALIAS_CHARDEV_MAJOR
299 MODULE_ALIAS_CHARDEV_MAJOR(NV_MAJOR_DEVICE_NUMBER);
301 unsigned int nv_remap_limit;
304 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
308 +int nv_update_memory_types = 1;
309 static int nv_mmconfig_failure_detected = 0;
311 static void *nv_pte_t_cache = NULL;
312 @@ -1030,30 +1028,26 @@
313 #endif /* defined(NV_BUILD_NV_PAT_SUPPORT) */
317 #if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT)
319 -/* nv_verify_cpa_interface - determine if the change_page_attr bug is fixed
322 + * nv_verify_cpa_interface() - determine if the change_page_attr() large page
323 + * management accounting bug known to exist in early Linux/x86-64 kernels
324 + * is present in this kernel.
326 - * there's really not a good way to determine if change_page_attr is fixed.
327 - * we can't really use cpa on 2.6 x86_64 kernels < 2.6.11, as if we run into
328 - * the accounting bug, the kernel will throw a BUG. this isn't 100% accurate,
329 - * as it doesn't throw a bug until we try to restore the caching attributes
330 - * of the page. so if we can track down a 4M allocation, we can mark it
331 - * uncached and see if the accounting was done correctly.
333 - * this is a little ugly, but the most accurate approach to determining if
334 - * this kernel is good.
335 + * There's really no good way to determine if change_page_attr() is working
336 + * correctly. We can't reliably use change_page_attr() on Linux/x86-64 2.6
337 + * kernels < 2.6.11: if we run into the accounting bug, the Linux kernel will
338 + * trigger a BUG() if we attempt to restore the WB memory type of a page
339 + * originally part of a large page.
341 - * why do we even bother? some distributions have back-ported the cpa fix to
342 - * kernels < 2.6.11. we want to use change_page_attr to avoid random corruption
343 - * and hangs, but need to make sure it's safe to do so.
344 + * So if we can successfully allocate such a page, change its memory type to
345 + * UC and check if the accounting was done correctly, we can determine if
346 + * the change_page_attr() interface can be used safely.
349 - * 0 - test passed, interface works
350 - * 1 - test failed, status unclear
351 - * -1 - test failed, interface broken
353 + * 0 - test passed, the change_page_attr() interface works
354 + * 1 - test failed, the status is unclear
355 + * -1 - test failed, the change_page_attr() interface is broken
358 static inline pte_t *check_large_page(unsigned long vaddr)
359 @@ -1061,7 +1055,7 @@
363 - pgd = NV_PGD_OFFSET(vaddr, 1, &init_mm);
364 + pgd = NV_PGD_OFFSET(vaddr, 1, NULL);
365 if (!NV_PGD_PRESENT(pgd))
368 @@ -1171,20 +1165,29 @@
373 #endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */
376 -// verify that the kernel's mapping matches the requested type
377 -// this is to protect against accidental cache aliasing problems
379 + * nv_verify_page_mappings() - verify that the kernel mapping of the specified
380 + * page matches the specified type. This is to help detect bugs in the Linux
381 + * kernel's change_page_attr() interface, early.
383 + * This function relies on the ability to perform kernel virtul address to PFN
384 + * translations and therefore on 'init_mm'. Unfortunately, the latter is no
385 + * longer exported in recent Linux/x86 2.6 kernels. The export was removed at
386 + * roughtly the same time as the set_pages_{uc,wb}() change_page_attr()
387 + * replacement interfaces were introduced; hopefully, it will be sufficient to
388 + * check for their presence.
390 int nv_verify_page_mappings(
392 unsigned int cachetype
395 +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) || \
396 + (defined(NV_SET_PAGES_UC_PRESENT) && !defined(NVCPU_X86))
397 unsigned long retval = -1;
398 #if defined(NVCPU_X86) || defined(NVCPU_X86_64)
399 - struct mm_struct *mm;
403 @@ -1192,15 +1195,12 @@
404 unsigned long address;
405 static int count = 0;
407 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
409 + if (!nv_update_memory_types)
413 address = (unsigned long)__va(page_ptr->phys_addr);
414 - mm = &init_mm; // always a kernel page
416 - pgd = NV_PGD_OFFSET(address, 1, mm);
417 + pgd = NV_PGD_OFFSET(address, 1, NULL);
418 if (!NV_PGD_PRESENT(pgd))
420 nv_printf(NV_DBG_ERRORS, "NVRM: pgd not present for addr 0x%lx\n", address);
421 @@ -1266,8 +1266,11 @@
426 +#endif /* defined(NVCPU_X86) || defined(NVCPU_X86_64) */
433 #if defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU)
434 @@ -1313,7 +1316,7 @@
435 static int __init nvidia_init_module(void)
439 + U032 i, count, data;
440 nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
441 nv_stack_t *sp = NULL;
443 @@ -1485,43 +1488,42 @@
444 /* create /proc/driver/nvidia */
447 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
449 + * Give users an opportunity to disable the driver's use of
450 + * the change_page_attr() and set_pages_{uc,wb}() kernel
453 + rc = rm_read_registry_dword(sp, nv,
454 + "NVreg", NV_REG_UPDATE_MEMORY_TYPES, &data);
455 + if ((rc == 0) && ((int)data != ~0))
459 - // allow the user to override us with a registry key
460 - rc = rm_read_registry_dword(sp, nv, "NVreg", "UseCPA", &data);
461 - if ((rc == 0) && (data != -1))
465 + nv_update_memory_types = data;
467 #if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT)
470 + * Unless we explicitely detect that the change_page_attr()
471 + * inteface is fixed, disable usage of the interface on
472 + * this kernel. Notify the user of this problem using the
473 + * driver's /proc warnings interface (read by the installer
474 + * and the bug report script).
478 + rc = nv_verify_cpa_interface();
482 - * Unless we explicitely detect that the change_page_attr()
483 - * inteface is fixed, disable usage of the interface on
484 - * this kernel. Notify the user of this problem using the
485 - * driver's /proc warnings interface (read by the installer
486 - * and the bug report script).
488 - rc = nv_verify_cpa_interface();
491 - nv_prints(NV_DBG_ERRORS, __cpgattr_warning);
492 - nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning);
497 - nv_prints(NV_DBG_ERRORS, __cpgattr_warning_2);
498 - nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning_2);
501 + nv_prints(NV_DBG_ERRORS, __cpgattr_warning);
502 + nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning);
503 + nv_update_memory_types = 0;
507 + nv_prints(NV_DBG_ERRORS, __cpgattr_warning_2);
508 + nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning_2);
509 + nv_update_memory_types = 0;
514 +#endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */
516 #if defined(NVCPU_X86_64) && defined(CONFIG_IA32_EMULATION) && !defined(HAVE_COMPAT_IOCTL)
517 /* Register ioctl()'s for 32-bit clients */
518 @@ -3482,8 +3484,21 @@
522 - mm = (kern) ? &init_mm : current->mm;
523 - if (!kern) down_read(¤t->mm->mmap_sem);
527 + down_read(&mm->mmap_sem);
531 +#if defined(NV_SET_PAGES_UC_PRESENT) && defined(NVCPU_X86)
532 + /* nv_printf(NV_DBG_ERRORS,
533 + "NVRM: can't translate KVA in nv_get_phys_address()!\n"); */
540 pgd = NV_PGD_OFFSET(address, kern, mm);
541 if (!NV_PGD_PRESENT(pgd))
542 @@ -3504,22 +3519,24 @@
546 - if (!kern) up_read(¤t->mm->mmap_sem);
548 + up_read(&mm->mmap_sem);
552 - if (!kern) up_read(¤t->mm->mmap_sem);
554 + up_read(&mm->mmap_sem);
558 NvU64 NV_API_CALL nv_get_kern_phys_address(NvU64 address)
560 - // make sure this address is a kernel pointer
561 + /* make sure this address is a kernel virtual address */
562 #if defined(DEBUG) && !defined(CONFIG_X86_4G)
563 if (address < PAGE_OFFSET)
565 nv_printf(NV_DBG_WARNINGS,
566 - "NVRM: user address passed to get_kern_phys_address: 0x%lx\n",
567 + "NVRM: user address passed to get_kern_phys_address: 0x%llx!\n",
571 @@ -3534,12 +3551,12 @@
573 NvU64 NV_API_CALL nv_get_kern_user_address(NvU64 address)
575 - // make sure this address is not a kernel pointer
576 + /* make sure this address is not a kernel virtual address */
577 #if defined(DEBUG) && !defined(CONFIG_X86_4G)
578 if (address >= PAGE_OFFSET)
580 nv_printf(NV_DBG_WARNINGS,
581 - "NVRM: kernel address passed to get_user_phys_address: 0x%lx\n",
582 + "NVRM: kernel address passed to get_user_phys_address: 0x%llx!\n",
586 @@ -4316,16 +4333,12 @@
590 -int NV_API_CALL nv_no_incoherent_mappings
594 +int NV_API_CALL nv_no_incoherent_mappings(void)
596 if(nv_ext_no_incoherent_mappings() == 1)
599 -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT)
601 +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) || defined(NV_SET_PAGES_UC_PRESENT)
602 + return (nv_update_memory_types);
606 diff -ru usr/src/nv/os-interface.c usr/src/nv.2286310/os-interface.c
607 --- usr/src/nv/os-interface.c 2008-03-16 14:13:09.000000000 -0700
608 +++ usr/src/nv.2286310/os-interface.c 2008-03-16 14:37:47.208131723 -0700
609 @@ -1198,6 +1198,18 @@
615 + if (mode != NV_MEMORY_CACHED)
617 + nv_printf(NV_DBG_ERRORS,
618 + "NVRM: os_map_kernel_space: won't map address 0x%0llx UC!\n", start);
622 + return (void *)PAGE_OFFSET;
627 nv_printf(NV_DBG_ERRORS,
628 @@ -1230,6 +1242,9 @@
632 + if (addr == (void *)PAGE_OFFSET)
635 NV_IOUNMAP(addr, size_bytes);