diff -ru usr/src/nv/Makefile.kbuild usr/src/nv.2286310/Makefile.kbuild --- usr/src/nv/Makefile.kbuild 2008-03-16 14:13:10.000000000 -0700 +++ usr/src/nv.2286310/Makefile.kbuild 2008-03-16 14:37:47.204131496 -0700 @@ -177,6 +177,7 @@ vmap \ signal_struct \ agp_backend_acquire \ + set_pages_uc \ change_page_attr \ pci_get_class \ sysctl_max_map_count \ diff -ru usr/src/nv/conftest.sh usr/src/nv.2286310/conftest.sh --- usr/src/nv/conftest.sh 2008-03-16 14:13:10.000000000 -0700 +++ usr/src/nv.2286310/conftest.sh 2008-03-16 14:37:47.204131496 -0700 @@ -100,6 +100,32 @@ fi ;; + set_pages_uc) + # + # Determine if the set_pages_uc() function is present. + # + echo "#include + #include + void conftest_set_pages_uc(void) { + set_pages_uc(); + }" > conftest$$.c + + $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1 + rm -f conftest$$.c + + if [ -f conftest$$.o ]; then + rm -f conftest$$.o + echo "#undef NV_SET_PAGES_UC_PRESENT" >> conftest.h + return + else + echo "#ifdef NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h + echo "#undef NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h + echo "#endif" >> conftest.h + echo "#define NV_SET_PAGES_UC_PRESENT" >> conftest.h + return + fi + ;; + change_page_attr) # # Determine if the change_page_attr() function is @@ -124,7 +150,9 @@ rm -f conftest$$.o return else + echo "#ifndef NV_SET_PAGES_UC_PRESENT" >> conftest.h echo "#define NV_CHANGE_PAGE_ATTR_PRESENT" >> conftest.h + echo "#endif" >> conftest.h return fi ;; @@ -524,6 +552,8 @@ return fi + rm -f conftest$$.o + echo "#include #include irq_handler_t conftest_isr; diff -ru usr/src/nv/nv-linux.h usr/src/nv.2286310/nv-linux.h --- usr/src/nv/nv-linux.h 2008-03-16 14:13:10.000000000 -0700 +++ usr/src/nv.2286310/nv-linux.h 2008-03-16 14:37:47.204131496 -0700 @@ -871,9 +871,10 @@ #define NV_PGD_OFFSET(address, kernel, mm) \ ({ \ + struct mm_struct *__mm = (mm); \ pgd_t *__pgd; \ if (!kernel) \ - __pgd = pgd_offset(mm, address); \ + __pgd = pgd_offset(__mm, address); \ else \ __pgd = pgd_offset_k(address); \ __pgd; \ @@ -1208,21 +1209,24 @@ nv_check_pci_config_space(nv, cb); \ } +extern int nv_update_memory_types; + /* - * a BUG() is triggered on early 2.6 x86_64 kernels. the underlying - * problem actually exists on many architectures and kernels, but - * these are the only kernels that check the condition and trigger - * a BUG(). note that this is a problem of the core kernel, not an - * nvidia bug (and can still be triggered by agpgart). let's avoid - * change_page_attr on those kernels. + * Using change_page_attr() on early Linux/x86-64 2.6 kernels may + * result in a BUG() being triggered. The underlying problem + * actually exists on multiple architectures and kernels, but only + * the above check for the condition and trigger a BUG(). + * + * Note that this is a due to a bug in the Linux kernel, not an + * NVIDIA driver bug (it can also be triggered by AGPGART). + * + * We therefore need to determine at runtime if change_page_attr() + * can be used safely on these kernels. */ -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) -extern int nv_use_cpa; - -#if defined(NVCPU_X86_64) && !defined(KERNEL_2_4) && \ - (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)) -#define NV_CHANGE_PAGE_ATTR_BUG_PRESENT 1 -#endif +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) && defined(NVCPU_X86_64) && \ + !defined(KERNEL_2_4) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)) +#define NV_CHANGE_PAGE_ATTR_BUG_PRESENT #endif #if defined(NVCPU_X86) || defined(NVCPU_X86_64) @@ -1234,7 +1238,7 @@ * * We need to be careful to mask out _PAGE_NX when the host system * doesn't support this feature or when it's disabled: the kernel - * may not do this in its implementation of the change_page_attr() + * may not do this in its implementation of the change_page_attr() * interface. */ #ifndef X86_FEATURE_NX diff -ru usr/src/nv/nv-reg.h usr/src/nv.2286310/nv-reg.h --- usr/src/nv/nv-reg.h 2008-03-16 14:13:10.000000000 -0700 +++ usr/src/nv.2286310/nv-reg.h 2008-03-16 14:37:47.204131496 -0700 @@ -391,34 +391,36 @@ #define NV_REG_REMAP_LIMIT NV_REG_STRING(__NV_REMAP_LIMIT) /* - * Option: UseCPA + * Option: UpdateMemoryTypes * * Description: * - * Many kernels have a broken implementation of change_page_attr that leads - * to cache aliasing problems. x86_64 kernels between 2.6.0 and 2.6.10 will - * force a kernel BUG_ON() when this condition is encountered. For this - * reason, the NVIDIA driver is very careful about not using the CPA kernel - * interface on these kernels. - * - * Some distributions have backported this fix to kernel versions that fall - * within this version range. The NVIDIA driver attempts to automatically - * detect these fixes and reenable usage of the change_page_attr interface. - * - * Due to the serious nature of the problems that can arise from this, the - * NVIDIA driver implements a manual registry key to force usage of this API - * to be enabled or disabled. This registry key can be used to force usage - * of the API on a known fixed kernel if the NVIDIA driver fails to detect - * the kernel as fixed. This registry key can also be used to disable usage - * of the API on a bad kernel that is misdetected as a fixed kernel. - * - * The default value is '-1' (use NVIDIA driver default logic) - * A value of '0' will forcibly disable change_page_attr calls. - * A value of '1' will forcibly enable change_page_attr calls. + * Many kernels have broken implementations of the change_page_attr() + * kernel interface that may cause cache aliasing problems. Linux/x86-64 + * kernels between 2.6.0 and 2.6.10 may prompt kernel BUG()s due to + * improper accounting in the interface's large page management code, for + * example. For this reason, the NVIDIA Linux driver is very careful about + * not using the change_page_attr() kernel interface on these kernels. + * + * Due to the serious nature of the problems that can arise from bugs in + * the change_page_attr(), set_pages_{uc,wb}() and other kernel interfaces + * used to modify memory types, the NVIDIA driver implements a manual + * registry key override to allow forcibly enabling or disabling use of + * these APIs. + * + * Possible values: + * + * ~0 = use the NVIDIA driver's default logic (default) + * 0 = enable use of change_page_attr(), etc. + * 1 = disable use of change_page_attr(), etc. + * + * By default, the NVIDIA driver will attempt to auto-detect if it can + * safely use the change_page_attr() and other kernel interfaces to modify + * the memory types of kernel mappings. */ -#define __NV_USE_CPA UseCPA -#define NV_REG_USE_CPA NV_REG_STRING(__NV_USE_CPA) +#define __NV_UPDATE_MEMORY_TYPES UpdateMemoryTypes +#define NV_REG_UPDATE_MEMORY_TYPES NV_REG_STRING(__NV_UPDATE_MEMORY_TYPES) /* * Option: RegistryDwords @@ -490,7 +492,7 @@ NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_GID, 0); NV_DEFINE_REG_ENTRY(__NV_DEVICE_FILE_MODE, 0666); NV_DEFINE_REG_ENTRY(__NV_REMAP_LIMIT, 0); -NV_DEFINE_REG_ENTRY(__NV_USE_CPA, -1); +NV_DEFINE_REG_ENTRY(__NV_UPDATE_MEMORY_TYPES, ~0); NV_DEFINE_REG_ENTRY(__NV_USE_VBIOS, 1); NV_DEFINE_REG_ENTRY(__NV_RM_EDGE_INTR_CHECK, 1); @@ -535,7 +537,7 @@ NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_GID), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DEVICE_FILE_MODE), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_REMAP_LIMIT), - NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_CPA), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_UPDATE_MEMORY_TYPES), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_VBIOS), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_EDGE_INTR_CHECK), {NULL, NULL, NULL} diff -ru usr/src/nv/nv-vm.c usr/src/nv.2286310/nv-vm.c --- usr/src/nv/nv-vm.c 2008-03-16 14:13:09.000000000 -0700 +++ usr/src/nv.2286310/nv-vm.c 2008-03-16 14:37:47.204131496 -0700 @@ -43,42 +43,40 @@ } #endif -/* - * AMD Athlon processors expose a subtle bug in the Linux - * kernel, that may lead to AGP memory corruption. Recent - * kernel versions had a workaround for this problem, but - * 2.4.20 is the first kernel to address it properly. The - * page_attr API provides the means to solve the problem. - */ - static inline void nv_set_page_attrib_uncached(nv_pte_t *page_ptr) { -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) - if (nv_use_cpa) + if (nv_update_memory_types) { - struct page *page = virt_to_page(__va(page_ptr->phys_addr)); +#if defined(NV_SET_PAGES_UC_PRESENT) + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr); + set_pages_uc(page, 1); +#elif defined(NV_CHANGE_PAGE_ATTR_PRESENT) + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr); pgprot_t prot = PAGE_KERNEL_NOCACHE; #if defined(NVCPU_X86) || defined(NVCPU_X86_64) pgprot_val(prot) &= __nv_supported_pte_mask; #endif change_page_attr(page, 1, prot); - } #endif + } } static inline void nv_set_page_attrib_cached(nv_pte_t *page_ptr) { -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) - if (nv_use_cpa) + if (nv_update_memory_types) { - struct page *page = virt_to_page(__va(page_ptr->phys_addr)); +#if defined(NV_SET_PAGES_UC_PRESENT) + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr); + set_pages_wb(page, 1); +#elif defined(NV_CHANGE_PAGE_ATTR_PRESENT) + struct page *page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr); pgprot_t prot = PAGE_KERNEL; #if defined(NVCPU_X86) || defined(NVCPU_X86_64) pgprot_val(prot) &= __nv_supported_pte_mask; #endif change_page_attr(page, 1, prot); +#endif } -#endif /* NV_CHANGE_PAGE_ATTR_PRESENT */ } static inline void nv_lock_page(nv_pte_t *page_ptr) @@ -360,7 +358,8 @@ #if defined(NV_CPA_NEEDS_FLUSHING) nv_execute_on_all_cpus(cache_flush, NULL); #endif -#if defined (NVCPU_X86) || defined (NVCPU_X86_64) +#if (defined(NVCPU_X86) || defined(NVCPU_X86_64)) && \ + defined(NV_CHANGE_PAGE_ATTR_PRESENT) global_flush_tlb(); #endif nv_ext_flush_caches(); // handle other platform flushes if present @@ -662,7 +661,7 @@ address = (unsigned long)virt_addr + i * PAGE_SIZE; - pgd = NV_PGD_OFFSET(address, 1, &init_mm); + pgd = NV_PGD_OFFSET(address, 1, NULL); if (!NV_PGD_PRESENT(pgd)) goto failed; diff -ru usr/src/nv/nv.c usr/src/nv.2286310/nv.c --- usr/src/nv/nv.c 2008-03-16 14:13:09.000000000 -0700 +++ usr/src/nv.2286310/nv.c 2008-03-16 14:37:47.208131723 -0700 @@ -15,6 +15,7 @@ #include "nv_compiler.h" #include "os-agp.h" #include "nv-vm.h" +#include "nv-reg.h" #ifdef MODULE_ALIAS_CHARDEV_MAJOR MODULE_ALIAS_CHARDEV_MAJOR(NV_MAJOR_DEVICE_NUMBER); @@ -116,10 +117,7 @@ unsigned int nv_remap_limit; #endif -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) -int nv_use_cpa = 1; -#endif - +int nv_update_memory_types = 1; static int nv_mmconfig_failure_detected = 0; static void *nv_pte_t_cache = NULL; @@ -1030,30 +1028,26 @@ #endif /* defined(NV_BUILD_NV_PAT_SUPPORT) */ } - #if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) - -/* nv_verify_cpa_interface - determine if the change_page_attr bug is fixed - * in this kernel. +/* + * nv_verify_cpa_interface() - determine if the change_page_attr() large page + * management accounting bug known to exist in early Linux/x86-64 kernels + * is present in this kernel. * - * there's really not a good way to determine if change_page_attr is fixed. - * we can't really use cpa on 2.6 x86_64 kernels < 2.6.11, as if we run into - * the accounting bug, the kernel will throw a BUG. this isn't 100% accurate, - * as it doesn't throw a bug until we try to restore the caching attributes - * of the page. so if we can track down a 4M allocation, we can mark it - * uncached and see if the accounting was done correctly. - * - * this is a little ugly, but the most accurate approach to determining if - * this kernel is good. + * There's really no good way to determine if change_page_attr() is working + * correctly. We can't reliably use change_page_attr() on Linux/x86-64 2.6 + * kernels < 2.6.11: if we run into the accounting bug, the Linux kernel will + * trigger a BUG() if we attempt to restore the WB memory type of a page + * originally part of a large page. * - * why do we even bother? some distributions have back-ported the cpa fix to - * kernels < 2.6.11. we want to use change_page_attr to avoid random corruption - * and hangs, but need to make sure it's safe to do so. + * So if we can successfully allocate such a page, change its memory type to + * UC and check if the accounting was done correctly, we can determine if + * the change_page_attr() interface can be used safely. * - * return values: - * 0 - test passed, interface works - * 1 - test failed, status unclear - * -1 - test failed, interface broken + * Return values: + * 0 - test passed, the change_page_attr() interface works + * 1 - test failed, the status is unclear + * -1 - test failed, the change_page_attr() interface is broken */ static inline pte_t *check_large_page(unsigned long vaddr) @@ -1061,7 +1055,7 @@ pgd_t *pgd = NULL; pmd_t *pmd = NULL; - pgd = NV_PGD_OFFSET(vaddr, 1, &init_mm); + pgd = NV_PGD_OFFSET(vaddr, 1, NULL); if (!NV_PGD_PRESENT(pgd)) return NULL; @@ -1171,20 +1165,29 @@ return 1; } - #endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */ - -// verify that the kernel's mapping matches the requested type -// this is to protect against accidental cache aliasing problems +/* + * nv_verify_page_mappings() - verify that the kernel mapping of the specified + * page matches the specified type. This is to help detect bugs in the Linux + * kernel's change_page_attr() interface, early. + * + * This function relies on the ability to perform kernel virtul address to PFN + * translations and therefore on 'init_mm'. Unfortunately, the latter is no + * longer exported in recent Linux/x86 2.6 kernels. The export was removed at + * roughtly the same time as the set_pages_{uc,wb}() change_page_attr() + * replacement interfaces were introduced; hopefully, it will be sufficient to + * check for their presence. + */ int nv_verify_page_mappings( nv_pte_t *page_ptr, unsigned int cachetype ) { +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) || \ + (defined(NV_SET_PAGES_UC_PRESENT) && !defined(NVCPU_X86)) unsigned long retval = -1; #if defined(NVCPU_X86) || defined(NVCPU_X86_64) - struct mm_struct *mm; pgd_t *pgd = NULL; pmd_t *pmd = NULL; pte_t *pte = NULL; @@ -1192,15 +1195,12 @@ unsigned long address; static int count = 0; -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) - if (!nv_use_cpa) + if (!nv_update_memory_types) return 0; -#endif address = (unsigned long)__va(page_ptr->phys_addr); - mm = &init_mm; // always a kernel page - pgd = NV_PGD_OFFSET(address, 1, mm); + pgd = NV_PGD_OFFSET(address, 1, NULL); if (!NV_PGD_PRESENT(pgd)) { nv_printf(NV_DBG_ERRORS, "NVRM: pgd not present for addr 0x%lx\n", address); @@ -1266,8 +1266,11 @@ } failed: -#endif +#endif /* defined(NVCPU_X86) || defined(NVCPU_X86_64) */ return retval; +#else + return 0; +#endif } #if defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) @@ -1313,7 +1316,7 @@ static int __init nvidia_init_module(void) { int rc; - U032 i, count; + U032 i, count, data; nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device); nv_stack_t *sp = NULL; @@ -1485,43 +1488,42 @@ /* create /proc/driver/nvidia */ nvos_proc_create(); -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) + /* + * Give users an opportunity to disable the driver's use of + * the change_page_attr() and set_pages_{uc,wb}() kernel + * interfaces. + */ + rc = rm_read_registry_dword(sp, nv, + "NVreg", NV_REG_UPDATE_MEMORY_TYPES, &data); + if ((rc == 0) && ((int)data != ~0)) { - int data; - - // allow the user to override us with a registry key - rc = rm_read_registry_dword(sp, nv, "NVreg", "UseCPA", &data); - if ((rc == 0) && (data != -1)) - { - nv_use_cpa = data; - } + nv_update_memory_types = data; + } #if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) - else + /* + * Unless we explicitely detect that the change_page_attr() + * inteface is fixed, disable usage of the interface on + * this kernel. Notify the user of this problem using the + * driver's /proc warnings interface (read by the installer + * and the bug report script). + */ + else + { + rc = nv_verify_cpa_interface(); + if (rc < 0) { - /* - * Unless we explicitely detect that the change_page_attr() - * inteface is fixed, disable usage of the interface on - * this kernel. Notify the user of this problem using the - * driver's /proc warnings interface (read by the installer - * and the bug report script). - */ - rc = nv_verify_cpa_interface(); - if (rc < 0) - { - nv_prints(NV_DBG_ERRORS, __cpgattr_warning); - nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning); - nv_use_cpa = 0; - } - else if (rc != 0) - { - nv_prints(NV_DBG_ERRORS, __cpgattr_warning_2); - nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning_2); - nv_use_cpa = 0; - } + nv_prints(NV_DBG_ERRORS, __cpgattr_warning); + nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning); + nv_update_memory_types = 0; + } + else if (rc != 0) + { + nv_prints(NV_DBG_ERRORS, __cpgattr_warning_2); + nvos_proc_add_warning_file("change_page_attr", __cpgattr_warning_2); + nv_update_memory_types = 0; } -#endif } -#endif +#endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */ #if defined(NVCPU_X86_64) && defined(CONFIG_IA32_EMULATION) && !defined(HAVE_COMPAT_IOCTL) /* Register ioctl()'s for 32-bit clients */ @@ -3482,8 +3484,21 @@ pte_t *pte = NULL; NvU64 retval; - mm = (kern) ? &init_mm : current->mm; - if (!kern) down_read(¤t->mm->mmap_sem); + if (!kern) + { + mm = current->mm; + down_read(&mm->mmap_sem); + } + else + { +#if defined(NV_SET_PAGES_UC_PRESENT) && defined(NVCPU_X86) + /* nv_printf(NV_DBG_ERRORS, + "NVRM: can't translate KVA in nv_get_phys_address()!\n"); */ + return 0; +#else + mm = NULL; +#endif + } pgd = NV_PGD_OFFSET(address, kern, mm); if (!NV_PGD_PRESENT(pgd)) @@ -3504,22 +3519,24 @@ retval &= ~_PAGE_NX; #endif - if (!kern) up_read(¤t->mm->mmap_sem); + if (!kern) + up_read(&mm->mmap_sem); return retval; failed: - if (!kern) up_read(¤t->mm->mmap_sem); + if (!kern) + up_read(&mm->mmap_sem); return 0; } NvU64 NV_API_CALL nv_get_kern_phys_address(NvU64 address) { - // make sure this address is a kernel pointer + /* make sure this address is a kernel virtual address */ #if defined(DEBUG) && !defined(CONFIG_X86_4G) if (address < PAGE_OFFSET) { nv_printf(NV_DBG_WARNINGS, - "NVRM: user address passed to get_kern_phys_address: 0x%lx\n", + "NVRM: user address passed to get_kern_phys_address: 0x%llx!\n", address); return 0; } @@ -3534,12 +3551,12 @@ NvU64 NV_API_CALL nv_get_kern_user_address(NvU64 address) { - // make sure this address is not a kernel pointer + /* make sure this address is not a kernel virtual address */ #if defined(DEBUG) && !defined(CONFIG_X86_4G) if (address >= PAGE_OFFSET) { nv_printf(NV_DBG_WARNINGS, - "NVRM: kernel address passed to get_user_phys_address: 0x%lx\n", + "NVRM: kernel address passed to get_user_phys_address: 0x%llx!\n", address); return 0; } @@ -4316,16 +4333,12 @@ return -1; } -int NV_API_CALL nv_no_incoherent_mappings -( - void -) +int NV_API_CALL nv_no_incoherent_mappings(void) { if(nv_ext_no_incoherent_mappings() == 1) return 1; - -#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) - return 1; +#if defined(NV_CHANGE_PAGE_ATTR_PRESENT) || defined(NV_SET_PAGES_UC_PRESENT) + return (nv_update_memory_types); #else return 0; #endif diff -ru usr/src/nv/os-interface.c usr/src/nv.2286310/os-interface.c --- usr/src/nv/os-interface.c 2008-03-16 14:13:09.000000000 -0700 +++ usr/src/nv.2286310/os-interface.c 2008-03-16 14:37:47.208131723 -0700 @@ -1198,6 +1198,18 @@ { void *vaddr; + if (start == 0) + { + if (mode != NV_MEMORY_CACHED) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: os_map_kernel_space: won't map address 0x%0llx UC!\n", start); + return NULL; + } + else + return (void *)PAGE_OFFSET; + } + if (!NV_MAY_SLEEP()) { nv_printf(NV_DBG_ERRORS, @@ -1230,6 +1242,9 @@ NvU64 size_bytes ) { + if (addr == (void *)PAGE_OFFSET) + return; + NV_IOUNMAP(addr, size_bytes); }