diff -ru usr/src/nv/Makefile.kbuild usr/src/nv.2419292/Makefile.kbuild --- usr/src/nv/Makefile.kbuild 2008-05-19 00:37:02.000000000 -0700 +++ usr/src/nv.2419292/Makefile.kbuild 2008-05-21 12:07:57.551201274 -0700 @@ -166,10 +166,6 @@ CFLAGS += -DNV_MAP_REGISTERS_EARLY endif -ifeq ($(shell echo $(NV_BUILD_NV_PAT_SUPPORT)),1) -CFLAGS += -DNV_BUILD_NV_PAT_SUPPORT -endif - ifneq ($(PATCHLEVEL), 4) COMPILE_TESTS = \ remap_page_range \ diff -ru usr/src/nv/Makefile.nvidia usr/src/nv.2419292/Makefile.nvidia --- usr/src/nv/Makefile.nvidia 2008-05-19 00:37:02.000000000 -0700 +++ usr/src/nv.2419292/Makefile.nvidia 2008-05-21 12:08:08.771840702 -0700 @@ -56,10 +56,6 @@ CFLAGS += -DNV_MAP_REGISTERS_EARLY endif -ifeq ($(shell echo $(NV_BUILD_NV_PAT_SUPPORT)),1) -CFLAGS += -DNV_BUILD_NV_PAT_SUPPORT -endif - INCLUDES += -I$(KERNEL_HEADERS) COMPILE_TESTS = \ diff -ru usr/src/nv/nv-linux.h usr/src/nv.2419292/nv-linux.h --- usr/src/nv/nv-linux.h 2008-05-19 00:37:01.000000000 -0700 +++ usr/src/nv.2419292/nv-linux.h 2008-05-21 12:07:19.741046595 -0700 @@ -145,16 +145,19 @@ #endif #if (defined(NVCPU_X86) || defined(NVCPU_X86_64)) && !defined(CONFIG_XEN) -#define NV_BUILD_NV_PAT_SUPPORT 1 +#define NV_ENABLE_PAT_SUPPORT #endif -#if defined(NV_BUILD_NV_PAT_SUPPORT) -#include "pat.h" +#define NV_PAT_MODE_DISABLED 0 +#define NV_PAT_MODE_KERNEL 1 +#define NV_PAT_MODE_BUILTIN 2 + +extern int nv_pat_mode; + #if defined(CONFIG_HOTPLUG_CPU) #include /* CPU hotplug support */ #include /* struct notifier_block, etc */ #endif -#endif #if (defined(CONFIG_I2C) || defined(CONFIG_I2C_MODULE)) #include @@ -681,6 +684,13 @@ #define nv_down(lock) down(&lock) #define nv_up(lock) up(&lock) +#define NV_INIT_MUTEX(mutex) \ + { \ + struct semaphore __mutex = \ + __SEMAPHORE_INITIALIZER(*(mutex), 1); \ + *(mutex) = __mutex; \ + } + #if defined (KERNEL_2_4) # define NV_IS_SUSER() suser() # define NV_PCI_DEVICE_NAME(dev) ((dev)->name) @@ -1029,20 +1039,6 @@ return new_prot; } #endif - -#if defined(NV_BUILD_NV_PAT_SUPPORT) && !defined (pgprot_writecombined) -static inline pgprot_t pgprot_writecombined(pgprot_t old_prot) - { - pgprot_t new_prot = old_prot; - if (boot_cpu_data.x86 > 3) - { - pgprot_val(old_prot) &= ~(_PAGE_PCD | _PAGE_PWT); - new_prot = __pgprot(pgprot_val(old_prot) | _PAGE_WRTCOMB); - } - return new_prot; - } -#endif - #endif /* defined(NVCPU_X86) || defined(NVCPU_X86_64) */ #if defined(KERNEL_2_4) && defined(NVCPU_X86) && !defined(pfn_to_page) @@ -1142,8 +1138,6 @@ struct semaphore at_lock; } nv_linux_state_t; -extern int nv_pat_enabled; - #if defined(NV_LINUX_ACPI_EVENTS_SUPPORTED) /* * acpi data storage structure diff -ru usr/src/nv/nv-reg.h usr/src/nv.2419292/nv-reg.h --- usr/src/nv/nv-reg.h 2008-05-19 00:37:01.000000000 -0700 +++ usr/src/nv.2419292/nv-reg.h 2008-05-21 12:07:23.313250167 -0700 @@ -471,6 +471,37 @@ #define __NV_RM_EDGE_INTR_CHECK RMEdgeIntrCheck #define NV_REG_RM_EDGE_INTR_CHECK NV_REG_STRING(__NV_RM_EDGE_INTR_CHECK) +/* + * Option: UsePageAttributeTable + * + * Description: + * + * Enable/disable use of the page attribute table (PAT) available in + * modern x86/x86-64 processors to set the effective memory type of memory + * mappings to write-combining (WC). If disabled, the driver will fall + * back to using MTRRs, if possible. + * + * If enabled, an x86 processor with PAT support is present and the host + * system's Linux kernel did not configure one of the PAT entries to + * indicate the WC memory type, the driver will change the second entry in + * the PAT from its default (write-through (WT)) to WC at module load + * time. If the kernel did update one of the PAT entries, the driver will + * not modify the PAT. + * + * In both cases, the driver will honor attempts to map memory with the WC + * memory type by selecting the appropriate PAT entry using the correct + * set of PTE flags. + * + * Possible values: + * + * ~0 = use the NVIDIA driver's default logic (default) + * 1 = enable use of the PAT for WC mappings. + * 0 = disable use of the PAT for WC mappings. + */ + +#define __NV_USE_PAGE_ATTRIBUTE_TABLE UsePageAttributeTable +#define NV_USE_PAGE_ATTRIBUTE_TABLE NV_REG_STRING(__NV_USE_PAGE_ATTRIBUTE_TABLE) + #if defined(NV_DEFINE_REGISTRY_KEY_TABLE) @@ -495,6 +526,7 @@ NV_DEFINE_REG_ENTRY(__NV_UPDATE_MEMORY_TYPES, ~0); NV_DEFINE_REG_ENTRY(__NV_USE_VBIOS, 1); NV_DEFINE_REG_ENTRY(__NV_RM_EDGE_INTR_CHECK, 1); +NV_DEFINE_REG_ENTRY(__NV_USE_PAGE_ATTRIBUTE_TABLE, ~0); #if defined(NV_LINUX) NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL); @@ -540,6 +572,7 @@ NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_UPDATE_MEMORY_TYPES), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_VBIOS), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_EDGE_INTR_CHECK), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_USE_PAGE_ATTRIBUTE_TABLE), {NULL, NULL, NULL} }; diff -ru usr/src/nv/nv.c usr/src/nv.2419292/nv.c --- usr/src/nv/nv.c 2008-05-19 00:37:01.000000000 -0700 +++ usr/src/nv.2419292/nv.c 2008-05-21 12:07:28.313535114 -0700 @@ -22,10 +22,9 @@ #endif #if defined(KERNEL_2_4) && (defined(CONFIG_I2C) || defined(CONFIG_I2C_MODULE)) -// weak linking? extern int i2c_add_adapter (struct i2c_adapter *) __attribute__ ((weak)); extern int i2c_del_adapter (struct i2c_adapter *) __attribute__ ((weak)); -#endif // defined(KERNEL_2_4) && (defined(CONFIG_I2C) || defined(CONFIG_I2C_MODULE)) +#endif /* * our global state; one per device @@ -41,14 +40,7 @@ static struct pm_dev *apm_nv_dev[NV_MAX_DEVICES] = { 0 }; #endif -int nv_pat_enabled = 0; - -#if !defined(NV_BUILD_NV_PAT_SUPPORT) -static int nv_disable_pat = 1; -#else -static int nv_disable_pat = 0; -NV_MODULE_PARAMETER(nv_disable_pat); -#endif +int nv_pat_mode = NV_PAT_MODE_DISABLED; #if defined(NVCPU_X86) || defined(NVCPU_X86_64) NvU64 __nv_supported_pte_mask = ~_PAGE_NX; @@ -611,7 +603,7 @@ nv_state_t *nv; nv_linux_state_t *nvl; - proc_nvidia = create_proc_entry("nvidia", d_flags, proc_root_driver); + proc_nvidia = create_proc_entry("driver/nvidia", d_flags, NULL); if (!proc_nvidia) goto failed; @@ -863,18 +855,20 @@ static int __nv_enable_pat_support (void); static void __nv_disable_pat_support (void); -#if defined(NV_BUILD_NV_PAT_SUPPORT) +#if defined(NV_ENABLE_PAT_SUPPORT) /* - * Private PAT support for use by the NVIDIA driver. This is an - * interim solution until the kernel offers PAT support. + * Private PAT support for use by the NVIDIA driver. This is used on + * kernels that do not modify the PAT to include a write-combining + * entry. */ -static int __check_pat_support (void); +static int __determine_pat_mode (void); static void __nv_setup_pat_entries (void *); static void __nv_restore_pat_entries (void *); -#define NV_READ_PAT_ENTRIES(pat1, pat2) rdmsr(IA32_CR_PAT, (pat1), (pat2)) -#define NV_WRITE_PAT_ENTRIES(pat1, pat2) wrmsr(IA32_CR_PAT, (pat1), (pat2)) -#define NV_PAT_ENTRY(pat, index) (((pat) & (0xff<<((index)*8)))>>((index)*8)) +#define NV_READ_PAT_ENTRIES(pat1, pat2) rdmsr(0x277, (pat1), (pat2)) +#define NV_WRITE_PAT_ENTRIES(pat1, pat2) wrmsr(0x277, (pat1), (pat2)) +#define NV_PAT_ENTRY(pat, index) \ + (((pat) & (0xff << ((index)*8))) >> ((index)*8)) static inline void __nv_disable_caches(unsigned long *cr4) { @@ -895,39 +889,47 @@ if (cr4 & 0x80) write_cr4(cr4); } -static int __check_pat_support() +static int __determine_pat_mode() { unsigned int pat1, pat2, i; + U008 PAT_WC_index; - if (!test_bit(X86_FEATURE_PAT, (volatile unsigned long *)&boot_cpu_data.x86_capability)) + if (!test_bit(X86_FEATURE_PAT, + (volatile unsigned long *)&boot_cpu_data.x86_capability)) { nv_printf(NV_DBG_ERRORS, "NVRM: CPU does not support the PAT, falling back to MTRRs.\n"); - return 0; + return NV_PAT_MODE_DISABLED; } NV_READ_PAT_ENTRIES(pat1, pat2); + PAT_WC_index = 0xf; for (i = 0; i < 4; i++) { - // we plan to mark PAT entry 1 as WC. if it's already marked such, - // that's fine, since it would be no different than us setting it. - if ((i != 1) && NV_PAT_ENTRY(pat1, i) == 1) - { - nv_printf(NV_DBG_ERRORS, "NVRM: PAT index %d already configured for Write-Combining!\n", i); - nv_printf(NV_DBG_ERRORS, "NVRM: Aborting, due to PAT already being configured\n"); - return 0; - } - - if (NV_PAT_ENTRY(pat2, i) == 1) - { - nv_printf(NV_DBG_ERRORS, "NVRM: PAT index %d already configured for Write-Combining!\n", i + 4); - nv_printf(NV_DBG_ERRORS, "NVRM: Aborting, due to PAT already being configured\n"); - return 0; - } + if (NV_PAT_ENTRY(pat1, i) == 0x01) + { + PAT_WC_index = i; + break; + } + + if (NV_PAT_ENTRY(pat2, i) == 0x01) + { + PAT_WC_index = (i + 4); + break; + } } - return 1; + if (PAT_WC_index == 1) + return NV_PAT_MODE_KERNEL; + else if (PAT_WC_index != 0xf) + { + nv_printf(NV_DBG_ERRORS, + "NVRM: PAT configuration unsupported, falling back to MTRRs.\n"); + return NV_PAT_MODE_DISABLED; + } + else + return NV_PAT_MODE_BUILTIN; } static unsigned long orig_pat1, orig_pat2; @@ -978,19 +980,30 @@ __nv_enable_caches(cr4); NV_RESTORE_FLAGS(eflags); } - -#endif /* defined(NV_BUILD_NV_PAT_SUPPORT) */ +#endif static int __nv_enable_pat_support() { -#if defined(NV_BUILD_NV_PAT_SUPPORT) +#if defined(NV_ENABLE_PAT_SUPPORT) unsigned long pat1, pat2; - if (nv_pat_enabled) + if (nv_pat_mode != NV_PAT_MODE_DISABLED) return 1; - if (!__check_pat_support()) - return 0; + nv_pat_mode = __determine_pat_mode(); + + switch (nv_pat_mode) + { + case NV_PAT_MODE_DISABLED: + /* avoid the PAT if unavailable/unusable */ + return 0; + case NV_PAT_MODE_KERNEL: + /* inherit the kernel's PAT layout */ + return 1; + case NV_PAT_MODE_BUILTIN: + /* use builtin code to modify the PAT layout */ + break; + } NV_READ_PAT_ENTRIES(orig_pat1, orig_pat2); nv_printf(NV_DBG_SETUP, "saved orig pats as 0x%lx 0x%lx\n", orig_pat1, orig_pat2); @@ -1001,31 +1014,28 @@ return 0; } - nv_pat_enabled = 1; - NV_READ_PAT_ENTRIES(pat1, pat2); nv_printf(NV_DBG_SETUP, "changed pats to 0x%lx 0x%lx\n", pat1, pat2); -#endif /* defined(NV_BUILD_NV_PAT_SUPPORT) */ - +#endif return 1; } static void __nv_disable_pat_support() { -#if defined(NV_BUILD_NV_PAT_SUPPORT) +#if defined(NV_ENABLE_PAT_SUPPORT) unsigned long pat1, pat2; - if (!nv_pat_enabled) + if (nv_pat_mode != NV_PAT_MODE_BUILTIN) return; if (nv_execute_on_all_cpus(__nv_restore_pat_entries, NULL) != 0) return; - nv_pat_enabled = 0; + nv_pat_mode = NV_PAT_MODE_DISABLED; NV_READ_PAT_ENTRIES(pat1, pat2); nv_printf(NV_DBG_SETUP, "restored orig pats as 0x%lx 0x%lx\n", pat1, pat2); -#endif /* defined(NV_BUILD_NV_PAT_SUPPORT) */ +#endif } #if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) @@ -1273,7 +1283,7 @@ #endif } -#if defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) +#if defined(NV_ENABLE_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) static int nv_kern_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -1305,8 +1315,7 @@ .notifier_call = nv_kern_cpu_callback, .priority = 0 }; - -#endif /* defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) */ +#endif /*** @@ -1315,7 +1324,7 @@ static int __init nvidia_init_module(void) { - int rc; + int rc, disable_pat = 0; U032 i, count, data; nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device); nv_stack_t *sp = NULL; @@ -1443,18 +1452,6 @@ nv_printf(NV_DBG_ERRORS, "NVRM: pte cache allocation failed\n"); goto failed; } - -#if defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) - if (!nv_disable_pat) - { - if (register_hotcpu_notifier(&nv_hotcpu_nfb) != 0) - { - rc = -EIO; - nv_printf(NV_DBG_ERRORS, "NVRM: CPU hotplug notifier registration failed!\n"); - goto failed; - } - } -#endif #if defined(NV_SG_MAP_BUFFERS) rm_read_registry_dword(sp, nv, "NVreg", "RemapLimit", &nv_remap_limit); @@ -1532,8 +1529,30 @@ nvos_proc_add_warning_file("README", __README_warning); - if (!nv_disable_pat) + rc = rm_read_registry_dword(sp, nv, + "NVreg", NV_USE_PAGE_ATTRIBUTE_TABLE, &data); + if ((rc == 0) && ((int)data != ~0)) + { + disable_pat = (data == 0); + } + + if (!disable_pat) + { __nv_enable_pat_support(); +#if defined(NV_ENABLE_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) + if (nv_pat_mode == NV_PAT_MODE_BUILTIN) + { + if (register_hotcpu_notifier(&nv_hotcpu_nfb) != 0) + { + __nv_disable_pat_support(); + rc = -EIO; + nv_printf(NV_DBG_ERRORS, + "NVRM: CPU hotplug notifier registration failed!\n"); + goto failed; + } + } +#endif + } else { nv_printf(NV_DBG_ERRORS, @@ -1686,10 +1705,10 @@ rm_unregister_compatible_ioctls(sp); #endif - if (nv_pat_enabled) + if (nv_pat_mode == NV_PAT_MODE_BUILTIN) { __nv_disable_pat_support(); -#if defined(NV_BUILD_NV_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) +#if defined(NV_ENABLE_PAT_SUPPORT) && defined(CONFIG_HOTPLUG_CPU) unregister_hotcpu_notifier(&nv_hotcpu_nfb); #endif } @@ -1825,6 +1844,8 @@ } } +#if !defined(NV_VM_INSERT_PAGE_PRESENT) +static struct page *nv_kern_vma_nopage( struct vm_area_struct *vma, unsigned long address, @@ -1835,22 +1856,21 @@ #endif ) { -#if !defined(NV_VM_INSERT_PAGE_PRESENT) struct page *page; page = pfn_to_page(vma->vm_pgoff); get_page(page); return page; -#else - return NOPAGE_SIGBUS; -#endif } +#endif struct vm_operations_struct nv_vm_ops = { .open = nv_kern_vma_open, .close = nv_kern_vma_release, /* "close" */ +#if !defined(NV_VM_INSERT_PAGE_PRESENT) .nopage = nv_kern_vma_nopage, +#endif }; static nv_file_private_t * @@ -1864,7 +1884,7 @@ memset(nvfp, 0, sizeof(nv_file_private_t)); - sema_init(&nvfp->sp_lock, 1); + NV_INIT_MUTEX(&nvfp->sp_lock); // initialize this file's event queue init_waitqueue_head(&nvfp->waitqueue); @@ -2214,11 +2234,12 @@ break; #if defined(NVCPU_X86) || defined(NVCPU_X86_64) case NV_MEMORY_WRITECOMBINED: -#if defined(NV_BUILD_NV_PAT_SUPPORT) - if (nv_pat_enabled && - (memory_type != NV_MEMORY_TYPE_REGISTERS)) +#if defined(NV_ENABLE_PAT_SUPPORT) + if ((nv_pat_mode != NV_PAT_MODE_DISABLED) && + (memory_type != NV_MEMORY_TYPE_REGISTERS)) { - *prot = pgprot_writecombined(*prot); + pgprot_val(*prot) &= ~(_PAGE_PSE | _PAGE_PCD | _PAGE_PWT); + *prot = __pgprot(pgprot_val(*prot) | _PAGE_PWT); break; } #endif @@ -2242,7 +2263,6 @@ return 1; #endif case NV_MEMORY_CACHED: - //case NV_MEMORY_WRITEBACK: #if !defined(NVCPU_X86) && !defined(NVCPU_X86_64) if (memory_type != NV_MEMORY_TYPE_REGISTERS) break; @@ -2264,8 +2284,6 @@ if (memory_type == NV_MEMORY_TYPE_SYSTEM) break; #endif - //case NV_MEMORY_WRITETHRU: - //case NV_MEMORY_WRITEPROTECT: default: if(nv_ext_encode_caching(prot, cache_type, memory_type) == 0) return 0; @@ -3502,6 +3520,10 @@ BOOL kern ) { +#if defined(NV_SET_PAGES_UC_PRESENT) && defined(NVCPU_X86) + nv_printf(NV_DBG_ERRORS, + "NVRM: can't translate address in nv_get_phys_address()!\n"); +#else struct mm_struct *mm; pgd_t *pgd = NULL; pmd_t *pmd = NULL; @@ -3514,15 +3536,7 @@ down_read(&mm->mmap_sem); } else - { -#if defined(NV_SET_PAGES_UC_PRESENT) && defined(NVCPU_X86) - nv_printf(NV_DBG_ERRORS, - "NVRM: can't translate KVA in nv_get_phys_address()!\n"); - return 0; -#else mm = NULL; -#endif - } pgd = NV_PGD_OFFSET(address, kern, mm); if (!NV_PGD_PRESENT(pgd)) @@ -3550,6 +3564,7 @@ failed: if (!kern) up_read(&mm->mmap_sem); +#endif return 0; } @@ -3790,8 +3805,9 @@ nv_init_lock(nvl->rm_lock); - sema_init(&nvl->ldata_lock, 1); - sema_init(&nvl->at_lock, 1); + NV_INIT_MUTEX(&nvl->ldata_lock); + NV_INIT_MUTEX(&nvl->at_lock); + NV_ATOMIC_SET(nvl->usage_count, 0); nvl->rm_lock_cpu = -1; diff -ru usr/src/nv/os-agp.c usr/src/nv.2419292/os-agp.c --- usr/src/nv/os-agp.c 2008-05-19 00:37:01.000000000 -0700 +++ usr/src/nv.2419292/os-agp.c 2008-05-21 12:07:30.437656163 -0700 @@ -115,7 +115,7 @@ goto release; } - if (!nv_pat_enabled) + if (nv_pat_mode == NV_PAT_MODE_DISABLED) { #ifdef CONFIG_MTRR /* @@ -175,7 +175,7 @@ failed: #ifdef CONFIG_MTRR - if (!nv_pat_enabled) + if (nv_pat_mode == NV_PAT_MODE_DISABLED) mtrr_del(-1, agp_info.aper_base, agp_info.aper_size << 20); #endif release: @@ -205,7 +205,7 @@ nvl = NV_GET_NVL_FROM_NV_STATE(nv); #ifdef CONFIG_MTRR - if (!nv_pat_enabled) + if (nv_pat_mode == NV_PAT_MODE_DISABLED) mtrr_del(-1, nv->agp.address, nv->agp.size); #endif diff -ru usr/src/nv/os-interface.c usr/src/nv.2419292/os-interface.c --- usr/src/nv/os-interface.c 2008-05-19 00:37:01.000000000 -0700 +++ usr/src/nv.2419292/os-interface.c 2008-05-21 12:07:33.365823028 -0700 @@ -1355,7 +1355,7 @@ BOOL NV_API_CALL os_pat_supported(void) { - return nv_pat_enabled; + return (nv_pat_mode != NV_PAT_MODE_DISABLED); } void NV_API_CALL os_dump_stack()