Patches contributed by Eötvös Lorand University
commit f180053694b43d5714bf56cb95499a3c32ff155c
Author: Ingo Molnar <mingo@elte.hu>
Date: Mon Mar 2 11:00:57 2009 +0100
x86, mm: dont use non-temporal stores in pagecache accesses
Impact: standardize IO on cached ops
On modern CPUs it is almost always a bad idea to use non-temporal stores,
as the regression in this commit has shown it:
30d697f: x86: fix performance regression in write() syscall
The kernel simply has no good information about whether using non-temporal
stores is a good idea or not - and trying to add heuristics only increases
complexity and inserts fragility.
The regression on cached write()s took very long to be found - over two
years. So dont take any chances and let the hardware decide how it makes
use of its caches.
The only exception is drivers/gpu/drm/i915/i915_gem.c: there were we are
absolutely sure that another entity (the GPU) will pick up the dirty
data immediately and that the CPU will not touch that data before the
GPU will.
Also, keep the _nocache() primitives to make it easier for people to
experiment with these details. There may be more clear-cut cases where
non-cached copies can be used, outside of filemap.c.
Cc: Salman Qazi <sqazi@google.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index a0ba61386972..5e06259e90e5 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -157,7 +157,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
}
static __always_inline unsigned long __copy_from_user_nocache(void *to,
- const void __user *from, unsigned long n, unsigned long total)
+ const void __user *from, unsigned long n)
{
might_fault();
if (__builtin_constant_p(n)) {
@@ -180,7 +180,7 @@ static __always_inline unsigned long __copy_from_user_nocache(void *to,
static __always_inline unsigned long
__copy_from_user_inatomic_nocache(void *to, const void __user *from,
- unsigned long n, unsigned long total)
+ unsigned long n)
{
return __copy_from_user_ll_nocache_nozero(to, from, n);
}
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index dcaa0404cf7b..8cc687326eb8 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -188,29 +188,18 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
extern long __copy_user_nocache(void *dst, const void __user *src,
unsigned size, int zerorest);
-static inline int __copy_from_user_nocache(void *dst, const void __user *src,
- unsigned size, unsigned long total)
+static inline int
+__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
{
might_sleep();
- /*
- * In practice this limit means that large file write()s
- * which get chunked to 4K copies get handled via
- * non-temporal stores here. Smaller writes get handled
- * via regular __copy_from_user():
- */
- if (likely(total >= PAGE_SIZE))
- return __copy_user_nocache(dst, src, size, 1);
- else
- return __copy_from_user(dst, src, size);
+ return __copy_user_nocache(dst, src, size, 1);
}
-static inline int __copy_from_user_inatomic_nocache(void *dst,
- const void __user *src, unsigned size, unsigned total)
+static inline int
+__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
+ unsigned size)
{
- if (likely(total >= PAGE_SIZE))
- return __copy_user_nocache(dst, src, size, 0);
- else
- return __copy_from_user_inatomic(dst, src, size);
+ return __copy_user_nocache(dst, src, size, 0);
}
unsigned long
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 6b209db8370d..818576654092 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -215,7 +215,7 @@ fast_user_write(struct io_mapping *mapping,
vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
- user_data, length, length);
+ user_data, length);
io_mapping_unmap_atomic(vaddr_atomic);
if (unwritten)
return -EFAULT;
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 6f3c603b0d67..6b58367d145e 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -41,13 +41,13 @@ static inline void pagefault_enable(void)
#ifndef ARCH_HAS_NOCACHE_UACCESS
static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
- const void __user *from, unsigned long n, unsigned long total)
+ const void __user *from, unsigned long n)
{
return __copy_from_user_inatomic(to, from, n);
}
static inline unsigned long __copy_from_user_nocache(void *to,
- const void __user *from, unsigned long n, unsigned long total)
+ const void __user *from, unsigned long n)
{
return __copy_from_user(to, from, n);
}
diff --git a/mm/filemap.c b/mm/filemap.c
index 60fd56772cc6..126d3973b3d1 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1816,14 +1816,14 @@ EXPORT_SYMBOL(file_remove_suid);
static size_t __iovec_copy_from_user_inatomic(char *vaddr,
const struct iovec *iov, size_t base, size_t bytes)
{
- size_t copied = 0, left = 0, total = bytes;
+ size_t copied = 0, left = 0;
while (bytes) {
char __user *buf = iov->iov_base + base;
int copy = min(bytes, iov->iov_len - base);
base = 0;
- left = __copy_from_user_inatomic_nocache(vaddr, buf, copy, total);
+ left = __copy_from_user_inatomic(vaddr, buf, copy);
copied += copy;
bytes -= copy;
vaddr += copy;
@@ -1851,9 +1851,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
if (likely(i->nr_segs == 1)) {
int left;
char __user *buf = i->iov->iov_base + i->iov_offset;
-
- left = __copy_from_user_inatomic_nocache(kaddr + offset,
- buf, bytes, bytes);
+ left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
copied = bytes - left;
} else {
copied = __iovec_copy_from_user_inatomic(kaddr + offset,
@@ -1881,8 +1879,7 @@ size_t iov_iter_copy_from_user(struct page *page,
if (likely(i->nr_segs == 1)) {
int left;
char __user *buf = i->iov->iov_base + i->iov_offset;
-
- left = __copy_from_user_nocache(kaddr + offset, buf, bytes, bytes);
+ left = __copy_from_user(kaddr + offset, buf, bytes);
copied = bytes - left;
} else {
copied = __iovec_copy_from_user_inatomic(kaddr + offset,
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
index bf54f8a2cf1d..0c04615651b7 100644
--- a/mm/filemap_xip.c
+++ b/mm/filemap_xip.c
@@ -354,7 +354,7 @@ __xip_file_write(struct file *filp, const char __user *buf,
break;
copied = bytes -
- __copy_from_user_nocache(xip_mem + offset, buf, bytes, bytes);
+ __copy_from_user_nocache(xip_mem + offset, buf, bytes);
if (likely(copied > 0)) {
status = copied;
commit 645af4e9e0e32481e3336dda813688732c7e5f0f
Merge: ecc25fbd6b9e 55f2b7899582
Author: Ingo Molnar <mingo@elte.hu>
Date: Sun Mar 1 12:48:28 2009 +0100
Merge branch 'x86/pat' into x86/core
commit 55f2b78995826d549401bdf20abeac1832636bb6
Merge: f5c1aa1537be 92b9af9e4f14
Author: Ingo Molnar <mingo@elte.hu>
Date: Sun Mar 1 12:47:58 2009 +0100
Merge branch 'x86/urgent' into x86/pat
commit f5c1aa1537be39d8b9bb5279b5881d81898fd3cd
Author: Ingo Molnar <mingo@elte.hu>
Date: Sun Mar 1 12:32:08 2009 +0100
Revert "gpu/drm, x86, PAT: PAT support for io_mapping_*"
This reverts commit 17581ad812a9abb0182260374ef2e52d4a808a64.
Sitsofe Wheeler reported that /dev/dri/card0 is MIA on his EeePC 900
and bisected it to this commit.
Graphics card is an i915 in an EeePC 900:
00:02.0 VGA compatible controller [0300]:
Intel Corporation Mobile 915GM/GMS/910GML
Express Graphics Controller [8086:2592] (rev 04)
( Most likely the ioremap() of the driver failed and hence the card
did not initialize. )
Reported-by: Sitsofe Wheeler <sitsofe@yahoo.com>
Bisected-by: Sitsofe Wheeler <sitsofe@yahoo.com>
Cc: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
Cc: Suresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/include/asm/iomap.h b/arch/x86/include/asm/iomap.h
index bd46495ff7de..86af26091d6c 100644
--- a/arch/x86/include/asm/iomap.h
+++ b/arch/x86/include/asm/iomap.h
@@ -24,10 +24,7 @@
#include <asm/tlbflush.h>
int
-reserve_io_memtype_wc(u64 base, unsigned long size, pgprot_t *prot);
-
-void
-free_io_memtype(u64 base, unsigned long size);
+is_io_mapping_possible(resource_size_t base, unsigned long size);
void *
iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index d5e28424622c..6c2b1af16926 100644
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -21,13 +21,13 @@
#include <linux/module.h>
#ifdef CONFIG_X86_PAE
-static int
+int
is_io_mapping_possible(resource_size_t base, unsigned long size)
{
return 1;
}
#else
-static int
+int
is_io_mapping_possible(resource_size_t base, unsigned long size)
{
/* There is no way to map greater than 1 << 32 address without PAE */
@@ -38,46 +38,6 @@ is_io_mapping_possible(resource_size_t base, unsigned long size)
}
#endif
-int
-reserve_io_memtype_wc(u64 base, unsigned long size, pgprot_t *prot)
-{
- unsigned long ret_flag;
-
- if (!is_io_mapping_possible(base, size))
- goto out_err;
-
- if (!pat_enabled) {
- *prot = pgprot_noncached(PAGE_KERNEL);
- return 0;
- }
-
- if (reserve_memtype(base, base + size, _PAGE_CACHE_WC, &ret_flag))
- goto out_err;
-
- if (ret_flag == _PAGE_CACHE_WB)
- goto out_free;
-
- if (kernel_map_sync_memtype(base, size, ret_flag))
- goto out_free;
-
- *prot = __pgprot(__PAGE_KERNEL | ret_flag);
- return 0;
-
-out_free:
- free_memtype(base, base + size);
-out_err:
- return -EINVAL;
-}
-EXPORT_SYMBOL_GPL(reserve_io_memtype_wc);
-
-void
-free_io_memtype(u64 base, unsigned long size)
-{
- if (pat_enabled)
- free_memtype(base, base + size);
-}
-EXPORT_SYMBOL_GPL(free_io_memtype);
-
/* Map 'pfn' using fixed map 'type' and protections 'prot'
*/
void *
diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h
index f1ed66c43787..cbc2f0cd631b 100644
--- a/include/linux/io-mapping.h
+++ b/include/linux/io-mapping.h
@@ -49,9 +49,8 @@ static inline struct io_mapping *
io_mapping_create_wc(resource_size_t base, unsigned long size)
{
struct io_mapping *iomap;
- pgprot_t prot;
- if (!reserve_io_memtype_wc(base, size, &prot))
+ if (!is_io_mapping_possible(base, size))
return NULL;
iomap = kmalloc(sizeof(*iomap), GFP_KERNEL);
@@ -60,14 +59,13 @@ io_mapping_create_wc(resource_size_t base, unsigned long size)
iomap->base = base;
iomap->size = size;
- iomap->prot = prot;
+ iomap->prot = pgprot_writecombine(__pgprot(__PAGE_KERNEL));
return iomap;
}
static inline void
io_mapping_free(struct io_mapping *mapping)
{
- free_io_memtype(mapping->base, mapping->size);
kfree(mapping);
}
commit 92b9af9e4f144535c65aee673cfad309f25fa465
Author: Ingo Molnar <mingo@elte.hu>
Date: Sat Feb 28 14:09:27 2009 +0100
x86: i915 needs pgprot_writecombine() and is_io_mapping_possible()
Impact: build fix
Theodore Ts reported that the i915 driver needs these symbols:
ERROR: "pgprot_writecombine" [drivers/gpu/drm/i915/i915.ko] undefined!
ERROR: "is_io_mapping_possible" [drivers/gpu/drm/i915/i915.ko] undefined!
Reported-by: Theodore Ts'o <tytso@mit.edu> wrote:
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index 6c2b1af16926..04102d42ff42 100644
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -20,23 +20,16 @@
#include <asm/pat.h>
#include <linux/module.h>
-#ifdef CONFIG_X86_PAE
-int
-is_io_mapping_possible(resource_size_t base, unsigned long size)
-{
- return 1;
-}
-#else
-int
-is_io_mapping_possible(resource_size_t base, unsigned long size)
+int is_io_mapping_possible(resource_size_t base, unsigned long size)
{
+#ifndef CONFIG_X86_PAE
/* There is no way to map greater than 1 << 32 address without PAE */
if (base + size > 0x100000000ULL)
return 0;
-
+#endif
return 1;
}
-#endif
+EXPORT_SYMBOL_GPL(is_io_mapping_possible);
/* Map 'pfn' using fixed map 'type' and protections 'prot'
*/
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index aebbf67a79d0..e0ab173b6974 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -11,6 +11,7 @@
#include <linux/bootmem.h>
#include <linux/debugfs.h>
#include <linux/kernel.h>
+#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/fs.h>
@@ -868,6 +869,7 @@ pgprot_t pgprot_writecombine(pgprot_t prot)
else
return pgprot_noncached(prot);
}
+EXPORT_SYMBOL_GPL(pgprot_writecombine);
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
commit acdb2c287916ce247c09d460266370af45da55c5
Merge: e9abf4c59d75 f2034f1e1ada
Author: Ingo Molnar <mingo@elte.hu>
Date: Sat Feb 28 10:15:58 2009 +0100
Merge branch 'tip/tracing/ftrace' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace into tracing/ftrace
commit e9abf4c59d75cd71f118764376d037a677f573d0
Merge: da74ff0f9b31 0cfe82451dfa
Author: Ingo Molnar <mingo@elte.hu>
Date: Sat Feb 28 09:03:58 2009 +0100
Merge branch 'tip/tracing/ftrace' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace into tracing/ftrace
commit 4387d6d6b8a74722bf1f6fc3e29dd2a84cc1d1c2
Merge: f701d3540759 da74ff0f9b31
Author: Ingo Molnar <mingo@elte.hu>
Date: Sat Feb 28 09:02:49 2009 +0100
Merge branch 'tracing/ftrace' into tracing/core
commit da74ff0f9b31ed97104c22c278bf2b41bcf8f33c
Merge: f701d3540759 5c6a3ae1b4be
Author: Ingo Molnar <mingo@elte.hu>
Date: Fri Feb 27 09:06:11 2009 +0100
Merge branch 'tip/tracing/ftrace' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace into tracing/ftrace
Conflicts:
kernel/sched_clock.c
commit f701d354075914296474d062f18fc8ee681a2318
Merge: 1b49061d400c 64e71303e44f
Author: Ingo Molnar <mingo@elte.hu>
Date: Fri Feb 27 09:04:43 2009 +0100
Merge branches 'tracing/ftrace' and 'linus' into tracing/core