Emulation/switch-linux
1
0
Fork 0
mirror of https://github.com/fail0verflow/switch-linux.git synced 2025-05-04 02:34:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6

Browse source 
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6:
  slub: Deal with hyperthetical case of PAGE_SIZE > 2M
  slub: Remove node check in slab_free
  slub: avoid label inside conditional
  slub: Make CONFIG_DEBUG_PAGE_ALLOC work with new fastpath
  slub: Avoid warning for !CONFIG_SLUB_DEBUG
  slub: Remove CONFIG_CMPXCHG_LOCAL ifdeffery
  slub: Move debug handlign in __slab_free
  slub: Move node determination out of hotpath
  slub: Eliminate repeated use of c->page through a new page variable
  slub: get_map() function to establish map of free objects in a slab
  slub: Use NUMA_NO_NODE in get_partial
  slub: Fix a typo in config name
This commit is contained in:
Linus Torvalds 2011-05-23 10:10:44 -07:00
commit 4867faab1e
2 changed files with 69 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
include/linux/slub_def.h
View file

@ -37,9 +37,7 @@ enum stat_item {
struct kmem_cache_cpu { struct kmem_cache_cpu {
void **freelist; /* Pointer to next available object */ void **freelist; /* Pointer to next available object */
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; /* Globally unique transaction id */ unsigned long tid; /* Globally unique transaction id */
#endif
struct page *page; /* The slab from which we are allocating */ struct page *page; /* The slab from which we are allocating */
int node; /* The node of the page (or -1 for debug) */ int node; /* The node of the page (or -1 for debug) */
#ifdef CONFIG_SLUB_STATS #ifdef CONFIG_SLUB_STATS
@ -179,7 +177,8 @@ static __always_inline int kmalloc_index(size_t size)
if (size <= 4 * 1024) return 12; if (size <= 4 * 1024) return 12;
/* /*
* The following is only needed to support architectures with a larger page * The following is only needed to support architectures with a larger page
* size than 4k. * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
* size we would have to go up to 128k.
*/ */
if (size <= 8 * 1024) return 13; if (size <= 8 * 1024) return 13;
if (size <= 16 * 1024) return 14; if (size <= 16 * 1024) return 14;
@ -190,7 +189,8 @@ static __always_inline int kmalloc_index(size_t size)
if (size <= 512 * 1024) return 19; if (size <= 512 * 1024) return 19;
if (size <= 1024 * 1024) return 20; if (size <= 1024 * 1024) return 20;
if (size <= 2 * 1024 * 1024) return 21; if (size <= 2 * 1024 * 1024) return 21;
return -1; BUG();
return -1; /* Will never be reached */
/* /*
* What we really wanted to do and cannot do because of compiler issues is: * What we really wanted to do and cannot do because of compiler issues is:

165
mm/slub.c
View file

@ -261,6 +261,18 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
return *(void **)(object + s->offset); return *(void **)(object + s->offset);
} }
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
void *p;
#ifdef CONFIG_DEBUG_PAGEALLOC
probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
#else
p = get_freepointer(s, object);
#endif
return p;
}
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{ {
*(void **)(object + s->offset) = fp; *(void **)(object + s->offset) = fp;
@ -271,10 +283,6 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
__p += (__s)->size) __p += (__s)->size)
/* Scan freelist */
#define for_each_free_object(__p, __s, __free) \
for (__p = (__free); __p; __p = get_freepointer((__s), __p))
/* Determine object index from a given position */ /* Determine object index from a given position */
static inline int slab_index(void *p, struct kmem_cache *s, void *addr) static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
{ {
@ -331,6 +339,21 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
} }
#ifdef CONFIG_SLUB_DEBUG #ifdef CONFIG_SLUB_DEBUG
/*
* Determine a map of object in use on a page.
*
* Slab lock or node listlock must be held to guarantee that the page does
* not vanish from under us.
*/
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
{
void *p;
void *addr = page_address(page);
for (p = page->freelist; p; p = get_freepointer(s, p))
set_bit(slab_index(p, s, addr), map);
}
/* /*
* Debug settings: * Debug settings:
*/ */
@ -1487,7 +1510,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node; int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
page = get_partial_node(get_node(s, searchnode)); page = get_partial_node(get_node(s, searchnode));
if (page || node != -1) if (page || node != NUMA_NO_NODE)
return page; return page;
return get_any_partial(s, flags); return get_any_partial(s, flags);
@ -1540,7 +1563,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
} }
} }
#ifdef CONFIG_CMPXCHG_LOCAL
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
/* /*
* Calculate the next globally unique transaction for disambiguiation * Calculate the next globally unique transaction for disambiguiation
@ -1600,17 +1622,12 @@ static inline void note_cmpxchg_failure(const char *n,
stat(s, CMPXCHG_DOUBLE_CPU_FAIL); stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
} }
#endif
void init_kmem_cache_cpus(struct kmem_cache *s) void init_kmem_cache_cpus(struct kmem_cache *s)
{ {
#ifdef CONFIG_CMPXCHG_LOCAL
int cpu; int cpu;
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
#endif
} }
/* /*
* Remove the cpu slab * Remove the cpu slab
@ -1643,9 +1660,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
page->inuse--; page->inuse--;
} }
c->page = NULL; c->page = NULL;
#ifdef CONFIG_CMPXCHG_LOCAL
c->tid = next_tid(c->tid); c->tid = next_tid(c->tid);
#endif
unfreeze_slab(s, page, tail); unfreeze_slab(s, page, tail);
} }
@ -1779,8 +1794,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
unsigned long addr, struct kmem_cache_cpu *c) unsigned long addr, struct kmem_cache_cpu *c)
{ {
void **object; void **object;
struct page *new; struct page *page;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long flags; unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
@ -1791,38 +1805,36 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
* pointer. * pointer.
*/ */
c = this_cpu_ptr(s->cpu_slab); c = this_cpu_ptr(s->cpu_slab);
#endif
#endif #endif
/* We handle __GFP_ZERO in the caller */ /* We handle __GFP_ZERO in the caller */
gfpflags &= ~__GFP_ZERO; gfpflags &= ~__GFP_ZERO;
if (!c->page) page = c->page;
if (!page)
goto new_slab; goto new_slab;
slab_lock(c->page); slab_lock(page);
if (unlikely(!node_match(c, node))) if (unlikely(!node_match(c, node)))
goto another_slab; goto another_slab;
stat(s, ALLOC_REFILL); stat(s, ALLOC_REFILL);
load_freelist: load_freelist:
object = c->page->freelist; object = page->freelist;
if (unlikely(!object)) if (unlikely(!object))
goto another_slab; goto another_slab;
if (kmem_cache_debug(s)) if (kmem_cache_debug(s))
goto debug; goto debug;
c->freelist = get_freepointer(s, object); c->freelist = get_freepointer(s, object);
c->page->inuse = c->page->objects; page->inuse = page->objects;
c->page->freelist = NULL; page->freelist = NULL;
c->node = page_to_nid(c->page);
unlock_out: unlock_out:
slab_unlock(c->page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
c->tid = next_tid(c->tid); c->tid = next_tid(c->tid);
local_irq_restore(flags); local_irq_restore(flags);
#endif
stat(s, ALLOC_SLOWPATH); stat(s, ALLOC_SLOWPATH);
return object; return object;
@ -1830,10 +1842,11 @@ another_slab:
deactivate_slab(s, c); deactivate_slab(s, c);
new_slab: new_slab:
new = get_partial(s, gfpflags, node); page = get_partial(s, gfpflags, node);
if (new) { if (page) {
c->page = new;
stat(s, ALLOC_FROM_PARTIAL); stat(s, ALLOC_FROM_PARTIAL);
c->node = page_to_nid(page);
c->page = page;
goto load_freelist; goto load_freelist;
} }
@ -1841,33 +1854,35 @@ new_slab:
if (gfpflags & __GFP_WAIT) if (gfpflags & __GFP_WAIT)
local_irq_enable(); local_irq_enable();
new = new_slab(s, gfpflags, node); page = new_slab(s, gfpflags, node);
if (gfpflags & __GFP_WAIT) if (gfpflags & __GFP_WAIT)
local_irq_disable(); local_irq_disable();
if (new) { if (page) {
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
stat(s, ALLOC_SLAB); stat(s, ALLOC_SLAB);
if (c->page) if (c->page)
flush_slab(s, c); flush_slab(s, c);
slab_lock(new);
__SetPageSlubFrozen(new); slab_lock(page);
c->page = new; __SetPageSlubFrozen(page);
c->node = page_to_nid(page);
c->page = page;
goto load_freelist; goto load_freelist;
} }
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
slab_out_of_memory(s, gfpflags, node); slab_out_of_memory(s, gfpflags, node);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
return NULL; return NULL;
debug: debug:
if (!alloc_debug_processing(s, c->page, object, addr)) if (!alloc_debug_processing(s, page, object, addr))
goto another_slab; goto another_slab;
c->page->inuse++; page->inuse++;
c->page->freelist = get_freepointer(s, object); page->freelist = get_freepointer(s, object);
deactivate_slab(s, c);
c->page = NULL;
c->node = NUMA_NO_NODE; c->node = NUMA_NO_NODE;
goto unlock_out; goto unlock_out;
} }
@ -1887,20 +1902,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
{ {
void **object; void **object;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; unsigned long tid;
#else
unsigned long flags;
#endif
if (slab_pre_alloc_hook(s, gfpflags)) if (slab_pre_alloc_hook(s, gfpflags))
return NULL; return NULL;
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_save(flags);
#else
redo: redo:
#endif
/* /*
* Must read kmem_cache cpu data via this cpu ptr. Preemption is * Must read kmem_cache cpu data via this cpu ptr. Preemption is
@ -1910,7 +1917,6 @@ redo:
*/ */
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
#ifdef CONFIG_CMPXCHG_LOCAL
/* /*
* The transaction ids are globally unique per cpu and per operation on * The transaction ids are globally unique per cpu and per operation on
* a per cpu queue. Thus they can be guarantee that the cmpxchg_double * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
@ -1919,7 +1925,6 @@ redo:
*/ */
tid = c->tid; tid = c->tid;
barrier(); barrier();
#endif
object = c->freelist; object = c->freelist;
if (unlikely(!object || !node_match(c, node))) if (unlikely(!object || !node_match(c, node)))
@ -1927,7 +1932,6 @@ redo:
object = __slab_alloc(s, gfpflags, node, addr, c); object = __slab_alloc(s, gfpflags, node, addr, c);
else { else {
#ifdef CONFIG_CMPXCHG_LOCAL
/* /*
* The cmpxchg will only match if there was no additional * The cmpxchg will only match if there was no additional
* operation and if we are on the right processor. * operation and if we are on the right processor.
@ -1943,21 +1947,14 @@ redo:
if (unlikely(!irqsafe_cpu_cmpxchg_double( if (unlikely(!irqsafe_cpu_cmpxchg_double(
s->cpu_slab->freelist, s->cpu_slab->tid, s->cpu_slab->freelist, s->cpu_slab->tid,
object, tid, object, tid,
get_freepointer(s, object), next_tid(tid)))) { get_freepointer_safe(s, object), next_tid(tid)))) {
note_cmpxchg_failure("slab_alloc", s, tid); note_cmpxchg_failure("slab_alloc", s, tid);
goto redo; goto redo;
} }
#else
c->freelist = get_freepointer(s, object);
#endif
stat(s, ALLOC_FASTPATH); stat(s, ALLOC_FASTPATH);
} }
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags);
#endif
if (unlikely(gfpflags & __GFP_ZERO) && object) if (unlikely(gfpflags & __GFP_ZERO) && object)
memset(object, 0, s->objsize); memset(object, 0, s->objsize);
@ -2034,18 +2031,15 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
{ {
void *prior; void *prior;
void **object = (void *)x; void **object = (void *)x;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long flags; unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
#endif
slab_lock(page); slab_lock(page);
stat(s, FREE_SLOWPATH); stat(s, FREE_SLOWPATH);
if (kmem_cache_debug(s)) if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
goto debug; goto out_unlock;
checks_ok:
prior = page->freelist; prior = page->freelist;
set_freepointer(s, object, prior); set_freepointer(s, object, prior);
page->freelist = object; page->freelist = object;
@ -2070,9 +2064,7 @@ checks_ok:
out_unlock: out_unlock:
slab_unlock(page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
return; return;
slab_empty: slab_empty:
@ -2084,17 +2076,9 @@ slab_empty:
stat(s, FREE_REMOVE_PARTIAL); stat(s, FREE_REMOVE_PARTIAL);
} }
slab_unlock(page); slab_unlock(page);
#ifdef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags); local_irq_restore(flags);
#endif
stat(s, FREE_SLAB); stat(s, FREE_SLAB);
discard_slab(s, page); discard_slab(s, page);
return;
debug:
if (!free_debug_processing(s, page, x, addr))
goto out_unlock;
goto checks_ok;
} }
/* /*
@ -2113,20 +2097,11 @@ static __always_inline void slab_free(struct kmem_cache *s,
{ {
void **object = (void *)x; void **object = (void *)x;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
#ifdef CONFIG_CMPXCHG_LOCAL
unsigned long tid; unsigned long tid;
#else
unsigned long flags;
#endif
slab_free_hook(s, x); slab_free_hook(s, x);
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_save(flags);
#else
redo: redo:
#endif
/* /*
* Determine the currently cpus per cpu slab. * Determine the currently cpus per cpu slab.
@ -2136,15 +2111,12 @@ redo:
*/ */
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
#ifdef CONFIG_CMPXCHG_LOCAL
tid = c->tid; tid = c->tid;
barrier(); barrier();
#endif
if (likely(page == c->page && c->node != NUMA_NO_NODE)) { if (likely(page == c->page)) {
set_freepointer(s, object, c->freelist); set_freepointer(s, object, c->freelist);
#ifdef CONFIG_CMPXCHG_LOCAL
if (unlikely(!irqsafe_cpu_cmpxchg_double( if (unlikely(!irqsafe_cpu_cmpxchg_double(
s->cpu_slab->freelist, s->cpu_slab->tid, s->cpu_slab->freelist, s->cpu_slab->tid,
c->freelist, tid, c->freelist, tid,
@ -2153,16 +2125,10 @@ redo:
note_cmpxchg_failure("slab_free", s, tid); note_cmpxchg_failure("slab_free", s, tid);
goto redo; goto redo;
} }
#else
c->freelist = object;
#endif
stat(s, FREE_FASTPATH); stat(s, FREE_FASTPATH);
} else } else
__slab_free(s, page, x, addr); __slab_free(s, page, x, addr);
#ifndef CONFIG_CMPXCHG_LOCAL
local_irq_restore(flags);
#endif
} }
void kmem_cache_free(struct kmem_cache *s, void *x) void kmem_cache_free(struct kmem_cache *s, void *x)
@ -2673,9 +2639,8 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
return; return;
slab_err(s, page, "%s", text); slab_err(s, page, "%s", text);
slab_lock(page); slab_lock(page);
for_each_free_object(p, s, page->freelist)
set_bit(slab_index(p, s, addr), map);
get_map(s, page, map);
for_each_object(p, s, addr, page->objects) { for_each_object(p, s, addr, page->objects) {
if (!test_bit(slab_index(p, s, addr), map)) { if (!test_bit(slab_index(p, s, addr), map)) {
@ -3203,7 +3168,7 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
list_for_each_entry(p, &n->partial, lru) list_for_each_entry(p, &n->partial, lru)
p->slab = s; p->slab = s;
#ifdef CONFIG_SLAB_DEBUG #ifdef CONFIG_SLUB_DEBUG
list_for_each_entry(p, &n->full, lru) list_for_each_entry(p, &n->full, lru)
p->slab = s; p->slab = s;
#endif #endif
@ -3610,10 +3575,11 @@ static int validate_slab(struct kmem_cache *s, struct page *page,
/* Now we know that a valid freelist exists */ /* Now we know that a valid freelist exists */
bitmap_zero(map, page->objects); bitmap_zero(map, page->objects);
for_each_free_object(p, s, page->freelist) { get_map(s, page, map);
set_bit(slab_index(p, s, addr), map); for_each_object(p, s, addr, page->objects) {
if (!check_object(s, page, p, SLUB_RED_INACTIVE)) if (test_bit(slab_index(p, s, addr), map))
return 0; if (!check_object(s, page, p, SLUB_RED_INACTIVE))
return 0;
} }
for_each_object(p, s, addr, page->objects) for_each_object(p, s, addr, page->objects)
@ -3821,8 +3787,7 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s,
void *p; void *p;
bitmap_zero(map, page->objects); bitmap_zero(map, page->objects);
for_each_free_object(p, s, page->freelist) get_map(s, page, map);
set_bit(slab_index(p, s, addr), map);
for_each_object(p, s, addr, page->objects) for_each_object(p, s, addr, page->objects)
if (!test_bit(slab_index(p, s, addr), map)) if (!test_bit(slab_index(p, s, addr), map))