From ff8d523cc4520a5ce86cde0fd57c304e2b4f61b3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 13 Oct 2024 20:45:57 +0200 Subject: [PATCH] debugobjects: Track object usage to avoid premature freeing of objects The freelist is freed at a constant rate independent of the actual usage requirements. That's bad in scenarios where usage comes in bursts. The end of a burst puts the objects on the free list and freeing proceeds even when the next burst which requires objects started again. Keep track of the usage with a exponentially wheighted moving average and take that into account in the worker function which frees objects from the free list. This further reduces the kmem_cache allocation/free rate for a full kernel compile: kmem_cache_alloc() kmem_cache_free() Baseline: 225k 173k Usage: 170k 117k Signed-off-by: Thomas Gleixner Reviewed-by: Zhen Lei Link: https://lore.kernel.org/all/87bjznhme2.ffs@tglx --- lib/debugobjects.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/lib/debugobjects.c b/lib/debugobjects.c index cc32844ca18d..7f50c4480a4e 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +87,7 @@ static struct obj_pool pool_to_free = { static HLIST_HEAD(pool_boot); +static unsigned long avg_usage; static bool obj_freeing; static int __data_racy debug_objects_maxchain __read_mostly; @@ -427,11 +429,31 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) return NULL; } +static void calc_usage(void) +{ + static DEFINE_RAW_SPINLOCK(avg_lock); + static unsigned long avg_period; + unsigned long cur, now = jiffies; + + if (!time_after_eq(now, READ_ONCE(avg_period))) + return; + + if (!raw_spin_trylock(&avg_lock)) + return; + + WRITE_ONCE(avg_period, now + msecs_to_jiffies(10)); + cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX; + WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur)); + raw_spin_unlock(&avg_lock); +} + static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr) { struct debug_obj *obj; + calc_usage(); + if (static_branch_likely(&obj_cache_enabled)) obj = pcpu_alloc(); else @@ -450,14 +472,26 @@ static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, /* workqueue function to free objects. */ static void free_obj_work(struct work_struct *work) { - bool free = true; + static unsigned long last_use_avg; + unsigned long cur_used, last_used, delta; + unsigned int max_free = 0; WRITE_ONCE(obj_freeing, false); + /* Rate limit freeing based on current use average */ + cur_used = READ_ONCE(avg_usage); + last_used = last_use_avg; + last_use_avg = cur_used; + if (!pool_count(&pool_to_free)) return; - for (unsigned int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { + if (cur_used <= last_used) { + delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX; + max_free = min(delta, ODEBUG_FREE_WORK_MAX); + } + + for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { HLIST_HEAD(tofree); /* Acquire and drop the lock for each batch */ @@ -468,9 +502,10 @@ static void free_obj_work(struct work_struct *work) /* Refill the global pool if possible */ if (pool_move_batch(&pool_global, &pool_to_free)) { /* Don't free as there seems to be demand */ - free = false; - } else if (free) { + max_free = 0; + } else if (max_free) { pool_pop_batch(&tofree, &pool_to_free); + max_free--; } else { return; } @@ -1110,7 +1145,7 @@ static int debug_stats_show(struct seq_file *m, void *v) for_each_possible_cpu(cpu) pcp_free += per_cpu(pool_pcpu.cnt, cpu); - pool_used = data_race(pool_global.stats.cur_used); + pool_used = READ_ONCE(pool_global.stats.cur_used); pcp_free = min(pool_used, pcp_free); pool_used -= pcp_free;