Skip to content

Commit

Permalink
bcachefs: optimize __bch2_trans_get(), kill DEBUG_TRANSACTIONS
Browse files Browse the repository at this point in the history 
 - Some tweaks to greatly reduce locking overhead for the list of btree
   transactions, so that it can always be enabled: leave btree_trans
   objects on the list when they're on the percpu single item freelist,
   and only check for duplicates in the same process when
   CONFIG_BCACHEFS_DEBUG is enabled

 - don't zero out the full btree_trans() unless we allocated it from
   the mempool

Signed-off-by: Kent Overstreet <[email protected]>
  • Loading branch information
Kent Overstreet committed Jan 1, 2024
1 parent fea153a commit 31403dc
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 106 deletions.
8 changes: 0 additions & 8 deletions fs/bcachefs/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,6 @@ config BCACHEFS_POSIX_ACL
depends on BCACHEFS_FS
select FS_POSIX_ACL

config BCACHEFS_DEBUG_TRANSACTIONS
bool "bcachefs runtime info"
depends on BCACHEFS_FS
help
This makes the list of running btree transactions available in debugfs.

This is a highly useful debugging feature but does add a small amount of overhead.

config BCACHEFS_DEBUG
bool "bcachefs debugging"
depends on BCACHEFS_FS
Expand Down
166 changes: 82 additions & 84 deletions fs/bcachefs/btree_iter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2714,24 +2714,27 @@ void bch2_trans_copy_iter(struct btree_iter *dst, struct btree_iter *src)

void *__bch2_trans_kmalloc(struct btree_trans *trans, size_t size)
{
struct bch_fs *c = trans->c;
unsigned new_top = trans->mem_top + size;
unsigned old_bytes = trans->mem_bytes;
unsigned new_bytes = roundup_pow_of_two(new_top);
int ret;
void *new_mem;
void *p;

trans->mem_max = max(trans->mem_max, new_top);

WARN_ON_ONCE(new_bytes > BTREE_TRANS_MEM_MAX);

struct btree_transaction_stats *s = btree_trans_stats(trans);
if (s)
s->max_mem = max(s->max_mem, new_bytes);

new_mem = krealloc(trans->mem, new_bytes, GFP_NOWAIT|__GFP_NOWARN);
if (unlikely(!new_mem)) {
bch2_trans_unlock(trans);

new_mem = krealloc(trans->mem, new_bytes, GFP_KERNEL);
if (!new_mem && new_bytes <= BTREE_TRANS_MEM_MAX) {
new_mem = mempool_alloc(&trans->c->btree_trans_mem_pool, GFP_KERNEL);
new_mem = mempool_alloc(&c->btree_trans_mem_pool, GFP_KERNEL);
new_bytes = BTREE_TRANS_MEM_MAX;
kfree(trans->mem);
}
Expand All @@ -2751,7 +2754,7 @@ void *__bch2_trans_kmalloc(struct btree_trans *trans, size_t size)
trans->mem_bytes = new_bytes;

if (old_bytes) {
trace_and_count(trans->c, trans_restart_mem_realloced, trans, _RET_IP_, new_bytes);
trace_and_count(c, trans_restart_mem_realloced, trans, _RET_IP_, new_bytes);
return ERR_PTR(btree_trans_restart(trans, BCH_ERR_transaction_restart_mem_realloced));
}

Expand Down Expand Up @@ -2860,25 +2863,6 @@ u32 bch2_trans_begin(struct btree_trans *trans)
return trans->restart_count;
}

static struct btree_trans *bch2_trans_alloc(struct bch_fs *c)
{
struct btree_trans *trans;

if (IS_ENABLED(__KERNEL__)) {
trans = this_cpu_xchg(c->btree_trans_bufs->trans, NULL);
if (trans)
return trans;
}

trans = mempool_alloc(&c->btree_trans_pool, GFP_NOFS);
/*
* paths need to be zeroed, bch2_check_for_deadlock looks at
* paths in other threads
*/
memset(&trans->paths, 0, sizeof(trans->paths));
return trans;
}

const char *bch2_btree_transaction_fns[BCH_TRANSACTIONS_NR];

unsigned bch2_trans_get_fn_idx(const char *fn)
Expand All @@ -2900,22 +2884,55 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx)
__acquires(&c->btree_trans_barrier)
{
struct btree_trans *trans;
struct btree_transaction_stats *s;

trans = bch2_trans_alloc(c);
if (IS_ENABLED(__KERNEL__)) {
trans = this_cpu_xchg(c->btree_trans_bufs->trans, NULL);
if (trans) {
memset(trans, 0, offsetof(struct btree_trans, updates));
goto got_trans;
}
}

trans = mempool_alloc(&c->btree_trans_pool, GFP_NOFS);
memset(trans, 0, sizeof(*trans));
closure_init_stack(&trans->ref);

seqmutex_lock(&c->btree_trans_lock);
if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) {
struct btree_trans *pos;
pid_t pid = current->pid;

trans->locking_wait.task = current;

list_for_each_entry(pos, &c->btree_trans_list, list) {
struct task_struct *pos_task = READ_ONCE(pos->locking_wait.task);
/*
* We'd much prefer to be stricter here and completely
* disallow multiple btree_trans in the same thread -
* but the data move path calls bch2_write when we
* already have a btree_trans initialized.
*/
BUG_ON(pos_task &&
pid == pos_task->pid &&
bch2_trans_locked(pos));

if (pos_task && pid < pos_task->pid) {
list_add_tail(&trans->list, &pos->list);
goto list_add_done;
}
}
}
list_add_tail(&trans->list, &c->btree_trans_list);
list_add_done:
seqmutex_unlock(&c->btree_trans_lock);
got_trans:
trans->c = c;
trans->fn = fn_idx < ARRAY_SIZE(bch2_btree_transaction_fns)
? bch2_btree_transaction_fns[fn_idx] : NULL;
trans->last_begin_time = local_clock();
trans->fn_idx = fn_idx;
trans->locking_wait.task = current;
trans->journal_replay_not_finished =
unlikely(!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)) &&
atomic_inc_not_zero(&c->journal_keys.ref);
closure_init_stack(&trans->ref);

trans->paths_allocated = trans->_paths_allocated;
trans->sorted = trans->_sorted;
trans->paths = trans->_paths;
Expand All @@ -2924,53 +2941,26 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx)

trans->paths_allocated[0] = 1;

s = btree_trans_stats(trans);
if (s && s->max_mem) {
unsigned expected_mem_bytes = roundup_pow_of_two(s->max_mem);
if (fn_idx < BCH_TRANSACTIONS_NR) {
trans->fn = bch2_btree_transaction_fns[fn_idx];

trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL);
struct btree_transaction_stats *s = &c->btree_transaction_stats[fn_idx];

if (!unlikely(trans->mem)) {
trans->mem = mempool_alloc(&c->btree_trans_mem_pool, GFP_KERNEL);
trans->mem_bytes = BTREE_TRANS_MEM_MAX;
} else {
trans->mem_bytes = expected_mem_bytes;
if (s->max_mem) {
unsigned expected_mem_bytes = roundup_pow_of_two(s->max_mem);

trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL);
if (likely(trans->mem))
trans->mem_bytes = expected_mem_bytes;
}
}

if (s) {
trans->nr_paths_max = s->nr_max_paths;
trans->journal_entries_size = s->journal_entries_size;
}

trans->srcu_idx = srcu_read_lock(&c->btree_trans_barrier);
trans->srcu_lock_time = jiffies;
trans->srcu_held = true;

if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG_TRANSACTIONS)) {
struct btree_trans *pos;

seqmutex_lock(&c->btree_trans_lock);
list_for_each_entry(pos, &c->btree_trans_list, list) {
/*
* We'd much prefer to be stricter here and completely
* disallow multiple btree_trans in the same thread -
* but the data move path calls bch2_write when we
* already have a btree_trans initialized.
*/
BUG_ON(trans->locking_wait.task->pid == pos->locking_wait.task->pid &&
bch2_trans_locked(pos));

if (trans->locking_wait.task->pid < pos->locking_wait.task->pid) {
list_add_tail(&trans->list, &pos->list);
goto list_add_done;
}
}
list_add_tail(&trans->list, &c->btree_trans_list);
list_add_done:
seqmutex_unlock(&c->btree_trans_lock);
}

return trans;
}

Expand Down Expand Up @@ -3001,24 +2991,13 @@ void bch2_trans_put(struct btree_trans *trans)
__releases(&c->btree_trans_barrier)
{
struct bch_fs *c = trans->c;
struct btree_transaction_stats *s = btree_trans_stats(trans);

bch2_trans_unlock(trans);

if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG_TRANSACTIONS)) {
seqmutex_lock(&c->btree_trans_lock);
list_del(&trans->list);
seqmutex_unlock(&c->btree_trans_lock);
}

closure_sync(&trans->ref);

if (s)
s->max_mem = max(s->max_mem, trans->mem_max);

trans_for_each_update(trans, i)
__btree_path_put(trans->paths + i->path, true);
trans->nr_updates = 0;
trans->nr_updates = 0;
trans->locking_wait.task = NULL;

check_btree_paths_leaked(trans);

Expand Down Expand Up @@ -3047,8 +3026,16 @@ void bch2_trans_put(struct btree_trans *trans)
/* Userspace doesn't have a real percpu implementation: */
if (IS_ENABLED(__KERNEL__))
trans = this_cpu_xchg(c->btree_trans_bufs->trans, trans);
if (trans)

if (trans) {
closure_sync(&trans->ref);

seqmutex_lock(&c->btree_trans_lock);
list_del(&trans->list);
seqmutex_unlock(&c->btree_trans_lock);

mempool_free(trans, &c->btree_trans_pool);
}
}

static void __maybe_unused
Expand Down Expand Up @@ -3146,15 +3133,26 @@ void bch2_fs_btree_iter_exit(struct bch_fs *c)
struct btree_trans *trans;
int cpu;

if (c->btree_trans_bufs)
for_each_possible_cpu(cpu) {
struct btree_trans *trans =
per_cpu_ptr(c->btree_trans_bufs, cpu)->trans;

if (trans) {
closure_sync(&trans->ref);

seqmutex_lock(&c->btree_trans_lock);
list_del(&trans->list);
seqmutex_unlock(&c->btree_trans_lock);
}
kfree(trans);
}
free_percpu(c->btree_trans_bufs);

trans = list_first_entry_or_null(&c->btree_trans_list, struct btree_trans, list);
if (trans)
panic("%s leaked btree_trans\n", trans->fn);

if (c->btree_trans_bufs)
for_each_possible_cpu(cpu)
kfree(per_cpu_ptr(c->btree_trans_bufs, cpu)->trans);
free_percpu(c->btree_trans_bufs);

for (s = c->btree_transaction_stats;
s < c->btree_transaction_stats + ARRAY_SIZE(c->btree_transaction_stats);
s++) {
Expand Down
3 changes: 2 additions & 1 deletion fs/bcachefs/btree_locking.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,10 @@ static noinline void print_chain(struct printbuf *out, struct lock_graph *g)
struct trans_waiting_for_lock *i;

for (i = g->g; i != g->g + g->nr; i++) {
struct task_struct *task = i->trans->locking_wait.task;
if (i != g->g)
prt_str(out, "<- ");
prt_printf(out, "%u ", i->trans->locking_wait.task->pid);
prt_printf(out, "%u ", task ?task->pid : 0);
}
prt_newline(out);
}
Expand Down
11 changes: 7 additions & 4 deletions fs/bcachefs/btree_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,6 @@ struct btree_trans {

void *mem;
unsigned mem_top;
unsigned mem_max;
unsigned mem_bytes;

btree_path_idx_t nr_sorted;
Expand All @@ -413,8 +412,6 @@ struct btree_trans {
unsigned long srcu_lock_time;

const char *fn;
struct closure ref;
struct list_head list;
struct btree_bkey_cached_common *locking;
struct six_lock_waiter locking_wait;
int srcu_idx;
Expand All @@ -424,7 +421,6 @@ struct btree_trans {
u16 journal_entries_size;
struct jset_entry *journal_entries;

struct btree_insert_entry updates[BTREE_ITER_MAX];
struct btree_trans_commit_hook *hooks;
struct journal_entry_pin *journal_pin;

Expand All @@ -435,6 +431,13 @@ struct btree_trans {
unsigned extra_disk_res; /* XXX kill */
struct replicas_delta_list *fs_usage_deltas;

/* Entries before this are zeroed out on every bch2_trans_get() call */

struct btree_insert_entry updates[BTREE_ITER_MAX];

struct list_head list;
struct closure ref;

unsigned long _paths_allocated[BITS_TO_LONGS(BTREE_ITER_MAX)];
struct btree_trans_paths trans_paths;
struct btree_path _paths[BTREE_ITER_MAX];
Expand Down
Loading

0 comments on commit 31403dc

Please sign in to comment.