Skip to content

Commit 88cddea

Browse files
committed
Allocate zfs_locked_range_t memory externally from zfs_rangelock_{try,}enter()
Typically, the memory allocated by kmem_alloc() can be trivially allocated from either the stack or as part of another structure. The only case where it cannot is in vdev_raidz_io_start(), although some further refactoring should be able to eliminate that case too. Allocating from the stack or as part of another data structure is faster as it gives us this memory for free, so there is little reason not to do it. This eliminates a non-neligible amount of CPU time that I have seen in flame graphs going back to the early days of OpenZFS when the tree was the ZFSOnLinux tree. This should make our VFS and zvol operations slightly faster. Some RAID-Z operations will also become slightly faster. Signed-off-by: Richard Yao <[email protected]>
1 parent 1acd246 commit 88cddea

13 files changed

+188
-190
lines changed

cmd/ztest.c

+17-23
Original file line numberDiff line numberDiff line change
@@ -1745,23 +1745,19 @@ ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
17451745
ztest_rll_unlock(rll);
17461746
}
17471747

1748-
static rl_t *
1749-
ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1748+
static void
1749+
ztest_range_lock(ztest_ds_t *zd, rl_t *rl, uint64_t object, uint64_t offset,
17501750
uint64_t size, rl_type_t type)
17511751
{
17521752
uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
17531753
rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1754-
rl_t *rl;
17551754

1756-
rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
17571755
rl->rl_object = object;
17581756
rl->rl_offset = offset;
17591757
rl->rl_size = size;
17601758
rl->rl_lock = rll;
17611759

17621760
ztest_rll_lock(rll, type);
1763-
1764-
return (rl);
17651761
}
17661762

17671763
static void
@@ -1770,8 +1766,6 @@ ztest_range_unlock(rl_t *rl)
17701766
rll_t *rll = rl->rl_lock;
17711767

17721768
ztest_rll_unlock(rll);
1773-
1774-
umem_free(rl, sizeof (*rl));
17751769
}
17761770

17771771
static void
@@ -2200,7 +2194,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22002194
dmu_tx_t *tx;
22012195
dmu_buf_t *db;
22022196
arc_buf_t *abuf = NULL;
2203-
rl_t *rl;
2197+
rl_t rl;
22042198

22052199
if (byteswap)
22062200
byteswap_uint64_array(lr, sizeof (*lr));
@@ -2224,7 +2218,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22242218
bt = NULL;
22252219

22262220
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2227-
rl = ztest_range_lock(zd, lr->lr_foid, offset, length, ZTRL_WRITER);
2221+
ztest_range_lock(zd, &rl, lr->lr_foid, offset, length, ZTRL_WRITER);
22282222

22292223
VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
22302224

@@ -2249,7 +2243,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22492243
if (abuf != NULL)
22502244
dmu_return_arcbuf(abuf);
22512245
dmu_buf_rele(db, FTAG);
2252-
ztest_range_unlock(rl);
2246+
ztest_range_unlock(&rl);
22532247
ztest_object_unlock(zd, lr->lr_foid);
22542248
return (ENOSPC);
22552249
}
@@ -2315,7 +2309,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
23152309

23162310
dmu_tx_commit(tx);
23172311

2318-
ztest_range_unlock(rl);
2312+
ztest_range_unlock(&rl);
23192313
ztest_object_unlock(zd, lr->lr_foid);
23202314

23212315
return (0);
@@ -2329,13 +2323,13 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23292323
objset_t *os = zd->zd_os;
23302324
dmu_tx_t *tx;
23312325
uint64_t txg;
2332-
rl_t *rl;
2326+
rl_t rl;
23332327

23342328
if (byteswap)
23352329
byteswap_uint64_array(lr, sizeof (*lr));
23362330

23372331
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2338-
rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
2332+
ztest_range_lock(zd, &rl, lr->lr_foid, lr->lr_offset, lr->lr_length,
23392333
ZTRL_WRITER);
23402334

23412335
tx = dmu_tx_create(os);
@@ -2344,7 +2338,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23442338

23452339
txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
23462340
if (txg == 0) {
2347-
ztest_range_unlock(rl);
2341+
ztest_range_unlock(&rl);
23482342
ztest_object_unlock(zd, lr->lr_foid);
23492343
return (ENOSPC);
23502344
}
@@ -2356,7 +2350,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23562350

23572351
dmu_tx_commit(tx);
23582352

2359-
ztest_range_unlock(rl);
2353+
ztest_range_unlock(&rl);
23602354
ztest_object_unlock(zd, lr->lr_foid);
23612355

23622356
return (0);
@@ -2472,12 +2466,12 @@ ztest_get_done(zgd_t *zgd, int error)
24722466
{
24732467
(void) error;
24742468
ztest_ds_t *zd = zgd->zgd_private;
2475-
uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2469+
uint64_t object = ((rl_t *)&zgd->zgd_lr)->rl_object;
24762470

24772471
if (zgd->zgd_db)
24782472
dmu_buf_rele(zgd->zgd_db, zgd);
24792473

2480-
ztest_range_unlock((rl_t *)zgd->zgd_lr);
2474+
ztest_range_unlock((rl_t *)&zgd->zgd_lr);
24812475
ztest_object_unlock(zd, object);
24822476

24832477
umem_free(zgd, sizeof (*zgd));
@@ -2527,7 +2521,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25272521
zgd->zgd_private = zd;
25282522

25292523
if (buf != NULL) { /* immediate write */
2530-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2524+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25312525
object, offset, size, ZTRL_READER);
25322526

25332527
error = dmu_read(os, object, offset, size, buf,
@@ -2543,7 +2537,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25432537
offset = 0;
25442538
}
25452539

2546-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2540+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25472541
object, offset, size, ZTRL_READER);
25482542

25492543
error = dmu_buf_hold_noread(os, object, offset, zgd, &db);
@@ -2790,12 +2784,12 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
27902784
objset_t *os = zd->zd_os;
27912785
dmu_tx_t *tx;
27922786
uint64_t txg;
2793-
rl_t *rl;
2787+
rl_t rl;
27942788

27952789
txg_wait_synced(dmu_objset_pool(os), 0);
27962790

27972791
ztest_object_lock(zd, object, ZTRL_READER);
2798-
rl = ztest_range_lock(zd, object, offset, size, ZTRL_WRITER);
2792+
ztest_range_lock(zd, &rl, object, offset, size, ZTRL_WRITER);
27992793

28002794
tx = dmu_tx_create(os);
28012795

@@ -2811,7 +2805,7 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
28112805
(void) dmu_free_long_range(os, object, offset, size);
28122806
}
28132807

2814-
ztest_range_unlock(rl);
2808+
ztest_range_unlock(&rl);
28152809
ztest_object_unlock(zd, object);
28162810
}
28172811

include/sys/dmu.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
#include <sys/fs/zfs.h>
5050
#include <sys/zio_compress.h>
5151
#include <sys/zio_priority.h>
52+
#include <sys/zfs_rlock.h>
5253
#include <sys/uio.h>
5354
#include <sys/zfs_file.h>
5455

@@ -1083,10 +1084,10 @@ uint64_t dmu_tx_get_txg(dmu_tx_t *tx);
10831084
* {zfs,zvol,ztest}_get_done() args
10841085
*/
10851086
typedef struct zgd {
1087+
zfs_locked_range_t zgd_lr;
10861088
struct lwb *zgd_lwb;
10871089
struct blkptr *zgd_bp;
10881090
dmu_buf_t *zgd_db;
1089-
struct zfs_locked_range *zgd_lr;
10901091
void *zgd_private;
10911092
} zgd_t;
10921093

include/sys/zfs_rlock.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ typedef struct zfs_locked_range {
6969
void zfs_rangelock_init(zfs_rangelock_t *, zfs_rangelock_cb_t *, void *);
7070
void zfs_rangelock_fini(zfs_rangelock_t *);
7171

72-
zfs_locked_range_t *zfs_rangelock_enter(zfs_rangelock_t *,
72+
void zfs_rangelock_enter(zfs_rangelock_t *, zfs_locked_range_t *,
7373
uint64_t, uint64_t, zfs_rangelock_type_t);
74-
zfs_locked_range_t *zfs_rangelock_tryenter(zfs_rangelock_t *,
74+
boolean_t zfs_rangelock_tryenter(zfs_rangelock_t *, zfs_locked_range_t *,
7575
uint64_t, uint64_t, zfs_rangelock_type_t);
7676
void zfs_rangelock_exit(zfs_locked_range_t *);
7777
void zfs_rangelock_reduce(zfs_locked_range_t *, uint64_t, uint64_t);

module/os/freebsd/zfs/zfs_vnops_os.c

+11-10
Original file line numberDiff line numberDiff line change
@@ -3924,7 +3924,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
39243924
{
39253925
znode_t *zp = VTOZ(vp);
39263926
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3927-
zfs_locked_range_t *lr;
3927+
zfs_locked_range_t lr;
3928+
boolean_t res;
39283929
vm_object_t object;
39293930
off_t start, end, obj_size;
39303931
uint_t blksz;
@@ -3948,9 +3949,9 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
39483949
blksz = zp->z_blksz;
39493950
len = roundup(end, blksz) - rounddown(start, blksz);
39503951

3951-
lr = zfs_rangelock_tryenter(&zp->z_rangelock,
3952+
res = zfs_rangelock_tryenter(&zp->z_rangelock, &lr,
39523953
rounddown(start, blksz), len, RL_READER);
3953-
if (lr == NULL) {
3954+
if (res == B_FALSE) {
39543955
/*
39553956
* Avoid a deadlock with update_pages(). We need to
39563957
* hold the range lock when copying from the DMU, so
@@ -3963,7 +3964,7 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
39633964
for (int i = 0; i < count; i++)
39643965
vm_page_xunbusy(ma[i]);
39653966

3966-
lr = zfs_rangelock_enter(&zp->z_rangelock,
3967+
zfs_rangelock_enter(&zp->z_rangelock, &lr,
39673968
rounddown(start, blksz), len, RL_READER);
39683969

39693970
zfs_vmobject_wlock(object);
@@ -3974,14 +3975,14 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
39743975
}
39753976
if (blksz == zp->z_blksz)
39763977
break;
3977-
zfs_rangelock_exit(lr);
3978+
zfs_rangelock_exit(&lr);
39783979
}
39793980

39803981
zfs_vmobject_wlock(object);
39813982
obj_size = object->un_pager.vnp.vnp_size;
39823983
zfs_vmobject_wunlock(object);
39833984
if (IDX_TO_OFF(ma[count - 1]->pindex) >= obj_size) {
3984-
zfs_rangelock_exit(lr);
3985+
zfs_rangelock_exit(&lr);
39853986
zfs_exit(zfsvfs, FTAG);
39863987
return (zfs_vm_pagerret_bad);
39873988
}
@@ -4032,7 +4033,7 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40324033
i += count1 - 1;
40334034
}
40344035

4035-
zfs_rangelock_exit(lr);
4036+
zfs_rangelock_exit(&lr);
40364037
ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
40374038

40384039
dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, count*PAGE_SIZE);
@@ -4075,7 +4076,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
40754076
{
40764077
znode_t *zp = VTOZ(vp);
40774078
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
4078-
zfs_locked_range_t *lr;
4079+
zfs_locked_range_t lr;
40794080
dmu_tx_t *tx;
40804081
struct sf_buf *sf;
40814082
vm_object_t object;
@@ -4107,7 +4108,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
41074108
blksz = zp->z_blksz;
41084109
lo_off = rounddown(off, blksz);
41094110
lo_len = roundup(len + (off - lo_off), blksz);
4110-
lr = zfs_rangelock_enter(&zp->z_rangelock, lo_off, lo_len, RL_WRITER);
4111+
zfs_rangelock_enter(&zp->z_rangelock, &lr, lo_off, lo_len, RL_WRITER);
41114112

41124113
zfs_vmobject_wlock(object);
41134114
if (len + off > object->un_pager.vnp.vnp_size) {
@@ -4213,7 +4214,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
42134214
dmu_tx_commit(tx);
42144215

42154216
out:
4216-
zfs_rangelock_exit(lr);
4217+
zfs_rangelock_exit(&lr);
42174218
if (commit)
42184219
zil_commit(zfsvfs->z_log, zp->z_id);
42194220

0 commit comments

Comments
 (0)