Skip to content

Commit 4a57609

Browse files
ryaoryao
committed
Allocate zfs_locked_range_t memory externally from zfs_rangelock_{try,}enter()
Typically, the memory allocated by kmem_alloc() can be trivially allocated from either the stack or as part of another structure. The only case where it cannot is in vdev_raidz_io_start(), although some further refactoring should be able to eliminate that case too. Allocating from the stack or as part of another data structure is faster as it gives us this memory for free, so there is little reason not to do it. This eliminates a non-neligible amount of CPU time that I have seen in flame graphs going back to the early days of OpenZFS when the tree was the ZFSOnLinux tree. This should make our VFS and zvol operations slightly faster. Some RAID-Z operations will also become slightly faster. Signed-off-by: Richard Yao <[email protected]>
1 parent 20c8bdd commit 4a57609

File tree

13 files changed

+182
-184
lines changed

13 files changed

+182
-184
lines changed

‎cmd/ztest.c

+17-23
Original file line numberDiff line numberDiff line change
@@ -1736,23 +1736,19 @@ ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
17361736
ztest_rll_unlock(rll);
17371737
}
17381738

1739-
static rl_t *
1740-
ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1739+
static void
1740+
ztest_range_lock(ztest_ds_t *zd, rl_t *rl, uint64_t object, uint64_t offset,
17411741
uint64_t size, rl_type_t type)
17421742
{
17431743
uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
17441744
rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1745-
rl_t *rl;
17461745

1747-
rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
17481746
rl->rl_object = object;
17491747
rl->rl_offset = offset;
17501748
rl->rl_size = size;
17511749
rl->rl_lock = rll;
17521750

17531751
ztest_rll_lock(rll, type);
1754-
1755-
return (rl);
17561752
}
17571753

17581754
static void
@@ -1761,8 +1757,6 @@ ztest_range_unlock(rl_t *rl)
17611757
rll_t *rll = rl->rl_lock;
17621758

17631759
ztest_rll_unlock(rll);
1764-
1765-
umem_free(rl, sizeof (*rl));
17661760
}
17671761

17681762
static void
@@ -2190,7 +2184,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
21902184
dmu_tx_t *tx;
21912185
dmu_buf_t *db;
21922186
arc_buf_t *abuf = NULL;
2193-
rl_t *rl;
2187+
rl_t rl;
21942188

21952189
if (byteswap)
21962190
byteswap_uint64_array(lr, sizeof (*lr));
@@ -2214,7 +2208,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22142208
bt = NULL;
22152209

22162210
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2217-
rl = ztest_range_lock(zd, lr->lr_foid, offset, length, ZTRL_WRITER);
2211+
ztest_range_lock(zd, &rl, lr->lr_foid, offset, length, ZTRL_WRITER);
22182212

22192213
VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
22202214

@@ -2239,7 +2233,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22392233
if (abuf != NULL)
22402234
dmu_return_arcbuf(abuf);
22412235
dmu_buf_rele(db, FTAG);
2242-
ztest_range_unlock(rl);
2236+
ztest_range_unlock(&rl);
22432237
ztest_object_unlock(zd, lr->lr_foid);
22442238
return (ENOSPC);
22452239
}
@@ -2298,7 +2292,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22982292

22992293
dmu_tx_commit(tx);
23002294

2301-
ztest_range_unlock(rl);
2295+
ztest_range_unlock(&rl);
23022296
ztest_object_unlock(zd, lr->lr_foid);
23032297

23042298
return (0);
@@ -2312,13 +2306,13 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23122306
objset_t *os = zd->zd_os;
23132307
dmu_tx_t *tx;
23142308
uint64_t txg;
2315-
rl_t *rl;
2309+
rl_t rl;
23162310

23172311
if (byteswap)
23182312
byteswap_uint64_array(lr, sizeof (*lr));
23192313

23202314
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2321-
rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
2315+
ztest_range_lock(zd, &rl, lr->lr_foid, lr->lr_offset, lr->lr_length,
23222316
ZTRL_WRITER);
23232317

23242318
tx = dmu_tx_create(os);
@@ -2327,7 +2321,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23272321

23282322
txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
23292323
if (txg == 0) {
2330-
ztest_range_unlock(rl);
2324+
ztest_range_unlock(&rl);
23312325
ztest_object_unlock(zd, lr->lr_foid);
23322326
return (ENOSPC);
23332327
}
@@ -2339,7 +2333,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23392333

23402334
dmu_tx_commit(tx);
23412335

2342-
ztest_range_unlock(rl);
2336+
ztest_range_unlock(&rl);
23432337
ztest_object_unlock(zd, lr->lr_foid);
23442338

23452339
return (0);
@@ -2455,12 +2449,12 @@ ztest_get_done(zgd_t *zgd, int error)
24552449
{
24562450
(void) error;
24572451
ztest_ds_t *zd = zgd->zgd_private;
2458-
uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2452+
uint64_t object = ((rl_t *)&zgd->zgd_lr)->rl_object;
24592453

24602454
if (zgd->zgd_db)
24612455
dmu_buf_rele(zgd->zgd_db, zgd);
24622456

2463-
ztest_range_unlock((rl_t *)zgd->zgd_lr);
2457+
ztest_range_unlock((rl_t *)&zgd->zgd_lr);
24642458
ztest_object_unlock(zd, object);
24652459

24662460
umem_free(zgd, sizeof (*zgd));
@@ -2510,7 +2504,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25102504
zgd->zgd_private = zd;
25112505

25122506
if (buf != NULL) { /* immediate write */
2513-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2507+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25142508
object, offset, size, ZTRL_READER);
25152509

25162510
error = dmu_read(os, object, offset, size, buf,
@@ -2526,7 +2520,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25262520
offset = 0;
25272521
}
25282522

2529-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2523+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25302524
object, offset, size, ZTRL_READER);
25312525

25322526
error = dmu_buf_hold_noread(os, object, offset, zgd, &db);
@@ -2772,12 +2766,12 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
27722766
objset_t *os = zd->zd_os;
27732767
dmu_tx_t *tx;
27742768
uint64_t txg;
2775-
rl_t *rl;
2769+
rl_t rl;
27762770

27772771
txg_wait_synced(dmu_objset_pool(os), 0);
27782772

27792773
ztest_object_lock(zd, object, ZTRL_READER);
2780-
rl = ztest_range_lock(zd, object, offset, size, ZTRL_WRITER);
2774+
ztest_range_lock(zd, &rl, object, offset, size, ZTRL_WRITER);
27812775

27822776
tx = dmu_tx_create(os);
27832777

@@ -2793,7 +2787,7 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
27932787
(void) dmu_free_long_range(os, object, offset, size);
27942788
}
27952789

2796-
ztest_range_unlock(rl);
2790+
ztest_range_unlock(&rl);
27972791
ztest_object_unlock(zd, object);
27982792
}
27992793

‎include/sys/dmu.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
#include <sys/fs/zfs.h>
5050
#include <sys/zio_compress.h>
5151
#include <sys/zio_priority.h>
52+
#include <sys/zfs_rlock.h>
5253
#include <sys/uio.h>
5354
#include <sys/zfs_file.h>
5455

@@ -1069,8 +1070,8 @@ typedef struct zgd {
10691070
struct lwb *zgd_lwb;
10701071
struct blkptr *zgd_bp;
10711072
dmu_buf_t *zgd_db;
1072-
struct zfs_locked_range *zgd_lr;
10731073
void *zgd_private;
1074+
zfs_locked_range_t zgd_lr;
10741075
} zgd_t;
10751076

10761077
typedef void dmu_sync_cb_t(zgd_t *arg, int error);

‎include/sys/zfs_rlock.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ typedef struct zfs_locked_range {
6969
void zfs_rangelock_init(zfs_rangelock_t *, zfs_rangelock_cb_t *, void *);
7070
void zfs_rangelock_fini(zfs_rangelock_t *);
7171

72-
zfs_locked_range_t *zfs_rangelock_enter(zfs_rangelock_t *,
72+
void zfs_rangelock_enter(zfs_rangelock_t *, zfs_locked_range_t *,
7373
uint64_t, uint64_t, zfs_rangelock_type_t);
74-
zfs_locked_range_t *zfs_rangelock_tryenter(zfs_rangelock_t *,
74+
boolean_t zfs_rangelock_tryenter(zfs_rangelock_t *, zfs_locked_range_t *,
7575
uint64_t, uint64_t, zfs_rangelock_type_t);
7676
void zfs_rangelock_exit(zfs_locked_range_t *);
7777
void zfs_rangelock_reduce(zfs_locked_range_t *, uint64_t, uint64_t);

‎module/os/freebsd/zfs/zfs_vnops_os.c

+12-11
Original file line numberDiff line numberDiff line change
@@ -4055,7 +4055,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40554055
{
40564056
znode_t *zp = VTOZ(vp);
40574057
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
4058-
zfs_locked_range_t *lr;
4058+
zfs_locked_range_t lr;
4059+
boolean_t res;
40594060
vm_object_t object;
40604061
off_t start, end, obj_size;
40614062
uint_t blksz;
@@ -4074,10 +4075,10 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40744075
*/
40754076
for (;;) {
40764077
blksz = zp->z_blksz;
4077-
lr = zfs_rangelock_tryenter(&zp->z_rangelock,
4078+
res = zfs_rangelock_tryenter(&zp->z_rangelock, &lr,
40784079
rounddown(start, blksz),
40794080
roundup(end, blksz) - rounddown(start, blksz), RL_READER);
4080-
if (lr == NULL) {
4081+
if (res == B_FALSE) {
40814082
if (rahead != NULL) {
40824083
*rahead = 0;
40834084
rahead = NULL;
@@ -4090,16 +4091,16 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40904091
}
40914092
if (blksz == zp->z_blksz)
40924093
break;
4093-
zfs_rangelock_exit(lr);
4094+
zfs_rangelock_exit(&lr);
40944095
}
40954096

40964097
object = ma[0]->object;
40974098
zfs_vmobject_wlock(object);
40984099
obj_size = object->un_pager.vnp.vnp_size;
40994100
zfs_vmobject_wunlock(object);
41004101
if (IDX_TO_OFF(ma[count - 1]->pindex) >= obj_size) {
4101-
if (lr != NULL)
4102-
zfs_rangelock_exit(lr);
4102+
if (res == B_TRUE)
4103+
zfs_rangelock_exit(&lr);
41034104
zfs_exit(zfsvfs, FTAG);
41044105
return (zfs_vm_pagerret_bad);
41054106
}
@@ -4127,8 +4128,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
41274128
error = dmu_read_pages(zfsvfs->z_os, zp->z_id, ma, count, &pgsin_b,
41284129
&pgsin_a, MIN(end, obj_size) - (end - PAGE_SIZE));
41294130

4130-
if (lr != NULL)
4131-
zfs_rangelock_exit(lr);
4131+
if (res == B_TRUE)
4132+
zfs_rangelock_exit(&lr);
41324133
ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
41334134

41344135
dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, count*PAGE_SIZE);
@@ -4171,7 +4172,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
41714172
{
41724173
znode_t *zp = VTOZ(vp);
41734174
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
4174-
zfs_locked_range_t *lr;
4175+
zfs_locked_range_t lr;
41754176
dmu_tx_t *tx;
41764177
struct sf_buf *sf;
41774178
vm_object_t object;
@@ -4203,7 +4204,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
42034204
blksz = zp->z_blksz;
42044205
lo_off = rounddown(off, blksz);
42054206
lo_len = roundup(len + (off - lo_off), blksz);
4206-
lr = zfs_rangelock_enter(&zp->z_rangelock, lo_off, lo_len, RL_WRITER);
4207+
zfs_rangelock_enter(&zp->z_rangelock, &lr, lo_off, lo_len, RL_WRITER);
42074208

42084209
zfs_vmobject_wlock(object);
42094210
if (len + off > object->un_pager.vnp.vnp_size) {
@@ -4309,7 +4310,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
43094310
dmu_tx_commit(tx);
43104311

43114312
out:
4312-
zfs_rangelock_exit(lr);
4313+
zfs_rangelock_exit(&lr);
43134314
if (commit)
43144315
zil_commit(zfsvfs->z_log, zp->z_id);
43154316

‎module/os/freebsd/zfs/zfs_znode.c

+17-15
Original file line numberDiff line numberDiff line change
@@ -1393,20 +1393,21 @@ zfs_extend(znode_t *zp, uint64_t end)
13931393
{
13941394
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
13951395
dmu_tx_t *tx;
1396-
zfs_locked_range_t *lr;
1396+
zfs_locked_range_t lr;
13971397
uint64_t newblksz;
13981398
int error;
13991399

14001400
/*
14011401
* We will change zp_size, lock the whole file.
14021402
*/
1403-
lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
1403+
lr = zfs_rangelock_enter(&zp->z_rangelock, &lr, 0, UINT64_MAX,
1404+
RL_WRITER);
14041405

14051406
/*
14061407
* Nothing to do if file already at desired length.
14071408
*/
14081409
if (end <= zp->z_size) {
1409-
zfs_rangelock_exit(lr);
1410+
zfs_rangelock_exit(&lr);
14101411
return (0);
14111412
}
14121413
tx = dmu_tx_create(zfsvfs->z_os);
@@ -1436,7 +1437,7 @@ zfs_extend(znode_t *zp, uint64_t end)
14361437
error = dmu_tx_assign(tx, TXG_WAIT);
14371438
if (error) {
14381439
dmu_tx_abort(tx);
1439-
zfs_rangelock_exit(lr);
1440+
zfs_rangelock_exit(&lr);
14401441
return (error);
14411442
}
14421443

@@ -1450,7 +1451,7 @@ zfs_extend(znode_t *zp, uint64_t end)
14501451

14511452
vnode_pager_setsize(ZTOV(zp), end);
14521453

1453-
zfs_rangelock_exit(lr);
1454+
zfs_rangelock_exit(&lr);
14541455

14551456
dmu_tx_commit(tx);
14561457

@@ -1470,19 +1471,19 @@ static int
14701471
zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
14711472
{
14721473
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1473-
zfs_locked_range_t *lr;
1474+
zfs_locked_range_t lr;
14741475
int error;
14751476

14761477
/*
14771478
* Lock the range being freed.
14781479
*/
1479-
lr = zfs_rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER);
1480+
lr = zfs_rangelock_enter(&zp->z_rangelock, &lr, off, len, RL_WRITER);
14801481

14811482
/*
14821483
* Nothing to do if file already at desired length.
14831484
*/
14841485
if (off >= zp->z_size) {
1485-
zfs_rangelock_exit(lr);
1486+
zfs_rangelock_exit(&lr);
14861487
return (0);
14871488
}
14881489

@@ -1504,7 +1505,7 @@ zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
15041505
#endif
15051506
}
15061507

1507-
zfs_rangelock_exit(lr);
1508+
zfs_rangelock_exit(&lr);
15081509

15091510
return (error);
15101511
}
@@ -1523,28 +1524,29 @@ zfs_trunc(znode_t *zp, uint64_t end)
15231524
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
15241525
vnode_t *vp = ZTOV(zp);
15251526
dmu_tx_t *tx;
1526-
zfs_locked_range_t *lr;
1527+
zfs_locked_range_t lr;
15271528
int error;
15281529
sa_bulk_attr_t bulk[2];
15291530
int count = 0;
15301531

15311532
/*
15321533
* We will change zp_size, lock the whole file.
15331534
*/
1534-
lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
1535+
lr = zfs_rangelock_enter(&zp->z_rangelock, &lr, 0, UINT64_MAX,
1536+
RL_WRITER);
15351537

15361538
/*
15371539
* Nothing to do if file already at desired length.
15381540
*/
15391541
if (end >= zp->z_size) {
1540-
zfs_rangelock_exit(lr);
1542+
zfs_rangelock_exit(&lr);
15411543
return (0);
15421544
}
15431545

15441546
error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, end,
15451547
DMU_OBJECT_END);
15461548
if (error) {
1547-
zfs_rangelock_exit(lr);
1549+
zfs_rangelock_exit(&lr);
15481550
return (error);
15491551
}
15501552
tx = dmu_tx_create(zfsvfs->z_os);
@@ -1554,7 +1556,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
15541556
error = dmu_tx_assign(tx, TXG_WAIT);
15551557
if (error) {
15561558
dmu_tx_abort(tx);
1557-
zfs_rangelock_exit(lr);
1559+
zfs_rangelock_exit(&lr);
15581560
return (error);
15591561
}
15601562

@@ -1579,7 +1581,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
15791581
*/
15801582
vnode_pager_setsize(vp, end);
15811583

1582-
zfs_rangelock_exit(lr);
1584+
zfs_rangelock_exit(&lr);
15831585

15841586
return (0);
15851587
}

0 commit comments

Comments
 (0)