Skip to content

Commit df82974

Browse files
ryaoryao
committed
Allocate zfs_locked_range_t memory externally from zfs_rangelock_{try,}enter()
Typically, the memory allocated by kmem_alloc() can be trivially allocated from either the stack or as part of another structure. The only case where it cannot is in vdev_raidz_io_start(), although some further refactoring should be able to eliminate that case too. Allocating from the stack or as part of another data structure is faster as it gives us this memory for free, so there is little reason not to do it. This eliminates a non-neligible amount of CPU time that I have seen in flame graphs going back to the early days of OpenZFS when the tree was the ZFSOnLinux tree. This should make our VFS and zvol operations slightly faster. Some RAID-Z operations will also become slightly faster. Signed-off-by: Richard Yao <[email protected]>
1 parent 20c8bdd commit df82974

File tree

13 files changed

+180
-184
lines changed

13 files changed

+180
-184
lines changed

‎cmd/ztest.c

+17-23
Original file line numberDiff line numberDiff line change
@@ -1736,23 +1736,19 @@ ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
17361736
ztest_rll_unlock(rll);
17371737
}
17381738

1739-
static rl_t *
1740-
ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1739+
static void
1740+
ztest_range_lock(ztest_ds_t *zd, rl_t *rl, uint64_t object, uint64_t offset,
17411741
uint64_t size, rl_type_t type)
17421742
{
17431743
uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
17441744
rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1745-
rl_t *rl;
17461745

1747-
rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
17481746
rl->rl_object = object;
17491747
rl->rl_offset = offset;
17501748
rl->rl_size = size;
17511749
rl->rl_lock = rll;
17521750

17531751
ztest_rll_lock(rll, type);
1754-
1755-
return (rl);
17561752
}
17571753

17581754
static void
@@ -1761,8 +1757,6 @@ ztest_range_unlock(rl_t *rl)
17611757
rll_t *rll = rl->rl_lock;
17621758

17631759
ztest_rll_unlock(rll);
1764-
1765-
umem_free(rl, sizeof (*rl));
17661760
}
17671761

17681762
static void
@@ -2190,7 +2184,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
21902184
dmu_tx_t *tx;
21912185
dmu_buf_t *db;
21922186
arc_buf_t *abuf = NULL;
2193-
rl_t *rl;
2187+
rl_t rl;
21942188

21952189
if (byteswap)
21962190
byteswap_uint64_array(lr, sizeof (*lr));
@@ -2214,7 +2208,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22142208
bt = NULL;
22152209

22162210
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2217-
rl = ztest_range_lock(zd, lr->lr_foid, offset, length, ZTRL_WRITER);
2211+
ztest_range_lock(zd, &rl, lr->lr_foid, offset, length, ZTRL_WRITER);
22182212

22192213
VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
22202214

@@ -2239,7 +2233,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22392233
if (abuf != NULL)
22402234
dmu_return_arcbuf(abuf);
22412235
dmu_buf_rele(db, FTAG);
2242-
ztest_range_unlock(rl);
2236+
ztest_range_unlock(&rl);
22432237
ztest_object_unlock(zd, lr->lr_foid);
22442238
return (ENOSPC);
22452239
}
@@ -2298,7 +2292,7 @@ ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
22982292

22992293
dmu_tx_commit(tx);
23002294

2301-
ztest_range_unlock(rl);
2295+
ztest_range_unlock(&rl);
23022296
ztest_object_unlock(zd, lr->lr_foid);
23032297

23042298
return (0);
@@ -2312,13 +2306,13 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23122306
objset_t *os = zd->zd_os;
23132307
dmu_tx_t *tx;
23142308
uint64_t txg;
2315-
rl_t *rl;
2309+
rl_t rl;
23162310

23172311
if (byteswap)
23182312
byteswap_uint64_array(lr, sizeof (*lr));
23192313

23202314
ztest_object_lock(zd, lr->lr_foid, ZTRL_READER);
2321-
rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
2315+
ztest_range_lock(zd, &rl, lr->lr_foid, lr->lr_offset, lr->lr_length,
23222316
ZTRL_WRITER);
23232317

23242318
tx = dmu_tx_create(os);
@@ -2327,7 +2321,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23272321

23282322
txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
23292323
if (txg == 0) {
2330-
ztest_range_unlock(rl);
2324+
ztest_range_unlock(&rl);
23312325
ztest_object_unlock(zd, lr->lr_foid);
23322326
return (ENOSPC);
23332327
}
@@ -2339,7 +2333,7 @@ ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
23392333

23402334
dmu_tx_commit(tx);
23412335

2342-
ztest_range_unlock(rl);
2336+
ztest_range_unlock(&rl);
23432337
ztest_object_unlock(zd, lr->lr_foid);
23442338

23452339
return (0);
@@ -2455,12 +2449,12 @@ ztest_get_done(zgd_t *zgd, int error)
24552449
{
24562450
(void) error;
24572451
ztest_ds_t *zd = zgd->zgd_private;
2458-
uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2452+
uint64_t object = ((rl_t *)&zgd->zgd_lr)->rl_object;
24592453

24602454
if (zgd->zgd_db)
24612455
dmu_buf_rele(zgd->zgd_db, zgd);
24622456

2463-
ztest_range_unlock((rl_t *)zgd->zgd_lr);
2457+
ztest_range_unlock((rl_t *)&zgd->zgd_lr);
24642458
ztest_object_unlock(zd, object);
24652459

24662460
umem_free(zgd, sizeof (*zgd));
@@ -2510,7 +2504,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25102504
zgd->zgd_private = zd;
25112505

25122506
if (buf != NULL) { /* immediate write */
2513-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2507+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25142508
object, offset, size, ZTRL_READER);
25152509

25162510
error = dmu_read(os, object, offset, size, buf,
@@ -2526,7 +2520,7 @@ ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
25262520
offset = 0;
25272521
}
25282522

2529-
zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2523+
ztest_range_lock(zd, (rl_t *)&zgd->zgd_lr,
25302524
object, offset, size, ZTRL_READER);
25312525

25322526
error = dmu_buf_hold_noread(os, object, offset, zgd, &db);
@@ -2772,12 +2766,12 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
27722766
objset_t *os = zd->zd_os;
27732767
dmu_tx_t *tx;
27742768
uint64_t txg;
2775-
rl_t *rl;
2769+
rl_t rl;
27762770

27772771
txg_wait_synced(dmu_objset_pool(os), 0);
27782772

27792773
ztest_object_lock(zd, object, ZTRL_READER);
2780-
rl = ztest_range_lock(zd, object, offset, size, ZTRL_WRITER);
2774+
ztest_range_lock(zd, &rl, object, offset, size, ZTRL_WRITER);
27812775

27822776
tx = dmu_tx_create(os);
27832777

@@ -2793,7 +2787,7 @@ ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
27932787
(void) dmu_free_long_range(os, object, offset, size);
27942788
}
27952789

2796-
ztest_range_unlock(rl);
2790+
ztest_range_unlock(&rl);
27972791
ztest_object_unlock(zd, object);
27982792
}
27992793

‎include/sys/dmu.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
#include <sys/fs/zfs.h>
5050
#include <sys/zio_compress.h>
5151
#include <sys/zio_priority.h>
52+
#include <sys/zfs_rlock.h>
5253
#include <sys/uio.h>
5354
#include <sys/zfs_file.h>
5455

@@ -1069,8 +1070,8 @@ typedef struct zgd {
10691070
struct lwb *zgd_lwb;
10701071
struct blkptr *zgd_bp;
10711072
dmu_buf_t *zgd_db;
1072-
struct zfs_locked_range *zgd_lr;
10731073
void *zgd_private;
1074+
zfs_locked_range_t zgd_lr;
10741075
} zgd_t;
10751076

10761077
typedef void dmu_sync_cb_t(zgd_t *arg, int error);

‎include/sys/zfs_rlock.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ typedef struct zfs_locked_range {
6969
void zfs_rangelock_init(zfs_rangelock_t *, zfs_rangelock_cb_t *, void *);
7070
void zfs_rangelock_fini(zfs_rangelock_t *);
7171

72-
zfs_locked_range_t *zfs_rangelock_enter(zfs_rangelock_t *,
72+
void zfs_rangelock_enter(zfs_rangelock_t *, zfs_locked_range_t *,
7373
uint64_t, uint64_t, zfs_rangelock_type_t);
74-
zfs_locked_range_t *zfs_rangelock_tryenter(zfs_rangelock_t *,
74+
boolean_t zfs_rangelock_tryenter(zfs_rangelock_t *, zfs_locked_range_t *,
7575
uint64_t, uint64_t, zfs_rangelock_type_t);
7676
void zfs_rangelock_exit(zfs_locked_range_t *);
7777
void zfs_rangelock_reduce(zfs_locked_range_t *, uint64_t, uint64_t);

‎module/os/freebsd/zfs/zfs_vnops_os.c

+12-11
Original file line numberDiff line numberDiff line change
@@ -4055,7 +4055,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40554055
{
40564056
znode_t *zp = VTOZ(vp);
40574057
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
4058-
zfs_locked_range_t *lr;
4058+
zfs_locked_range_t lr;
4059+
boolean_t res;
40594060
vm_object_t object;
40604061
off_t start, end, obj_size;
40614062
uint_t blksz;
@@ -4074,10 +4075,10 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40744075
*/
40754076
for (;;) {
40764077
blksz = zp->z_blksz;
4077-
lr = zfs_rangelock_tryenter(&zp->z_rangelock,
4078+
res = zfs_rangelock_tryenter(&zp->z_rangelock, &lr,
40784079
rounddown(start, blksz),
40794080
roundup(end, blksz) - rounddown(start, blksz), RL_READER);
4080-
if (lr == NULL) {
4081+
if (res == B_FALSE) {
40814082
if (rahead != NULL) {
40824083
*rahead = 0;
40834084
rahead = NULL;
@@ -4090,16 +4091,16 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
40904091
}
40914092
if (blksz == zp->z_blksz)
40924093
break;
4093-
zfs_rangelock_exit(lr);
4094+
zfs_rangelock_exit(&lr);
40944095
}
40954096

40964097
object = ma[0]->object;
40974098
zfs_vmobject_wlock(object);
40984099
obj_size = object->un_pager.vnp.vnp_size;
40994100
zfs_vmobject_wunlock(object);
41004101
if (IDX_TO_OFF(ma[count - 1]->pindex) >= obj_size) {
4101-
if (lr != NULL)
4102-
zfs_rangelock_exit(lr);
4102+
if (res == B_TRUE)
4103+
zfs_rangelock_exit(&lr);
41034104
zfs_exit(zfsvfs, FTAG);
41044105
return (zfs_vm_pagerret_bad);
41054106
}
@@ -4127,8 +4128,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind,
41274128
error = dmu_read_pages(zfsvfs->z_os, zp->z_id, ma, count, &pgsin_b,
41284129
&pgsin_a, MIN(end, obj_size) - (end - PAGE_SIZE));
41294130

4130-
if (lr != NULL)
4131-
zfs_rangelock_exit(lr);
4131+
if (res == B_TRUE)
4132+
zfs_rangelock_exit(&lr);
41324133
ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
41334134

41344135
dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, count*PAGE_SIZE);
@@ -4171,7 +4172,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
41714172
{
41724173
znode_t *zp = VTOZ(vp);
41734174
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
4174-
zfs_locked_range_t *lr;
4175+
zfs_locked_range_t lr;
41754176
dmu_tx_t *tx;
41764177
struct sf_buf *sf;
41774178
vm_object_t object;
@@ -4203,7 +4204,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
42034204
blksz = zp->z_blksz;
42044205
lo_off = rounddown(off, blksz);
42054206
lo_len = roundup(len + (off - lo_off), blksz);
4206-
lr = zfs_rangelock_enter(&zp->z_rangelock, lo_off, lo_len, RL_WRITER);
4207+
zfs_rangelock_enter(&zp->z_rangelock, &lr, lo_off, lo_len, RL_WRITER);
42074208

42084209
zfs_vmobject_wlock(object);
42094210
if (len + off > object->un_pager.vnp.vnp_size) {
@@ -4309,7 +4310,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
43094310
dmu_tx_commit(tx);
43104311

43114312
out:
4312-
zfs_rangelock_exit(lr);
4313+
zfs_rangelock_exit(&lr);
43134314
if (commit)
43144315
zil_commit(zfsvfs->z_log, zp->z_id);
43154316

‎module/os/freebsd/zfs/zfs_znode.c

+15-15
Original file line numberDiff line numberDiff line change
@@ -1393,20 +1393,20 @@ zfs_extend(znode_t *zp, uint64_t end)
13931393
{
13941394
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
13951395
dmu_tx_t *tx;
1396-
zfs_locked_range_t *lr;
1396+
zfs_locked_range_t lr;
13971397
uint64_t newblksz;
13981398
int error;
13991399

14001400
/*
14011401
* We will change zp_size, lock the whole file.
14021402
*/
1403-
lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
1403+
zfs_rangelock_enter(&zp->z_rangelock, &lr, 0, UINT64_MAX, RL_WRITER);
14041404

14051405
/*
14061406
* Nothing to do if file already at desired length.
14071407
*/
14081408
if (end <= zp->z_size) {
1409-
zfs_rangelock_exit(lr);
1409+
zfs_rangelock_exit(&lr);
14101410
return (0);
14111411
}
14121412
tx = dmu_tx_create(zfsvfs->z_os);
@@ -1436,7 +1436,7 @@ zfs_extend(znode_t *zp, uint64_t end)
14361436
error = dmu_tx_assign(tx, TXG_WAIT);
14371437
if (error) {
14381438
dmu_tx_abort(tx);
1439-
zfs_rangelock_exit(lr);
1439+
zfs_rangelock_exit(&lr);
14401440
return (error);
14411441
}
14421442

@@ -1450,7 +1450,7 @@ zfs_extend(znode_t *zp, uint64_t end)
14501450

14511451
vnode_pager_setsize(ZTOV(zp), end);
14521452

1453-
zfs_rangelock_exit(lr);
1453+
zfs_rangelock_exit(&lr);
14541454

14551455
dmu_tx_commit(tx);
14561456

@@ -1470,19 +1470,19 @@ static int
14701470
zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
14711471
{
14721472
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1473-
zfs_locked_range_t *lr;
1473+
zfs_locked_range_t lr;
14741474
int error;
14751475

14761476
/*
14771477
* Lock the range being freed.
14781478
*/
1479-
lr = zfs_rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER);
1479+
zfs_rangelock_enter(&zp->z_rangelock, &lr, off, len, RL_WRITER);
14801480

14811481
/*
14821482
* Nothing to do if file already at desired length.
14831483
*/
14841484
if (off >= zp->z_size) {
1485-
zfs_rangelock_exit(lr);
1485+
zfs_rangelock_exit(&lr);
14861486
return (0);
14871487
}
14881488

@@ -1504,7 +1504,7 @@ zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
15041504
#endif
15051505
}
15061506

1507-
zfs_rangelock_exit(lr);
1507+
zfs_rangelock_exit(&lr);
15081508

15091509
return (error);
15101510
}
@@ -1523,28 +1523,28 @@ zfs_trunc(znode_t *zp, uint64_t end)
15231523
zfsvfs_t *zfsvfs = zp->z_zfsvfs;
15241524
vnode_t *vp = ZTOV(zp);
15251525
dmu_tx_t *tx;
1526-
zfs_locked_range_t *lr;
1526+
zfs_locked_range_t lr;
15271527
int error;
15281528
sa_bulk_attr_t bulk[2];
15291529
int count = 0;
15301530

15311531
/*
15321532
* We will change zp_size, lock the whole file.
15331533
*/
1534-
lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
1534+
zfs_rangelock_enter(&zp->z_rangelock, &lr, 0, UINT64_MAX, RL_WRITER);
15351535

15361536
/*
15371537
* Nothing to do if file already at desired length.
15381538
*/
15391539
if (end >= zp->z_size) {
1540-
zfs_rangelock_exit(lr);
1540+
zfs_rangelock_exit(&lr);
15411541
return (0);
15421542
}
15431543

15441544
error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, end,
15451545
DMU_OBJECT_END);
15461546
if (error) {
1547-
zfs_rangelock_exit(lr);
1547+
zfs_rangelock_exit(&lr);
15481548
return (error);
15491549
}
15501550
tx = dmu_tx_create(zfsvfs->z_os);
@@ -1554,7 +1554,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
15541554
error = dmu_tx_assign(tx, TXG_WAIT);
15551555
if (error) {
15561556
dmu_tx_abort(tx);
1557-
zfs_rangelock_exit(lr);
1557+
zfs_rangelock_exit(&lr);
15581558
return (error);
15591559
}
15601560

@@ -1579,7 +1579,7 @@ zfs_trunc(znode_t *zp, uint64_t end)
15791579
*/
15801580
vnode_pager_setsize(vp, end);
15811581

1582-
zfs_rangelock_exit(lr);
1582+
zfs_rangelock_exit(&lr);
15831583

15841584
return (0);
15851585
}

0 commit comments

Comments
 (0)