Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

executor: Add TiKV read throughput metric (#28701) #28913

Open
wants to merge 2 commits into
base: release-5.2
Choose a base branch
from
Open

executor: Add TiKV read throughput metric (#28701) #28913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2da0862
cherry pick #28701 to release-5.2
jyz0309 Oct 11, 2021
34c4833
fix conflict
jyz0309 Oct 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions br/tests/br_rawkv/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func randGen(client *rawkv.Client, startKey, endKey []byte, maxLen int, concurre
values = append(values, value)
}

err := client.BatchPut(context.TODO(), keys, values)
err := client.BatchPut(context.TODO(), keys, values, nil)
if err != nil {
errCh <- errors.Trace(err)
}
Expand Down Expand Up @@ -307,7 +307,7 @@ func put(client *rawkv.Client, dataStr string) error {

log.Info("Put rawkv data", zap.ByteStrings("keys", keys), zap.ByteStrings("values", values))

err := client.BatchPut(context.TODO(), keys, values)
err := client.BatchPut(context.TODO(), keys, values, nil)
return errors.Trace(err)
}

Expand Down
3 changes: 2 additions & 1 deletion distsql/select_result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,8 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr
if copStats.ScanDetail != nil {
readKeys := copStats.ScanDetail.ProcessedKeys
readTime := copStats.TimeDetail.KvReadWallTimeMs.Seconds()
tikvmetrics.ObserveReadSLI(uint64(readKeys), readTime)
readSize := float64(copStats.ScanDetail.ProcessedKeysSize)
tikvmetrics.ObserveReadSLI(uint64(readKeys), readTime, readSize)
}

if r.stats == nil {
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ require (
github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63
github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059
github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4
github.com/pingcap/kvproto v0.0.0-20210915062418-0f5764a128ad
github.com/pingcap/log v0.0.0-20210818144256-6455d4a4c6f9
github.com/pingcap/parser v0.0.0-20210831085004-b5390aa83f65
github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5
Expand All @@ -66,8 +66,8 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210831090540-391fcd842dc8
github.com/tikv/pd v1.1.0-beta.0.20210818112400-0c5667766690
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20211011034620-128ccbde43c4
github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d
github.com/twmb/murmur3 v1.1.3
github.com/uber-go/atomic v1.4.0
github.com/uber/jaeger-client-go v2.22.1+incompatible
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,8 @@ github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO
github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210219064844-c1844a4775d6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210805052247-76981389e818/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 h1:4EUpHzPFHwleKkVALyMqQbQcNziPZvU+vhUT9Wzj93E=
github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20210915062418-0f5764a128ad h1:suBPTeuY6yVF7xvTGeTQ9+tiGzufnORJpCRwzbdN2sc=
github.com/pingcap/kvproto v0.0.0-20210915062418-0f5764a128ad/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
Expand Down Expand Up @@ -694,11 +694,11 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK
github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210831090540-391fcd842dc8 h1:8nv1BaOlTfYtK3Pyb0fHQ0zm9mRMPlXFbAWUhTEvRdo=
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210831090540-391fcd842dc8/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o=
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20211011034620-128ccbde43c4 h1:bk6InEr6TB1Hs48+v/k2D9IS6vGPbwK3covHMF6myNY=
github.com/tikv/client-go/v2 v2.0.0-alpha.0.20211011034620-128ccbde43c4/go.mod h1:00plYwQsQ5kBUmafHO+JkjznGgFaBokMZl82TZIbsQk=
github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw=
github.com/tikv/pd v1.1.0-beta.0.20210818112400-0c5667766690 h1:qGn7fDqj7IZ5dozy7QVkoj+0bama92ruVGHqoCBg7W4=
github.com/tikv/pd v1.1.0-beta.0.20210818112400-0c5667766690/go.mod h1:rammPjeZgpvfrQRPkijcx8tlxF1XM5+m6kRXrkDzCAA=
github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d h1:AFm1Dzw+QRUevWRfrFp45CPPkuK/zdSWcfxI10z+WVE=
github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d/go.mod h1:rammPjeZgpvfrQRPkijcx8tlxF1XM5+m6kRXrkDzCAA=
github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
Expand Down
6 changes: 4 additions & 2 deletions util/execdetails/execdetails_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ func TestCopRuntimeStats(t *testing.T) {
scanDetail := &util.ScanDetail{
TotalKeys: 15,
ProcessedKeys: 10,
ProcessedKeysSize: 10,
RocksdbDeleteSkippedCount: 5,
RocksdbKeySkippedCount: 1,
RocksdbBlockCacheHitCount: 10,
Expand All @@ -109,7 +110,7 @@ func TestCopRuntimeStats(t *testing.T) {

cop := stats.GetOrCreateCopStats(tableScanID, "tikv")
expected := "tikv_task:{proc max:2ns, min:1ns, p80:2ns, p95:2ns, iters:3, tasks:2}, " +
"scan_detail: {total_process_keys: 10, total_keys: 15, rocksdb: {delete_skipped_count: 5, key_skipped_count: 1, block: {cache_hit_count: 10, read_count: 20, read_byte: 100 Bytes}}}"
"scan_detail: {total_process_keys: 10, total_process_keys_size: 10, total_keys: 15, rocksdb: {delete_skipped_count: 5, key_skipped_count: 1, block: {cache_hit_count: 10, read_count: 20, read_byte: 100 Bytes}}}"
require.Equal(t, expected, cop.String())

copStats := cop.stats["8.8.8.8"]
Expand All @@ -125,11 +126,12 @@ func TestCopRuntimeStats(t *testing.T) {
require.True(t, stats.ExistsRootStats(tableReaderID))

cop.scanDetail.ProcessedKeys = 0
cop.scanDetail.ProcessedKeysSize = 0
cop.scanDetail.RocksdbKeySkippedCount = 0
cop.scanDetail.RocksdbBlockReadCount = 0
// Print all fields even though the value of some fields is 0.
str := "tikv_task:{proc max:1s, min:2ns, p80:1s, p95:1s, iters:4, tasks:2}, " +
"scan_detail: {total_process_keys: 0, total_keys: 15, rocksdb: {delete_skipped_count: 5, key_skipped_count: 0, block: {cache_hit_count: 10, read_count: 0, read_byte: 100 Bytes}}}"
"scan_detail: {total_process_keys: 0, total_process_keys_size: 0, total_keys: 15, rocksdb: {delete_skipped_count: 5, key_skipped_count: 0, block: {cache_hit_count: 10, read_count: 0, read_byte: 100 Bytes}}}"
require.Equal(t, str, cop.String())

zeroScanDetail := util.ScanDetail{}
Expand Down