Skip to content

storagev2包中的uptoken.NewSigner构造出的uptoken.Provider存在线程不安全的问题 #169

@zhangzqs

Description

@zhangzqs
[test] ==================
[test] WARNING: DATA RACE
[test] Read at 0x00c0002098a8 by goroutine 14:
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).onceGetUpToken()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uptoken/uploadtoken.go:89 +0x44
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).GetUpToken()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uptoken/uploadtoken.go:72 +0x3a
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*innerPostObjectRequest).build()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/apis/api_post_object.go:40 +0x11b
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/apis/api_post_object.go:89 +0x197
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadFile()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:87 +0x3fd
[test]   io.Copy()
[test]       io/io.go:388 +0x1b3
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.crc32FromReadSeeker()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:339 +0x102
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadFile()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:82 +0x34d
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadFile()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadFile()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:215 +0x35a
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:162 +0x4fa
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:93 +0x86
[test] 
[test] Previous write at 0x00c0002098a8 by goroutine 13:
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).onceGetUpToken()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uptoken/uploadtoken.go:100 +0xc4
[test]   github.com/qiniu/go-sdk/v7/storagev2/uptoken.(*signer).GetUpToken()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uptoken/uploadtoken.go:72 +0x3a
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*innerPostObjectRequest).build()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/apis/api_post_object.go:40 +0x11b
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/apis/api_post_object.go:89 +0x197
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadReader()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:127 +0x4af
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadReader()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadReader()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:248 +0x654
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:180 +0x988
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:93 +0x86
[test] 
[test] Goroutine 14 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:345 +0x14d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:369 +0x379
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] 
[test] Goroutine 13 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:349 +0xcd
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] ==================
[test] ==================
[test] WARNING: DATA RACE
[test] Write at 0x00c0001487a0 by goroutine 20:
[test]   github.com/qiniu/go-sdk/v7/internal/uplog.(*RequestUplog).Intercept.func7()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/uplog/request_uplog.go:154 +0x104
[test]   net.(*sysDialer).dialSingle.func1()
[test]       net/dial.go:711 +0xda
[test]   runtime.deferreturn()
[test]       runtime/panic.go:589 +0x5d
[test]   net.(*sysDialer).dialSerial()
[test]       net/dial.go:686 +0x288
[test]   net.(*sysDialer).dialParallel()
[test]       net/dial.go:587 +0x525
[test]   net.(*Dialer).DialContext()
[test]       net/dial.go:578 +0xbbb
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextSync()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/dialer/dialer.go:84 +0x1a8
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextAsync.func1()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/dialer/dialer.go:90 +0x16c
[test] 
[test] Previous read at 0x00c0001487a0 by goroutine 13:
[test]   reflect.Value.Uint()
[test]       reflect/value.go:2528 +0x333
[test]   reflect.Value.IsZero()
[test]       reflect/value.go:1643 +0x260
[test]   encoding/json.isEmptyValue()
[test]       encoding/json/encode.go:361 +0x85
[test]   encoding/json.structEncoder.encode()
[test]       encoding/json/encode.go:746 +0x175
[test]   encoding/json.structEncoder.encode-fm()
[test]       <autogenerated>:1 +0xe4
[test]   encoding/json.ptrEncoder.encode()
[test]       encoding/json/encode.go:930 +0x3c2
[test]   encoding/json.ptrEncoder.encode-fm()
[test]       <autogenerated>:1 +0x84
[test]   encoding/json.(*encodeState).reflectValue()
[test]       encoding/json/encode.go:367 +0x83
[test]   encoding/json.(*encodeState).marshal()
[test]       encoding/json/encode.go:343 +0xdb
[test]   encoding/json.Marshal()
[test]       encoding/json/encode.go:209 +0x11e
[test]   github.com/qiniu/go-sdk/v7/internal/uplog.(*RequestUplog).Intercept()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/uplog/request_uplog.go:218 +0x1c24
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*simpleRetryInterceptor).callHandler()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/interceptor_retry_simple.go:171 +0x87
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*simpleRetryInterceptor).Intercept()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/interceptor_retry_simple.go:111 +0x516
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*hostsRetryInterceptor).Intercept()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/interceptor_retry_hosts.go:84 +0x22d
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do.func2()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/client.go:92 +0x54
[test]   github.com/qiniu/go-sdk/v7/internal/clientv2.(*client).Do()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/clientv2/client.go:96 +0x1c1
[test]   github.com/qiniu/go-sdk/v7/storagev2/http_client.(*Client).Do()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/http_client/http_client.go:275 +0x4c1
[test]   github.com/qiniu/go-sdk/v7/storagev2/http_client.(*Client).DoAndAcceptJSON()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/http_client/http_client.go:280 +0x55
[test]   github.com/qiniu/go-sdk/v7/storagev2/apis.(*Storage).PostObject()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/apis/api_post_object.go:148 +0x11ca
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.uploadToRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:160 +0x3ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload.func1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:132 +0x10b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.forEachRegion()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:453 +0x1ed
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.upload()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:131 +0x1ba
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.formUploader.UploadReader()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/uploaders.go:127 +0x4af
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*formUploader).UploadReader()
[test]       <autogenerated>:1 +0xbb
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadReader()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:248 +0x654
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3.1()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:180 +0x988
[test]   golang.org/x/sync/errgroup.(*Group).Go.func1()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:93 +0x86
[test] 
[test] Goroutine 20 (running) created at:
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.dialContextAsync()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/dialer/dialer.go:88 +0x22d
[test]   github.com/qiniu/go-sdk/v7/internal/dialer.DialContext()
[test]       github.com/qiniu/go-sdk/[email protected]/internal/dialer/dialer.go:48 +0x476
[test]   github.com/qiniu/go-sdk/v7/client.defaultDialFunc()
[test]       github.com/qiniu/go-sdk/[email protected]/client/dialer.go:36 +0x309
[test]   net/http.(*Transport).dial()
[test]       net/http/transport.go:1278 +0xe6
[test]   net/http.(*Transport).dialConn()
[test]       net/http/transport.go:1783 +0xd64
[test]   net/http.(*Transport).dialConnFor()
[test]       net/http/transport.go:1618 +0x10f
[test]   net/http.(*Transport).startDialConnForLocked.func1()
[test]       net/http/transport.go:1600 +0x3c
[test] 
[test] Goroutine 13 (running) created at:
[test]   golang.org/x/sync/errgroup.(*Group).Go()
[test]       golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x11c
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory.func3()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:141 +0x32d
[test]   path/filepath.walk()
[test]       path/filepath/path.go:349 +0xcd
[test]   path/filepath.Walk()
[test]       path/filepath/path.go:427 +0x7b
[test]   github.com/qiniu/go-sdk/v7/storagev2/uploader.(*UploadManager).UploadDirectory()
[test]       github.com/qiniu/go-sdk/[email protected]/storagev2/uploader/upload_manager.go:137 +0x5b0
[test]   github.com/qbox/las/internal/rfsjob/service.(*KodoUploader).Upload()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader.go:151 +0x987
[test]   github.com/qbox/las/internal/rfsjob/service.TestKodoUploader_Retry()
[test]       github.com/qbox/las/internal/rfsjob/service/kodo_uploader_test.go:205 +0xaaa
[test]   testing.tRunner()
[test]       testing/testing.go:1934 +0x21c
[test]   testing.(*T).Run.gowrap1()
[test]       testing/testing.go:1997 +0x44
[test] ==================
[test] --- FAIL: TestKodoUploader_Retry (0.18s)
****************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
[test]     writer.go:27: {"level":"debug","time":1762399299494844515,"message":"Using accessKey/secretKey for Kodo upload"}
[test]     writer.go:27: {"level":"warn","url":"http://127.0.0.1:44741/","attempts":0,"backoff_delay":"67ms","time":1762399299508193116,"message":"kodo upload retry backoff"}
[test]     writer.go:27: {"level":"warn","url":"http://127.0.0.1:44741/","attempts":1,"backoff_delay":"89ms","time":1762399299577824496,"message":"kodo upload retry backoff"}
[test]     testing.go:1617: race detected during execution of test
[test] FAIL
[test] coverage: 8.2% of statements
[test] FAIL	github.com/qbox/las/internal/rfsjob/service	0.290s

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions