Support to config charset and collation (#1368)

* feat(feat-config-connection-charset): support config connection charset

* feat(feat-config-connection-charset): feat-config-connection-charset

The ability to specify character set and collation is supported.

---------

Co-authored-by: shaohoukun <[email protected]>

Author: shaohk

doc/command-line-flags.md

@@ -261,6 +261,9 @@ But RocksDB currently lacks a few features support compared to InnoDB:
 
 When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB.
 
+### charset
+The default charset for the database connection is utf8mb4, utf8, latin1. The ability to specify character set and collation is supported, eg: utf8mb4_general_ci,utf8_general_ci,latin1. 
+
 ### test-on-replica
 
 Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md)

go/base/context.go

@@ -305,6 +305,15 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string) error {
 	return nil
 }
 
+func (this *MigrationContext) SetConnectionCharset(charset string) {
+	if charset == "" {
+		charset = "utf8mb4,utf8,latin1"
+	}
+
+	this.InspectorConnectionConfig.Charset = charset
+	this.ApplierConnectionConfig.Charset = charset
+}
+
 func getSafeTableName(baseName string, suffix string) string {
 	name := fmt.Sprintf("_%s_%s", baseName, suffix)
 	if len(name) <= mysql.MaxTableNameLength {

go/cmd/gh-ost/main.go

@@ -57,6 +57,7 @@ func main() {
 	flag.StringVar(&migrationContext.CliMasterPassword, "master-password", "", "MySQL password on master, if different from that on replica. Requires --assume-master-host")
 	flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file")
 	askPass := flag.Bool("ask-pass", false, "prompt for MySQL password")
+	charset := flag.String("charset", "utf8mb4,utf8,latin1", "The default charset for the database connection is utf8mb4, utf8, latin1.")
 
 	flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts")
 	flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl")
@@ -191,6 +192,8 @@ func main() {
 		migrationContext.Log.Fatale(err)
 	}
 
+	migrationContext.SetConnectionCharset(*charset)
+
 	if migrationContext.AlterStatement == "" {
 		log.Fatal("--alter must be provided and statement must not be empty")
 	}

go/mysql/connection.go

@@ -30,6 +30,7 @@ type ConnectionConfig struct {
 	tlsConfig            *tls.Config
 	Timeout              float64
 	TransactionIsolation string
+	Charset              string
 }
 
 func NewConnectionConfig() *ConnectionConfig {
@@ -49,6 +50,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC
 		tlsConfig:            this.tlsConfig,
 		Timeout:              this.Timeout,
 		TransactionIsolation: this.TransactionIsolation,
+		Charset:              this.Charset,
 	}
 	config.ImpliedKey = &config.Key
 	return config
@@ -122,10 +124,15 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
 	if this.tlsConfig != nil {
 		tlsOption = TLS_CONFIG_KEY
 	}
+
+	if this.Charset == "" {
+		this.Charset = "utf8mb4,utf8,latin1"
+	}
+
 	connectionParams := []string{
 		"autocommit=true",
-		"charset=utf8mb4,utf8,latin1",
 		"interpolateParams=true",
+		fmt.Sprintf("charset=%s", this.Charset),
 		fmt.Sprintf("tls=%s", tlsOption),
 		fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation),
 		fmt.Sprintf("timeout=%fs", this.Timeout),

go/mysql/connection_test.go

@@ -30,6 +30,7 @@ func TestNewConnectionConfig(t *testing.T) {
 	test.S(t).ExpectEquals(c.User, "")
 	test.S(t).ExpectEquals(c.Password, "")
 	test.S(t).ExpectEquals(c.TransactionIsolation, "")
+	test.S(t).ExpectEquals(c.Charset, "")
 }
 
 func TestDuplicateCredentials(t *testing.T) {
@@ -42,6 +43,7 @@ func TestDuplicateCredentials(t *testing.T) {
 		ServerName:         "feathers",
 	}
 	c.TransactionIsolation = transactionIsolation
+	c.Charset = "utf8mb4"
 
 	dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
 	test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
@@ -52,6 +54,7 @@ func TestDuplicateCredentials(t *testing.T) {
 	test.S(t).ExpectEquals(dup.Password, "penguin")
 	test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig)
 	test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation)
+	test.S(t).ExpectEquals(dup.Charset, c.Charset)
 }
 
 func TestDuplicate(t *testing.T) {
@@ -60,6 +63,7 @@ func TestDuplicate(t *testing.T) {
 	c.User = "gromit"
 	c.Password = "penguin"
 	c.TransactionIsolation = transactionIsolation
+	c.Charset = "utf8mb4"
 
 	dup := c.Duplicate()
 	test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
@@ -69,6 +73,7 @@ func TestDuplicate(t *testing.T) {
 	test.S(t).ExpectEquals(dup.User, "gromit")
 	test.S(t).ExpectEquals(dup.Password, "penguin")
 	test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation)
+	test.S(t).ExpectEquals(dup.Charset, "utf8mb4")
 }
 
 func TestGetDBUri(t *testing.T) {
@@ -78,9 +83,10 @@ func TestGetDBUri(t *testing.T) {
 	c.Password = "penguin"
 	c.Timeout = 1.2345
 	c.TransactionIsolation = transactionIsolation
+	c.Charset = "utf8mb4,utf8,latin1"
 
 	uri := c.GetDBUri("test")
-	test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
+	test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&interpolateParams=true&charset=utf8mb4,utf8,latin1&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
 }
 
 func TestGetDBUriWithTLSSetup(t *testing.T) {
@@ -91,7 +97,8 @@ func TestGetDBUriWithTLSSetup(t *testing.T) {
 	c.Timeout = 1.2345
 	c.tlsConfig = &tls.Config{}
 	c.TransactionIsolation = transactionIsolation
+	c.Charset = "utf8mb4_general_ci,utf8_general_ci,latin1"
 
 	uri := c.GetDBUri("test")
-	test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
+	test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&interpolateParams=true&charset=utf8mb4_general_ci,utf8_general_ci,latin1&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
 }