Add support for authentication via mTLS (authentication with TLS certificates)

.github/workflows/test.yml

@@ -20,64 +20,56 @@ jobs:
         image: clickhouse/clickhouse-server
         ports:
           - 9000:9000
+        env:
+          CLICKHOUSE_SKIP_USER_SETUP: 1
     steps:
       - uses: actions/checkout@v3
       - name: Build
         run: cargo build --verbose
       - name: Run tests
         run: cargo test --verbose
 
-  build-native-tls:
+  build-tls:
+    strategy:
+      fail-fast: false
+      matrix:
+        feature:
+          - tls-native-tls
+          - tls-rustls
+        database_url:
+          # for TLS we need skip_verify for self-signed certificate
+          - tcp://localhost:9440?skip_verify=true
+          # we don't need skip_verify when we pass CA cert
+          - tcp://localhost:9440?ca_certificate=tls/ca.pem
+          # mTLS
+          - tcp://tls@localhost:9440?ca_certificate=tls/ca.pem&client_certificate=tls/client.crt&client_private_key=tls/client.key
     runs-on: ubuntu-latest
     env:
       # NOTE: not all tests "secure" aware, so let's define DATABASE_URL explicitly
       # NOTE: sometimes for native-tls default connection_timeout (500ms) is not enough, interestingly that for rustls it is OK.
-      DATABASE_URL: "tcp://localhost:9440?compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&skip_verify=true&connection_timeout=5s"
+      DATABASE_URL: ${{ matrix.database_url }}&compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&connection_timeout=5s
     steps:
       - uses: actions/checkout@v3
+      - name: Generate TLS certificates
+        run: |
+          extras/ci/generate_certs.sh tls
       # NOTE:
       # - we cannot use "services" because they are executed before the steps, i.e. repository checkout.
       # - "job.container.network" is empty, hence "host"
       # - github actions does not support YAML anchors (sigh)
       - name: Run clickhouse-server
         run: docker run
-          -v ./extras/ci/generate_certs.sh:/docker-entrypoint-initdb.d/generate_certs.sh
           -v ./extras/ci/overrides.xml:/etc/clickhouse-server/config.d/overrides.xml
-          -e CH_SSL_CERTIFICATE=/etc/clickhouse-server/config.d/server.crt
-          -e CH_SSL_PRIVATE_KEY=/etc/clickhouse-server/config.d/server.key
+          -v ./extras/ci/users-overrides.yaml:/etc/clickhouse-server/users.d/overrides.yaml
+          -v ./tls:/etc/clickhouse-server/tls
+          -e CLICKHOUSE_SKIP_USER_SETUP=1
           --network host
+          --name clickhouse
           --rm
           --detach
           --publish 9440:9440
           clickhouse/clickhouse-server
       - name: Build
-        run: cargo build --features tls-native-tls --verbose
+        run: cargo build --features ${{ matrix.feature }} --verbose
       - name: Run tests
-        run: cargo test --features tls-native-tls --verbose
-
-  build-rustls:
-    runs-on: ubuntu-latest
-    env:
-      # NOTE: not all tests "secure" aware, so let's define DATABASE_URL explicitly
-      DATABASE_URL: "tcp://localhost:9440?compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&skip_verify=true"
-    steps:
-      - uses: actions/checkout@v3
-      # NOTE:
-      # - we cannot use "services" because they are executed before the steps, i.e. repository checkout.
-      # - "job.container.network" is empty, hence "host"
-      # - github actions does not support YAML anchors (sigh)
-      - name: Run clickhouse-server
-        run: docker run
-          -v ./extras/ci/generate_certs.sh:/docker-entrypoint-initdb.d/generate_certs.sh
-          -v ./extras/ci/overrides.xml:/etc/clickhouse-server/config.d/overrides.xml
-          -e CH_SSL_CERTIFICATE=/etc/clickhouse-server/config.d/server.crt
-          -e CH_SSL_PRIVATE_KEY=/etc/clickhouse-server/config.d/server.key
-          --network host
-          --rm
-          --detach
-          --publish 9440:9440
-          clickhouse/clickhouse-server
-      - name: Build
-        run: cargo build --features tls-rustls --verbose
-      - name: Run tests
-        run: cargo test --features tls-rustls --verbose
+        run: cargo test --features ${{ matrix.feature }} --verbose

README.md

@@ -72,7 +72,13 @@ for the most common use cases. The following features are available.
 
 - `tokio_io` *(enabled by default)* — I/O based on [Tokio](https://tokio.rs/).
 - `async_std` — I/O based on [async-std](https://async.rs/) (doesn't work together with `tokio_io`).
-- `tls` — TLS support (allowed only with `tokio_io`).
+- `tls` — TLS support (allowed only with `tokio_io` and one of TLS libraries, under `tls-rustls` or `tls-native-tls` features).
+
+### TLS
+
+- `skip_verify` - do not verify the server certificate (**insecure**)
+- `ca_certificate` - instead of `skip_verify` it is better to pass CA certificate explicitly (in case of self-signed certificates).
+- `client_certificate`/`client_private_key` - authentication using TLS certificates (mTLS) (see [ClickHouse documentation](https://clickhouse.com/docs/operations/external-authenticators/ssl-x509) for more info)
 
 ## Example
 

extras/ci/generate_certs.sh

@@ -1,7 +1,52 @@
 #!/usr/bin/env bash
 
-crt=$CH_SSL_CERTIFICATE
-key=$CH_SSL_PRIVATE_KEY
+out=$1 && shift
+mkdir -p "$out"
+cd "$out"
 
-openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout "$key" -out "$crt"
-chown clickhouse:clickhouse "$crt" "$key"
+#
+# CA
+#
+openssl genrsa -out ca.key 4096
+openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 -out ca.pem -subj "/C=US/ST=DevState/O=DevOrg/CN=MyDevCA"
+
+#
+# server
+#
+openssl genrsa -out server.key 2048
+openssl req -new -key server.key -out server.csr -subj "/C=US/ST=DevState/O=DevOrg/CN=localhost"
+
+cat > server.ext <<EOL
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+EOL
+
+openssl x509 -req -in server.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.crt -days 825 -sha256 -extfile server.ext
+openssl verify -CAfile ca.pem server.crt
+
+#
+# client
+#
+cat > client.ext <<EOL
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature
+extendedKeyUsage = clientAuth
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+EOL
+
+openssl genrsa -out client.key 2048
+openssl req -new -key client.key -out client.csr -subj "/C=US/ST=DevState/O=DevOrg/CN=MyClient"
+openssl x509 -req -in client.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client.crt -days 3650 -sha256 -extfile client.ext
+openssl verify -CAfile ca.pem client.crt
+
+# server needs access to those
+chmod 644 ca.pem server.key server.crt

extras/ci/overrides.xml

@@ -1,9 +1,10 @@
 <clickhouse>
     <openSSL>
         <server>
-            <certificateFile from_env="CH_SSL_CERTIFICATE" replace="1"></certificateFile>
-            <privateKeyFile from_env="CH_SSL_PRIVATE_KEY" replace="1"></privateKeyFile>
-            <verificationMode>none</verificationMode>
+            <certificateFile>/etc/clickhouse-server/tls/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/tls/server.key</privateKeyFile>
+            <caConfig>/etc/clickhouse-server/tls/ca.pem</caConfig>
+            <verificationMode>relaxed</verificationMode>
             <loadDefaultCAFile>true</loadDefaultCAFile>
             <cacheSessions>true</cacheSessions>
             <disableProtocols>sslv2,sslv3</disableProtocols>

extras/ci/users-overrides.yaml

@@ -0,0 +1,6 @@
+---
+users:
+  tls:
+    ssl_certificates:
+      subject_alt_name:
+      - DNS:localhost

src/connecting_stream.rs

@@ -35,6 +35,8 @@ use crate::{errors::ConnectionError, io::Stream as InnerStream, Options};
 use tokio_native_tls::TlsStream;
 #[cfg(feature = "tls-rustls")]
 use tokio_rustls::client::TlsStream;
+#[cfg(feature = "_tls")]
+use crate::types::ClientTlsIdentity;
 
 type Result<T> = std::result::Result<T, ConnectionError>;
 
@@ -104,6 +106,11 @@ impl State {
         State::Tcp(TcpState::Fail(Some(conn_error)))
     }
 
+    #[cfg(feature = "tls-rustls")]
+    fn tls_err(e: TlsError) -> Self {
+        State::Tls(TlsState::Fail(Some(ConnectionError::TlsError(e))))
+    }
+
     #[cfg(feature = "_tls")]
     fn tls_host_err() -> Self {
         State::Tls(TlsState::Fail(Some(ConnectionError::TlsHostNotProvided)))
@@ -125,6 +132,7 @@ pub(crate) struct ConnectingStream {
     state: State,
 }
 
+#[cfg(feature = "tls-rustls")]
 #[derive(Debug)]
 struct DummyTlsVerifier;
 
@@ -231,10 +239,14 @@ impl ConnectingStream {
             Some(host) => {
                 let mut builder = TlsConnector::builder();
                 builder.danger_accept_invalid_certs(options.skip_verify);
-                if let Some(certificate) = options.certificate.clone() {
+                if let Some(certificate) = options.ca_certificate.clone() {
                     let native_cert = native_tls::Certificate::from(certificate);
                     builder.add_root_certificate(native_cert);
                 }
+                if let Some(identity) = &options.client_tls_identity {
+                    let ClientTlsIdentity::Pkcs(pkcs) = identity;
+                    builder.identity(pkcs.clone());
+                }
 
                 Self {
                     state: State::tls_wait(Box::pin(async move {
@@ -261,19 +273,18 @@ impl ConnectingStream {
                 state: State::tls_host_err(),
             },
             Some(host) => {
-                let config = if options.skip_verify {
+                let builder = if options.skip_verify {
                     ClientConfig::builder()
                         .dangerous()
                         .with_custom_certificate_verifier(Arc::new(DummyTlsVerifier))
-                        .with_no_client_auth()
                 } else {
                     let mut cert_store = RootCertStore::empty();
                     cert_store.extend(
                         webpki_roots::TLS_SERVER_ROOTS
                             .iter()
                             .cloned()
                     );
-                    if let Some(certificates) = options.certificate.clone() {
+                    if let Some(certificates) = options.ca_certificate.clone() {
                         for certificate in
                             Into::<Vec<rustls::pki_types::CertificateDer<'static>>>::into(
                                 certificates,
@@ -293,7 +304,18 @@ impl ConnectingStream {
                     }
                     ClientConfig::builder()
                         .with_root_certificates(cert_store)
-                        .with_no_client_auth()
+                };
+                let config = if let Some(identity) = &options.client_tls_identity {
+                    let ClientTlsIdentity::Pem { key, certs } = identity;
+                    builder.with_client_auth_cert(certs.clone().into(), key.clone_key())
+                } else {
+                    Ok(builder.with_no_client_auth())
+                };
+                let config = match config {
+                    Ok(config) => config,
+                    Err(err) => {
+                        return Self { state: State::tls_err(err) };
+                    },
                 };
                 Self {
                     state: State::tls_wait(Box::pin(async move {

src/io/transport.rs

@@ -287,6 +287,15 @@ impl Stream for ClickhouseTransport {
         }
 
         if *this.done {
+            // We still have may have something in buffer, since the client may read something
+            // first, and only after the server will close the connection, likely in case of
+            // exception, let's try to parse it here
+            if !this.rd.is_empty() {
+                if let Poll::Ready(ret) = this.try_parse_msg()? {
+                    return Poll::Ready(ret.map(Ok));
+                }
+            }
+
             return Poll::Ready(None);
         }
 

src/lib.rs

@@ -328,7 +328,7 @@ impl ClientHandle {
         }
 
         self.inner = h;
-        self.context.server_info = info.unwrap();
+        self.context.server_info = info.ok_or(Error::Other("Missing Hello/Exception packet".into()))?;
         Ok(())
     }
 

src/types/mod.rs

@@ -21,6 +21,8 @@ pub use self::{
     value::Value,
     value_ref::ValueRef,
 };
+#[cfg(feature = "_tls")]
+pub use self::options::ClientTlsIdentity;
 
 pub(crate) use self::{
     cmd::Cmd,