diff --git a/Cargo.lock b/Cargo.lock index a48498701fe2d..3099270c94fa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3035,7 +3035,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.1" -source = "git+https://github.com/MaterializeInc/rust-postgres#fea178cfc3114d0c20622cb4372d78b22282aa72" +source = "git+https://github.com/MaterializeInc/rust-postgres#0f57dede5f8633f079a658ddbafe059b27b719ba" dependencies = [ "bytes", "fallible-iterator", @@ -3048,7 +3048,7 @@ dependencies = [ [[package]] name = "postgres-openssl" version = "0.5.0" -source = "git+https://github.com/MaterializeInc/rust-postgres#fea178cfc3114d0c20622cb4372d78b22282aa72" +source = "git+https://github.com/MaterializeInc/rust-postgres#0f57dede5f8633f079a658ddbafe059b27b719ba" dependencies = [ "futures", "openssl", @@ -3060,7 +3060,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.1" -source = "git+https://github.com/MaterializeInc/rust-postgres#fea178cfc3114d0c20622cb4372d78b22282aa72" +source = "git+https://github.com/MaterializeInc/rust-postgres#0f57dede5f8633f079a658ddbafe059b27b719ba" dependencies = [ "base64", "byteorder", @@ -3077,7 +3077,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.1" -source = "git+https://github.com/MaterializeInc/rust-postgres#fea178cfc3114d0c20622cb4372d78b22282aa72" +source = "git+https://github.com/MaterializeInc/rust-postgres#0f57dede5f8633f079a658ddbafe059b27b719ba" dependencies = [ "bytes", "chrono", @@ -4891,7 +4891,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.1" -source = "git+https://github.com/MaterializeInc/rust-postgres#fea178cfc3114d0c20622cb4372d78b22282aa72" +source = "git+https://github.com/MaterializeInc/rust-postgres#0f57dede5f8633f079a658ddbafe059b27b719ba" dependencies = [ "async-trait", "byteorder", diff --git a/src/postgres-util/src/lib.rs b/src/postgres-util/src/lib.rs index aa7cf32307480..295dbc73cfba6 100644 --- a/src/postgres-util/src/lib.rs +++ b/src/postgres-util/src/lib.rs @@ -11,10 +11,10 @@ use std::fmt; -use anyhow::anyhow; -use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode}; +use anyhow::{anyhow, bail}; +use openssl::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode}; use postgres_openssl::MakeTlsConnector; -use tokio_postgres::types::Type as PgType; +use tokio_postgres::{config::SslMode, types::Type as PgType}; use tokio_postgres::{Client, Config}; use sql_parser::ast::display::{AstDisplay, AstFormatter}; @@ -88,13 +88,59 @@ pub struct TableInfo { pub schema: Vec, } -/// Creates a TLS connector that respects the Postgres' connector Config, in particular `sslmode`. -pub fn make_tls(_config: &Config) -> Result { +/// Creates a TLS connector for the given [`Config`]. +pub fn make_tls(config: &Config) -> Result { let mut builder = SslConnector::builder(SslMethod::tls_client())?; - // Currently supported sslmodes (disable, prefer, require) don't verify peer certificates. - // todo(uce): add additional sslmodes (see #6716) - builder.set_verify(SslVerifyMode::NONE); - Ok(MakeTlsConnector::new(builder.build())) + // The mode dictates whether we verify peer certs and hostnames. By default, Postgres is + // pretty relaxed and recommends SslMode::VerifyCa or SslMode::VerifyFull for security. + // + // For more details, check out Table 33.1. SSL Mode Descriptions in + // https://postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION. + let (verify_mode, verify_hostname) = match config.get_ssl_mode() { + SslMode::Disable | SslMode::Prefer => (SslVerifyMode::NONE, false), + SslMode::Require => match config.get_ssl_root_cert() { + // If a root CA file exists, the behavior of sslmode=require will be the same as + // that of verify-ca, meaning the server certificate is validated against the CA. + // + // For more details, check out the note about backwards compatibility in + // https://postgresql.org/docs/current/libpq-ssl.html#LIBQ-SSL-CERTIFICATES. + Some(_) => (SslVerifyMode::PEER, false), + None => (SslVerifyMode::NONE, false), + }, + SslMode::VerifyCa => (SslVerifyMode::PEER, false), + SslMode::VerifyFull => (SslVerifyMode::PEER, true), + _ => panic!("unexpected sslmode {:?}", config.get_ssl_mode()), + }; + + // Configure peer verification + builder.set_verify(verify_mode); + + // Configure certificates + match (config.get_ssl_cert(), config.get_ssl_key()) { + (Some(ssl_cert), Some(ssl_key)) => { + builder.set_certificate_file(ssl_cert, SslFiletype::PEM)?; + builder.set_private_key_file(ssl_key, SslFiletype::PEM)?; + } + (None, Some(_)) => bail!("must provide both sslcert and sslkey, but only provided sslkey"), + (Some(_), None) => bail!("must provide both sslcert and sslkey, but only provided sslcert"), + _ => {} + } + if let Some(ssl_root_cert) = config.get_ssl_root_cert() { + builder.set_ca_file(ssl_root_cert)? + } + + let mut tls_connector = MakeTlsConnector::new(builder.build()); + + // Configure hostname verification + match (verify_mode, verify_hostname) { + (SslVerifyMode::PEER, false) => tls_connector.set_callback(|connect, _| { + connect.set_verify_hostname(false); + Ok(()) + }), + _ => {} + } + + Ok(tls_connector) } /// Fetches table schema information from an upstream Postgres source for all tables that are part diff --git a/test/pg-cdc/mzcompose.yml b/test/pg-cdc/mzcompose.yml index b1572a019f589..9ff8369de93b6 100644 --- a/test/pg-cdc/mzcompose.yml +++ b/test/pg-cdc/mzcompose.yml @@ -7,7 +7,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -version: '3.7' +version: "3.7" mzworkflows: pg-cdc: @@ -22,6 +22,10 @@ mzworkflows: command: ${TD_TEST:-*.td} services: + test-certs: + mzbuild: test-certs + volumes: + - secrets:/secrets testdrive-svc: mzbuild: testdrive entrypoint: @@ -41,9 +45,12 @@ services: command: --experimental --disable-telemetry ports: - 6875 + volumes: + - secrets:/share/secrets environment: - - MZ_DEV=1 - - MZ_LOG + - MZ_DEV=1 + - MZ_LOG + depends_on: [test-certs] postgres: mzbuild: postgres @@ -58,3 +65,5 @@ services: -c max_replication_slots=20 -c ssl=on -c hba_file=/share/conf/pg_hba.conf +volumes: + secrets: diff --git a/test/pg-cdc/pg-cdc-ssl.td b/test/pg-cdc/pg-cdc-ssl.td index f212c35c48572..08ecb07abe932 100644 --- a/test/pg-cdc/pg-cdc-ssl.td +++ b/test/pg-cdc/pg-cdc-ssl.td @@ -32,6 +32,9 @@ CREATE USER hostssl LOGIN SUPERUSER; DROP USER IF EXISTS hostnossl; CREATE USER hostnossl LOGIN SUPERUSER; +DROP USER IF EXISTS certuser; +CREATE USER certuser LOGIN SUPERUSER; + DROP TABLE IF EXISTS numbers; CREATE TABLE numbers (number int PRIMARY KEY, is_prime bool, name text); ALTER TABLE numbers REPLICA IDENTITY FULL; @@ -134,6 +137,52 @@ INSERT INTO numbers VALUES (2, true, 'two'); $ postgres-execute connection=postgres://postgres:postgres@postgres:5432 DELETE FROM numbers WHERE number = 2; +# server: hostssl, client: verify-ca => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostssl dbname=postgres sslmode=verify-ca' + PUBLICATION 'mz_source'; +self signed certificate in certificate chain + +# server: hostssl, client: verify-ca => OK +> CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostssl dbname=postgres sslmode=verify-ca sslrootcert=/share/secrets/ca.crt' + PUBLICATION 'mz_source'; +> CREATE VIEWS FROM SOURCE "mz_source" ("numbers") +> SELECT * FROM "numbers"; +1 true one +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +INSERT INTO numbers VALUES (2, true, 'two'); +> SELECT * FROM "numbers"; +1 true one +2 true two +> DROP VIEW "numbers"; +> DROP SOURCE "mz_source"; +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +DELETE FROM numbers WHERE number = 2; + +# server: hostssl, client: verify-full => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostssl dbname=postgres sslmode=verify-full' + PUBLICATION 'mz_source'; +self signed certificate in certificate chain + +# server: hostssl, client: verify-full => OK +> CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostssl dbname=postgres sslmode=verify-full sslrootcert=/share/secrets/ca.crt' + PUBLICATION 'mz_source'; +> CREATE VIEWS FROM SOURCE "mz_source" ("numbers") +> SELECT * FROM "numbers"; +1 true one +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +INSERT INTO numbers VALUES (2, true, 'two'); +> SELECT * FROM "numbers"; +1 true one +2 true two +> DROP VIEW "numbers"; +> DROP SOURCE "mz_source"; +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +DELETE FROM numbers WHERE number = 2; + # server: hostnossl, client: disable => OK > CREATE MATERIALIZED SOURCE "mz_source" FROM POSTGRES HOST 'host=postgres port=5432 user=hostnossl sslmode=disable dbname=postgres' @@ -163,3 +212,107 @@ db error: FATAL: no pg_hba.conf entry for host "(HOST)", user "hostnossl", datab FROM POSTGRES HOST 'host=postgres port=5432 user=hostnossl sslmode=require dbname=postgres' PUBLICATION 'mz_source'; db error: FATAL: no pg_hba.conf entry for host "(HOST)", user "hostnossl", database "postgres", SSL on + +# server: hostnossl, client: verify-ca => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostnossl sslmode=verify-ca dbname=postgres' + PUBLICATION 'mz_source'; +self signed certificate in certificate chain + +# server: hostnossl, client: verify-full => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=hostnossl sslmode=verify-full dbname=postgres' + PUBLICATION 'mz_source'; +self signed certificate in certificate chain + +# server: certuser, client: require => OK +> CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=require sslcert=/share/secrets/certuser.crt sslkey=/share/secrets/certuser.key sslrootcert=/share/secrets/ca.crt dbname=postgres' + PUBLICATION 'mz_source' +> CREATE VIEWS FROM SOURCE "mz_source" ("numbers") +> SELECT * FROM "numbers"; +1 true one +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +INSERT INTO numbers VALUES (2, true, 'two'); +> SELECT * FROM "numbers"; +1 true one +2 true two +> DROP VIEW "numbers"; +> DROP SOURCE "mz_source"; +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +DELETE FROM numbers WHERE number = 2; + +# server: certuser, client: verify-ca => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser dbname=postgres sslmode=verify-ca sslrootcert=/share/secrets/ca.crt' + PUBLICATION 'mz_source' +db error: FATAL: connection requires a valid client certificate + +# server: certuser, client: verify-ca (wrong cert) => ERROR +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser dbname=postgres sslmode=verify-ca sslcert=/share/secrets/postgres.crt sslkey=/share/secrets/postgres.key sslrootcert=/share/secrets/ca.crt' + PUBLICATION 'mz_source' +db error: FATAL: certificate authentication failed for user "certuser" + +# server: certuser, client: verify-ca => OK +> CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/certuser.crt sslkey=/share/secrets/certuser.key sslrootcert=/share/secrets/ca.crt dbname=postgres' + PUBLICATION 'mz_source' +> CREATE VIEWS FROM SOURCE "mz_source" ("numbers") +> SELECT * FROM "numbers"; +1 true one +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +INSERT INTO numbers VALUES (2, true, 'two'); +> SELECT * FROM "numbers"; +1 true one +2 true two +> DROP VIEW "numbers"; +> DROP SOURCE "mz_source"; +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +DELETE FROM numbers WHERE number = 2; + +# server: certuser, client: verify-full => OK +> CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/certuser.crt sslkey=/share/secrets/certuser.key sslrootcert=/share/secrets/ca.crt dbname=postgres' + PUBLICATION 'mz_source' +> CREATE VIEWS FROM SOURCE "mz_source" ("numbers") +> SELECT * FROM "numbers"; +1 true one +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +INSERT INTO numbers VALUES (2, true, 'two'); +> SELECT * FROM "numbers"; +1 true one +2 true two +> DROP VIEW "numbers"; +> DROP SOURCE "mz_source"; +$ postgres-execute connection=postgres://postgres:postgres@postgres:5432 +DELETE FROM numbers WHERE number = 2; + +# missing sslcert +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/missing sslkey=/share/secrets/certuser.key sslrootcert=/share/secrets/ca.crt dbname=postgres' + PUBLICATION 'mz_source' +invalid connection string: invalid value for option `sslcert` + +# missing sslkey +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/certuser.crt sslkey=/share/secrets/missing sslrootcert=/share/secrets/ca.crt dbname=postgres' + PUBLICATION 'mz_source' +invalid connection string: invalid value for option `sslkey` + +# missing sslrootcert +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/certuser.crt sslkey=/share/secrets/certuser.key sslrootcert=/share/secrets/missing dbname=postgres' + PUBLICATION 'mz_source' +invalid connection string: invalid value for option `sslrootcert` + +# require both sslcert and sslkey +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslcert=/share/secrets/certuser.crt dbname=postgres' + PUBLICATION 'mz_source' +must provide both sslcert and sslkey, but only provided sslcert + +! CREATE MATERIALIZED SOURCE "mz_source" + FROM POSTGRES HOST 'host=postgres port=5432 user=certuser sslmode=verify-ca sslkey=/share/secrets/certuser.key dbname=postgres' + PUBLICATION 'mz_source' +must provide both sslcert and sslkey, but only provided sslkey diff --git a/test/pg-cdc/pg_hba.conf b/test/pg-cdc/pg_hba.conf index 316add842beed..4217f17b2145e 100644 --- a/test/pg-cdc/pg_hba.conf +++ b/test/pg-cdc/pg_hba.conf @@ -15,3 +15,4 @@ host all no_replication all trust host all host all trust hostssl all hostssl all trust hostnossl all hostnossl all trust +hostssl all certuser all cert diff --git a/test/pg-cdc/postgres/setup-postgres.sh b/test/pg-cdc/postgres/setup-postgres.sh index 06a89de78612d..d51ddfe1f9526 100755 --- a/test/pg-cdc/postgres/setup-postgres.sh +++ b/test/pg-cdc/postgres/setup-postgres.sh @@ -14,4 +14,5 @@ set -e cat >> "$PGDATA/postgresql.conf" <<-EOCONF ssl_cert_file = '/share/secrets/postgres.crt' ssl_key_file = '/share/secrets/postgres.key' +ssl_ca_file = '/share/secrets/ca.crt' EOCONF diff --git a/test/test-certs/create-certs.sh b/test/test-certs/create-certs.sh index 38d202d7213d8..2aee54c00aeb8 100755 --- a/test/test-certs/create-certs.sh +++ b/test/test-certs/create-certs.sh @@ -28,7 +28,7 @@ openssl req \ -passin pass:$SSL_SECRET \ -passout pass:$SSL_SECRET -for i in kafka kafka1 kafka2 schema-registry materialized producer postgres +for i in kafka kafka1 kafka2 schema-registry materialized producer postgres certuser do # Create key & csr openssl req -nodes \ diff --git a/test/test-certs/openssl.cnf b/test/test-certs/openssl.cnf index 510f1fbc1215b..e708ca5f94f47 100644 --- a/test/test-certs/openssl.cnf +++ b/test/test-certs/openssl.cnf @@ -39,3 +39,9 @@ authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment subjectAltName = DNS:schema-registry + +[ certuser ] +authorityKeyIdentifier=keyid,issuer +basicConstraints=CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = DNS:schema-registry