From 8aef2828103dd67e583fa18370e0fb39210131c2 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 14 May 2025 09:22:58 +1000 Subject: [PATCH 1/4] fix: cast literals and params as encrypted --- packages/eql-mapper/src/lib.rs | 18 +++++----- ...erals.rs => cast_literals_as_encrypted.rs} | 36 ++++--------------- ..._in_row.rs => cast_params_as_encrypted.rs} | 21 +++++------ .../src/transformation_rules/helpers.rs | 36 +++++++++---------- .../src/transformation_rules/mod.rs | 8 ++--- .../eql-mapper/src/type_checked_statement.rs | 10 +++--- 6 files changed, 49 insertions(+), 80 deletions(-) rename packages/eql-mapper/src/transformation_rules/{replace_plaintext_eql_literals.rs => cast_literals_as_encrypted.rs} (57%) rename packages/eql-mapper/src/transformation_rules/{wrap_eql_params_in_row.rs => cast_params_as_encrypted.rs} (85%) diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index e9988c4..d9fa8d4 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -983,7 +983,7 @@ mod test { } #[test] - fn select_with_literal_subsitution() { + fn select_with_literal_cast_as_encrypted() { // init_tracing(); let schema = resolver(schema! { @@ -1026,14 +1026,14 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "SELECT * FROM employees WHERE salary > ROW('ENCRYPTED'::JSONB)" + "SELECT * FROM employees WHERE salary > 'ENCRYPTED'::JSONB::eql_v1_encrypted" ), Err(err) => panic!("statement transformation failed: {}", err), }; } #[test] - fn insert_with_literal_subsitution() { + fn insert_with_literal_cast_as_encrypted() { // init_tracing(); let schema = resolver(schema! { @@ -1073,7 +1073,7 @@ mod test { )])) { Ok(transformed_statement) => assert_eq!( transformed_statement.to_string(), - "INSERT INTO employees (salary) VALUES (ROW('ENCRYPTED'::JSONB))" + "INSERT INTO employees (salary) VALUES ('ENCRYPTED'::JSONB::eql_v1_encrypted)" ), Err(err) => panic!("statement transformation failed: {}", err), }; @@ -1392,7 +1392,7 @@ mod test { } #[test] - fn eql_params_are_wrapped_in_row() { + fn select_with_params_cast_as_encrypted() { // init_tracing(); let schema = resolver(schema! { tables: { @@ -1415,7 +1415,7 @@ mod test { Ok(statement) => { assert_eq!( statement.to_string(), - "SELECT * FROM employees WHERE eql_col = ROW($1::JSONB) AND native_col = $2" + "SELECT * FROM employees WHERE eql_col = $1::JSONB::eql_v1_encrypted AND native_col = $2" ); } Err(err) => panic!("transformation failed: {err}"), @@ -1450,7 +1450,7 @@ mod test { Ok(statement) => { assert_eq!( statement.to_string(), - "SELECT eql_v1.jsonb_path_query(eql_col, ROW(''::JSONB)), jsonb_path_query(native_col, '$.not-secret') FROM employees" + "SELECT eql_v1.jsonb_path_query(eql_col, ''::JSONB::eql_v1_encrypted), jsonb_path_query(native_col, '$.not-secret') FROM employees" ); } Err(err) => panic!("transformation failed: {err}"), @@ -1567,7 +1567,7 @@ mod test { .map(|expr| match expr { ast::Expr::Identifier(ident) => ident.to_string(), ast::Expr::Value(ast::Value::SingleQuotedString(s)) => { - format!("ROW(''::JSONB)", s) + format!("''::JSONB::eql_v1_encrypted", s) } _ => panic!("unsupported expr type in test util"), }) @@ -1623,7 +1623,7 @@ mod test { match typed.transform(test_helpers::dummy_encrypted_json_selector(&statement, ast::Value::SingleQuotedString("medications".to_owned()))) { Ok(statement) => assert_eq!( statement.to_string(), - format!("SELECT id, notes {} ROW(''::JSONB) AS meds FROM patients", op) + format!("SELECT id, notes {} ''::JSONB::eql_v1_encrypted AS meds FROM patients", op) ), Err(err) => panic!("transformation failed: {err}"), } diff --git a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs similarity index 57% rename from packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs rename to packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs index a153b11..af19513 100644 --- a/packages/eql-mapper/src/transformation_rules/replace_plaintext_eql_literals.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_literals_as_encrypted.rs @@ -1,27 +1,25 @@ use std::{any::type_name, collections::HashMap}; -use sqltk::parser::ast::{ - CastKind, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, - FunctionArguments, Ident, ObjectName, Value, -}; +use sqltk::parser::ast::{Expr, Value}; use sqltk::{NodeKey, NodePath, Visitable}; use crate::EqlMapperError; +use super::helpers::cast_as_encrypted; use super::TransformationRule; #[derive(Debug)] -pub struct ReplacePlaintextEqlLiterals<'ast> { +pub struct CastLiteralsAsEncrypted<'ast> { encrypted_literals: HashMap, Value>, } -impl<'ast> ReplacePlaintextEqlLiterals<'ast> { +impl<'ast> CastLiteralsAsEncrypted<'ast> { pub fn new(encrypted_literals: HashMap, Value>) -> Self { Self { encrypted_literals } } } -impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { +impl<'ast> TransformationRule<'ast> for CastLiteralsAsEncrypted<'ast> { fn apply( &mut self, node_path: &NodePath<'ast>, @@ -31,7 +29,7 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { if let Some((Expr::Value(value),)) = node_path.last_1_as::() { if let Some(replacement) = self.encrypted_literals.remove(&NodeKey::new(value)) { let target_node = target_node.downcast_mut::().unwrap(); - *target_node = make_row_expression(replacement); + *target_node = cast_as_encrypted(replacement); return Ok(true); } } @@ -58,25 +56,3 @@ impl<'ast> TransformationRule<'ast> for ReplacePlaintextEqlLiterals<'ast> { } } } - -fn make_row_expression(replacement: Value) -> Expr { - Expr::Function(Function { - name: ObjectName(vec![Ident::new("ROW")]), - uses_odbc_syntax: false, - parameters: FunctionArguments::None, - args: FunctionArguments::List(FunctionArgumentList { - duplicate_treatment: None, - clauses: vec![], - args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Cast { - kind: CastKind::DoubleColon, - expr: Box::new(Expr::Value(replacement)), - data_type: DataType::JSONB, - format: None, - }))], - }), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }) -} diff --git a/packages/eql-mapper/src/transformation_rules/wrap_eql_params_in_row.rs b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs similarity index 85% rename from packages/eql-mapper/src/transformation_rules/wrap_eql_params_in_row.rs rename to packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs index 514ddf8..ee89fa6 100644 --- a/packages/eql-mapper/src/transformation_rules/wrap_eql_params_in_row.rs +++ b/packages/eql-mapper/src/transformation_rules/cast_params_as_encrypted.rs @@ -1,26 +1,23 @@ -use std::collections::HashMap; -use std::sync::Arc; - +use super::helpers::cast_as_encrypted; +use super::TransformationRule; +use crate::{EqlMapperError, Type}; use sqltk::parser::ast::{Expr, Value}; use sqltk::{NodeKey, NodePath, Visitable}; - -use crate::{EqlMapperError, Type}; - -use super::helpers::make_row_expression; -use super::TransformationRule; +use std::collections::HashMap; +use std::sync::Arc; #[derive(Debug)] -pub struct WrapEqlParamsInRow<'ast> { +pub struct CastParamsAsEncrypted<'ast> { node_types: Arc, Type>>, } -impl<'ast> WrapEqlParamsInRow<'ast> { +impl<'ast> CastParamsAsEncrypted<'ast> { pub fn new(node_types: Arc, Type>>) -> Self { Self { node_types } } } -impl<'ast> TransformationRule<'ast> for WrapEqlParamsInRow<'ast> { +impl<'ast> TransformationRule<'ast> for CastParamsAsEncrypted<'ast> { fn apply( &mut self, node_path: &NodePath<'ast>, @@ -33,7 +30,7 @@ impl<'ast> TransformationRule<'ast> for WrapEqlParamsInRow<'ast> { unreachable!("the Expr is known to be Expr::Value(Value::Placeholder(_))") }; - *expr = make_row_expression(value); + *expr = cast_as_encrypted(value); return Ok(true); } } diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index 675e9a2..a4dc965 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -48,26 +48,22 @@ pub(crate) fn wrap_in_1_arg_function(expr: Expr, name: ObjectName) -> Expr { }) } -pub(crate) fn make_row_expression(wrapped: sqltk::parser::ast::Value) -> Expr { - Expr::Function(Function { - name: ObjectName(vec![Ident::new("ROW")]), - uses_odbc_syntax: false, - parameters: FunctionArguments::None, - args: FunctionArguments::List(FunctionArgumentList { - duplicate_treatment: None, - clauses: vec![], - args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Cast { - kind: CastKind::DoubleColon, - expr: Box::new(Expr::Value(wrapped)), - data_type: DataType::JSONB, - format: None, - }))], - }), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }) +pub(crate) fn cast_as_encrypted(wrapped: sqltk::parser::ast::Value) -> Expr { + let cast_jsonb = Expr::Cast { + kind: CastKind::DoubleColon, + expr: Box::new(Expr::Value(wrapped)), + data_type: DataType::JSONB, + format: None, + }; + + let encrypted_type = ObjectName(vec![Ident::new("eql_v1_encrypted")]); + + Expr::Cast { + kind: CastKind::DoubleColon, + expr: Box::new(cast_jsonb), + data_type: DataType::Custom(encrypted_type, vec![]), + format: None, + } } struct ContainsExprWithType<'ast, 't> { diff --git a/packages/eql-mapper/src/transformation_rules/mod.rs b/packages/eql-mapper/src/transformation_rules/mod.rs index 7054e70..33964e9 100644 --- a/packages/eql-mapper/src/transformation_rules/mod.rs +++ b/packages/eql-mapper/src/transformation_rules/mod.rs @@ -11,24 +11,24 @@ mod helpers; +mod cast_literals_as_encrypted; +mod cast_params_as_encrypted; mod fail_on_placeholder_change; mod group_by_eql_col; mod preserve_effective_aliases; -mod replace_plaintext_eql_literals; mod rewrite_standard_sql_fns_on_eql_types; mod wrap_eql_cols_in_order_by_with_ore_fn; -mod wrap_eql_params_in_row; mod wrap_grouped_eql_col_in_aggregate_fn; use std::marker::PhantomData; +pub(crate) use cast_literals_as_encrypted::*; +pub(crate) use cast_params_as_encrypted::*; pub(crate) use fail_on_placeholder_change::*; pub(crate) use group_by_eql_col::*; pub(crate) use preserve_effective_aliases::*; -pub(crate) use replace_plaintext_eql_literals::*; pub(crate) use rewrite_standard_sql_fns_on_eql_types::*; pub(crate) use wrap_eql_cols_in_order_by_with_ore_fn::*; -pub(crate) use wrap_eql_params_in_row::*; pub(crate) use wrap_grouped_eql_col_in_aggregate_fn::*; use crate::EqlMapperError; diff --git a/packages/eql-mapper/src/type_checked_statement.rs b/packages/eql-mapper/src/type_checked_statement.rs index 8c0be31..425390c 100644 --- a/packages/eql-mapper/src/type_checked_statement.rs +++ b/packages/eql-mapper/src/type_checked_statement.rs @@ -4,10 +4,10 @@ use sqltk::parser::ast::{self, Statement}; use sqltk::{AsNodeKey, NodeKey, Transformable}; use crate::{ - DryRunnable, EqlMapperError, EqlValue, FailOnPlaceholderChange, GroupByEqlCol, Param, - PreserveEffectiveAliases, Projection, ReplacePlaintextEqlLiterals, + CastLiteralsAsEncrypted, CastParamsAsEncrypted, DryRunnable, EqlMapperError, EqlValue, + FailOnPlaceholderChange, GroupByEqlCol, Param, PreserveEffectiveAliases, Projection, RewriteStandardSqlFnsOnEqlTypes, TransformationRule, Type, Value, - WrapEqlColsInOrderByWithOreFn, WrapEqlParamsInRow, WrapGroupedEqlColInAggregateFn, + WrapEqlColsInOrderByWithOreFn, WrapGroupedEqlColInAggregateFn, }; /// A `TypeCheckedStatement` is returned from a successful call to [`crate::type_check`]. @@ -145,9 +145,9 @@ impl<'ast> TypeCheckedStatement<'ast> { GroupByEqlCol::new(Arc::clone(&self.node_types)), WrapEqlColsInOrderByWithOreFn::new(Arc::clone(&self.node_types)), PreserveEffectiveAliases, - ReplacePlaintextEqlLiterals::new(encrypted_literals), + CastLiteralsAsEncrypted::new(encrypted_literals), FailOnPlaceholderChange::new(), - WrapEqlParamsInRow::new(Arc::clone(&self.node_types)), + CastParamsAsEncrypted::new(Arc::clone(&self.node_types)), )) } } From 8f53a362e97f309eca313534c0e5105932993d08 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 14 May 2025 10:23:39 +1000 Subject: [PATCH 2/4] fix: skip seralizing unconfigured index terms --- packages/cipherstash-proxy/src/encrypt/mod.rs | 1 - packages/cipherstash-proxy/src/eql/mod.rs | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/cipherstash-proxy/src/encrypt/mod.rs b/packages/cipherstash-proxy/src/encrypt/mod.rs index 681e4b7..386fa8d 100644 --- a/packages/cipherstash-proxy/src/encrypt/mod.rs +++ b/packages/cipherstash-proxy/src/encrypt/mod.rs @@ -60,7 +60,6 @@ impl Encrypt { let rows = client .query("SELECT eql_v1.version() AS version;", &[]) .await; - // let rows = client.query("SELECT 'WAT' AS version;", &[]).await; match rows { Ok(rows) => rows.first().map(|row| row.get("version")), diff --git a/packages/cipherstash-proxy/src/eql/mod.rs b/packages/cipherstash-proxy/src/eql/mod.rs index 03ec014..a896be2 100644 --- a/packages/cipherstash-proxy/src/eql/mod.rs +++ b/packages/cipherstash-proxy/src/eql/mod.rs @@ -84,25 +84,25 @@ pub struct EqlEncryptedBody { #[derive(Debug, Deserialize, Serialize, Default)] pub struct EqlEncryptedIndexes { - #[serde(rename = "o")] + #[serde(rename = "o", skip_serializing_if = "Option::is_none")] pub(crate) ore_index: Option>, - #[serde(rename = "m")] + #[serde(rename = "m", skip_serializing_if = "Option::is_none")] pub(crate) match_index: Option>, - #[serde(rename = "u")] + #[serde(rename = "u", skip_serializing_if = "Option::is_none")] pub(crate) unique_index: Option, - #[serde(rename = "s")] + #[serde(rename = "s", skip_serializing_if = "Option::is_none")] pub(crate) selector: Option, - #[serde(rename = "b")] + #[serde(rename = "b", skip_serializing_if = "Option::is_none")] pub(crate) blake3_index: Option, - #[serde(rename = "ocf")] + #[serde(rename = "ocf", skip_serializing_if = "Option::is_none")] pub(crate) ore_cclw_fixed_index: Option, - #[serde(rename = "ocv")] + #[serde(rename = "ocv", skip_serializing_if = "Option::is_none")] pub(crate) ore_cclw_var_index: Option, - #[serde(rename = "sv")] + #[serde(rename = "sv", skip_serializing_if = "Option::is_none")] pub(crate) ste_vec_index: Option>, } From 00d85dbae57734853cf90ad84e88418a87beb361 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 14 May 2025 10:29:30 +1000 Subject: [PATCH 3/4] fix: ensure data for update query --- .../cipherstash-proxy-integration/src/map_concat.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/cipherstash-proxy-integration/src/map_concat.rs b/packages/cipherstash-proxy-integration/src/map_concat.rs index 0400522..90abd9e 100644 --- a/packages/cipherstash-proxy-integration/src/map_concat.rs +++ b/packages/cipherstash-proxy-integration/src/map_concat.rs @@ -1,11 +1,19 @@ #[cfg(test)] mod tests { - use crate::common::{connect_with_tls, PROXY}; + use crate::common::{clear, connect_with_tls, id, PROXY}; #[tokio::test] async fn map_concat_regression() { let client = connect_with_tls(PROXY).await; + clear().await; + + let id = id(); + let encrypted_text = "hello@cipherstash.com"; + + let sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; + client.query(sql, &[&id, &encrypted_text]).await.unwrap(); + let sql = "UPDATE encrypted SET encrypted_text = encrypted_text || 'suffix';"; client From f5be624021b3fe1876c23ff357f2d267c59c929d Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 14 May 2025 10:34:32 +1000 Subject: [PATCH 4/4] chore: clippy tweaks --- packages/cipherstash-proxy/src/postgresql/backend.rs | 2 +- .../src/postgresql/messages/data_row.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/backend.rs b/packages/cipherstash-proxy/src/postgresql/backend.rs index 1e162b6..8722b76 100644 --- a/packages/cipherstash-proxy/src/postgresql/backend.rs +++ b/packages/cipherstash-proxy/src/postgresql/backend.rs @@ -265,7 +265,7 @@ where // Each row is converted into Vec> let ciphertexts: Vec> = rows .iter_mut() - .flat_map(|row| row.to_ciphertext(projection_columns)) + .flat_map(|row| row.as_ciphertext(projection_columns)) .collect::>(); let start = Instant::now(); diff --git a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs index 9d5c670..5190d91 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/data_row.rs @@ -20,7 +20,7 @@ pub struct DataColumn { } impl DataRow { - pub fn to_ciphertext( + pub fn as_ciphertext( &mut self, column_configuration: &Vec>, ) -> Vec> { @@ -240,7 +240,7 @@ mod tests { use crate::{ config::{LogConfig, LogLevel}, log, - postgresql::{data, messages::data_row::DataColumn, Column}, + postgresql::{messages::data_row::DataColumn, Column}, }; use bytes::BytesMut; use cipherstash_client::schema::{ColumnConfig, ColumnType}; @@ -270,7 +270,7 @@ mod tests { let mut data_row = DataRow::try_from(&bytes).unwrap(); let column_config = column_config_with_id("encrypted_text"); - let encrypted = data_row.to_ciphertext(&column_config); + let encrypted = data_row.as_ciphertext(&column_config); assert_eq!(encrypted.len(), 2); @@ -299,7 +299,7 @@ mod tests { assert!(data_row.columns[1].bytes.is_some()); let column_config = column_config_with_id("encrypted_text"); - let encrypted = data_row.to_ciphertext(&column_config); + let encrypted = data_row.as_ciphertext(&column_config); assert_eq!(encrypted.len(), 2); @@ -322,7 +322,7 @@ mod tests { assert!(data_row.columns[0].bytes.is_some()); let column_config = vec![column_config("encrypted_jsonb")]; - let encrypted = data_row.to_ciphertext(&column_config); + let encrypted = data_row.as_ciphertext(&column_config); assert_eq!(encrypted.len(), 1); assert!(encrypted[0].is_some()); @@ -358,7 +358,7 @@ mod tests { column_config("encrypted_jsonb"), ]; - let encrypted = data_row.to_ciphertext(&column_config); + let encrypted = data_row.as_ciphertext(&column_config); assert_eq!(encrypted.len(), 10);