From 2e9c1664691a178845d323f57091751aa86c6705 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 29 Oct 2024 10:01:47 +1100 Subject: [PATCH 1/4] Allow jsonb columns as encrtyped targets --- sql/021-config-functions.sql | 13 ++++-- sql/030-encryptindex.sql | 2 +- tests/encryptindex.sql | 79 ++++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/sql/021-config-functions.sql b/sql/021-config-functions.sql index 87c6f751..d67e7a04 100644 --- a/sql/021-config-functions.sql +++ b/sql/021-config-functions.sql @@ -228,18 +228,25 @@ $$ LANGUAGE plpgsql; DROP FUNCTION IF EXISTS cs_encrypt_v1(); -CREATE FUNCTION cs_encrypt_v1() +CREATE FUNCTION cs_encrypt_v1(force boolean DEFAULT false) RETURNS boolean AS $$ BEGIN - IF NOT cs_ready_for_encryption_v1() THEN - RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; + + IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting') THEN + RAISE EXCEPTION 'An encryption is already in progress'; END IF; IF NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN RAISE EXCEPTION 'No pending configuration exists to encrypt'; END IF; + IF NOT force THEN + IF NOT cs_ready_for_encryption_v1() THEN + RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; + END IF; + END IF; + UPDATE cs_configuration_v1 SET state = 'encrypting' WHERE state = 'pending'; RETURN true; END; diff --git a/sql/030-encryptindex.sql b/sql/030-encryptindex.sql index afcf1dc0..95ae358b 100644 --- a/sql/030-encryptindex.sql +++ b/sql/030-encryptindex.sql @@ -87,7 +87,7 @@ AS $$ LEFT JOIN information_schema.columns s ON s.table_name = c.table_name AND (s.column_name = c.column_name OR s.column_name = c.column_name || '_encrypted') AND - s.domain_name = 'cs_encrypted_v1'; + (s.domain_name = 'cs_encrypted_v1' OR s.data_type = 'jsonb'); $$ LANGUAGE sql; diff --git a/tests/encryptindex.sql b/tests/encryptindex.sql index 61f04abf..26d3e420 100644 --- a/tests/encryptindex.sql +++ b/tests/encryptindex.sql @@ -114,7 +114,7 @@ $$ LANGUAGE plpgsql; -- ----------------------------------------------- --- Start encryptindexing wwith no target table +-- Start encryptindexing with no target table -- -- The schema should be validated first. -- Users table does not exist, so should fail. @@ -129,7 +129,7 @@ DO $$ BEGIN PERFORM cs_encrypt_v1(); - RAISE NOTICE 'Missinbg users table. Encrypt should have failed.'; + RAISE NOTICE 'Missing users table. Encrypt should have failed.'; ASSERT false; -- skipped by exception EXCEPTION WHEN OTHERS THEN @@ -143,6 +143,29 @@ DO $$ $$ LANGUAGE plpgsql; +-- ----------------------------------------------- +-- FORCE start encryptindexing with no target table +-- +-- Schema validation is skipped +-- ----------------------------------------------- +DROP TABLE IF EXISTS users; +TRUNCATE TABLE cs_configuration_v1; + + +DO $$ + BEGIN + PERFORM cs_add_index_v1('users', 'name', 'match'); + + PERFORM cs_encrypt_v1(true); + RAISE NOTICE 'Missing users table. Encrypt should have failed.'; + + -- configuration state should be changed + ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); + + END; +$$ LANGUAGE plpgsql; + -- ----------------------------------------------- -- With existing active config @@ -171,7 +194,7 @@ INSERT INTO cs_configuration_v1 (state, data) VALUES ( }'::jsonb ); --- Create a table with multiple plaintext columns +-- Create a table with plaintext and encrypted columns DROP TABLE IF EXISTS users; CREATE TABLE users ( @@ -195,6 +218,56 @@ DO $$ $$ LANGUAGE plpgsql; +-- ----------------------------------------------- +-- With existing active config and an updated schema using a raw JSONB column +-- Start encryptindexing +-- The active config is unchanged +-- The pending config should now be encrypting +-- ----------------------------------------------- +TRUNCATE TABLE cs_configuration_v1; + +-- create an active configuration +INSERT INTO cs_configuration_v1 (state, data) VALUES ( + 'active', + '{ + "v": 1, + "tables": { + "users": { + "name": { + "cast_as": "text", + "indexes": { + "unique": {} + } + } + } + } + }'::jsonb +); + +-- Create a table with plaintext and jsonb column +DROP TABLE IF EXISTS users; +CREATE TABLE users +( + id bigint GENERATED ALWAYS AS IDENTITY, + name TEXT, + name_encrypted jsonb, + PRIMARY KEY(id) +); + + +-- An encrypting config should exist +DO $$ + BEGIN + PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM cs_encrypt_v1(); + + ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'active')); + ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); + ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + END; +$$ LANGUAGE plpgsql; + + -- ----------------------------------------------- -- With existing active config -- Activate encrypting config From 7a473cd0f37770b853e0b2b083b992677c44dd75 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 29 Oct 2024 11:36:52 +1100 Subject: [PATCH 2/4] README for SQL code --- sql/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 sql/README.md diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 00000000..34ea8c4f --- /dev/null +++ b/sql/README.md @@ -0,0 +1,13 @@ +### Adding SQL + + +- Never drop the configuration table as it may contain customer data and needs to live across EQL versions +- Everything else should have a DROP IF EXISTS +- Functions should be DROP and CREATE, instead of CREATE OR REPLACE + - Data types cannot be changed once created, so dropping first is more flexible +- Keep DROP and CREATE together in the code +- Types need to be dropped last, add to the `666-drop_types.sql` + + + + From 92ca509a6ef94809cd4c2cad254d7638eaf23723 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 29 Oct 2024 13:04:31 +1100 Subject: [PATCH 3/4] Update sql/README.md Co-authored-by: Lindsay Holmwood --- sql/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/README.md b/sql/README.md index 34ea8c4f..6a14605e 100644 --- a/sql/README.md +++ b/sql/README.md @@ -2,10 +2,10 @@ - Never drop the configuration table as it may contain customer data and needs to live across EQL versions -- Everything else should have a DROP IF EXISTS -- Functions should be DROP and CREATE, instead of CREATE OR REPLACE +- Everything else should have a `DROP IF EXISTS` +- Functions should be `DROP` and `CREATE`, instead of `CREATE OR REPLACE` - Data types cannot be changed once created, so dropping first is more flexible -- Keep DROP and CREATE together in the code +- Keep `DROP` and `CREATE` together in the code - Types need to be dropped last, add to the `666-drop_types.sql` From 84b5f3177e3bb24385316aedd75f6692617f2e49 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 29 Oct 2024 13:05:24 +1100 Subject: [PATCH 4/4] document cs_encrypt force --- sql/021-config-functions.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/021-config-functions.sql b/sql/021-config-functions.sql index d67e7a04..511b53bf 100644 --- a/sql/021-config-functions.sql +++ b/sql/021-config-functions.sql @@ -226,6 +226,18 @@ AS $$ $$ LANGUAGE plpgsql; + +-- +-- +-- Marks the currently `pending` configuration as `encrypting`. +-- +-- Validates the database schema and raises an exception if the configured columns are not of `jsonb` or `cs_encrypted_v1` type. +-- +-- Accepts an optional `force` parameter. +-- If `force` is `true`, the schema validation is skipped. +-- +-- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt. +-- DROP FUNCTION IF EXISTS cs_encrypt_v1(); CREATE FUNCTION cs_encrypt_v1(force boolean DEFAULT false)