From 5bc209b6e06422fd6f5187c7b8bfb2e61de63298 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:26:24 -0400 Subject: [PATCH 1/7] feat: add logic to handle array types for custom decoders --- connection/connection_params.ts | 12 +++++- query/array_parser.ts | 10 +++++ query/decode.ts | 24 +++++++++-- tests/query_client_test.ts | 73 +++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index 7b68ea9c..cc3971f7 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -3,6 +3,7 @@ import { ConnectionParamsError } from "../client/error.ts"; import { fromFileUrl, isAbsolute } from "../deps.ts"; import { OidType } from "../query/oid.ts"; import { DebugControls } from "../debug.ts"; +import { ParseArrayFunc } from "../query/array_parser.ts"; /** * The connection string must match the following URI structure. All parameters but database and user are optional @@ -108,9 +109,16 @@ export type Decoders = { /** * A decoder function that takes a string value and returns a parsed value of some type. - * the Oid is also passed to the function for reference + * + * @param value The string value to parse + * @param oid The OID of the column type the value is from + * @param parseArray A helper function that parses SQL array-formatted strings and parses each array value using a transform function. */ -export type DecoderFunction = (value: string, oid: number) => unknown; +export type DecoderFunction = ( + value: string, + oid: number, + parseArray: ParseArrayFunc, +) => unknown; /** * Control the behavior for the client instance diff --git a/query/array_parser.ts b/query/array_parser.ts index 9fd043bd..312d0406 100644 --- a/query/array_parser.ts +++ b/query/array_parser.ts @@ -6,6 +6,16 @@ type AllowedSeparators = "," | ";"; type ArrayResult = Array>; type Transformer = (value: string) => T; +export type ParseArrayFunc = typeof parseArray; + +/** + * Parse a string into an array of values using the provided transform function. + * + * @param source The string to parse + * @param transform A function to transform each value in the array + * @param separator The separator used to split the string into values + * @returns + */ export function parseArray( source: string, transform: Transformer, diff --git a/query/decode.ts b/query/decode.ts index c2b5ec42..fb13afa3 100644 --- a/query/decode.ts +++ b/query/decode.ts @@ -1,4 +1,4 @@ -import { Oid, OidTypes, OidValue } from "./oid.ts"; +import { Oid, OidType, OidTypes, OidValue } from "./oid.ts"; import { bold, yellow } from "../deps.ts"; import { decodeBigint, @@ -36,6 +36,7 @@ import { decodeTidArray, } from "./decoders.ts"; import { ClientControls } from "../connection/connection_params.ts"; +import { parseArray } from "./array_parser.ts"; export class Column { constructor( @@ -216,12 +217,29 @@ export function decode( // check if there is a custom decoder if (controls?.decoders) { + const oidType = OidTypes[column.typeOid as OidValue]; // check if there is a custom decoder by oid (number) or by type name (string) const decoderFunc = controls.decoders?.[column.typeOid] || - controls.decoders?.[OidTypes[column.typeOid as OidValue]]; + controls.decoders?.[oidType]; if (decoderFunc) { - return decoderFunc(strValue, column.typeOid); + return decoderFunc(strValue, column.typeOid, parseArray); + } // if no custom decoder is found and the oid is for an array type, check if there is + // a decoder for the base type and use that with the array parser + else if (oidType.includes("_array")) { + const baseOidType = oidType.replace("_array", "") as OidType; + // check if the base type is in the Oid object + if (baseOidType in Oid) { + // check if there is a custom decoder for the base type by oid (number) or by type name (string) + const decoderFunc = controls.decoders?.[Oid[baseOidType]] || + controls.decoders?.[baseOidType]; + if (decoderFunc) { + return parseArray( + strValue, + (value: string) => decoderFunc(value, column.typeOid, parseArray), + ); + } + } } } diff --git a/tests/query_client_test.ts b/tests/query_client_test.ts index 0e71da69..c096049a 100644 --- a/tests/query_client_test.ts +++ b/tests/query_client_test.ts @@ -241,6 +241,79 @@ Deno.test( ), ); +Deno.test( + "Custom decoders with arrays", + withClient( + async (client) => { + const result = await client.queryObject( + `SELECT + ARRAY[true, false, true] AS _bool_array, + ARRAY['2024-01-01'::date, '2024-01-02'::date, '2024-01-03'::date] AS _date_array, + ARRAY[1.5:: REAL, 2.5::REAL, 3.5::REAL] AS _float_array, + ARRAY[10, 20, 30] AS _int_array, + ARRAY[ + '{"key1": "value1", "key2": "value2"}'::jsonb, + '{"key3": "value3", "key4": "value4"}'::jsonb, + '{"key5": "value5", "key6": "value6"}'::jsonb + ] AS _jsonb_array, + ARRAY['string1', 'string2', 'string3'] AS _text_array + ;`, + ); + + assertEquals(result.rows, [ + { + _bool_array: [ + { boolean: true }, + { boolean: false }, + { boolean: true }, + ], + _date_array: [ + new Date("2024-01-11T00:00:00.000Z"), + new Date("2024-01-12T00:00:00.000Z"), + new Date("2024-01-13T00:00:00.000Z"), + ], + _float_array: [15, 25, 35], + _int_array: [110, 120, 130], + _jsonb_array: [ + { key1: "value1", key2: "value2" }, + { key3: "value3", key4: "value4" }, + { key5: "value5", key6: "value6" }, + ], + _text_array: ["string1_!", "string2_!", "string3_!"], + }, + ]); + }, + { + controls: { + decoders: { + // convert to object + [Oid.bool]: (value: string) => ({ boolean: value === "t" }), + // 1082 = date : convert to date and add 10 days + "1082": (value: string) => { + const d = new Date(value); + return new Date(d.setDate(d.getDate() + 10)); + }, + // multiply by 20, should not be used! + float4: (value: string) => parseFloat(value) * 20, + // multiply by 10 + float4_array: (value: string, _, parseArray) => + parseArray(value, (v) => parseFloat(v) * 10), + // return 0, should not be used! + [Oid.int4]: () => 0, + // add 100 + [Oid.int4_array]: (value: string, _, parseArray) => + parseArray(value, (v) => parseInt(v, 10) + 100), + // split string and reverse, should not be used! + [Oid.text]: (value: string) => value.split("").reverse(), + // 1009 = text_array : append "_!" to each string + 1009: (value: string, _, parseArray) => + parseArray(value, (v) => `${v}_!`), + }, + }, + }, + ), +); + Deno.test( "Custom decoder precedence", withClient( From ad3e8f9d5f91f4dfbce999c522125ba3ef3c5c00 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:40:53 -0400 Subject: [PATCH 2/7] chore: update docs --- docs/README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 477b86f4..fc802f37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -758,10 +758,10 @@ available: You can also provide custom decoders to the client that will be used to decode the result data. This can be done by setting the `decoders` controls option in the client configuration. This option is a map object where the keys are the -type names or Oid numbers and the values are the custom decoder functions. +type names or OID numbers and the values are the custom decoder functions. You can use it with the decode strategy. Custom decoders take precedence over -the strategy and internal parsers. +the strategy and internal decoders. ```ts { @@ -785,7 +785,36 @@ the strategy and internal parsers. const result = await client.queryObject( "SELECT ID, NAME, IS_ACTIVE FROM PEOPLE", ); - console.log(result.rows[0]); // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} + console.log(result.rows[0]); + // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} +} +``` + +The driver takes care of parsing the related `array` OID types automatically. +For example, if a custom decoder is defined for the `int4` type, it will be applied +when parsing `int4[]` arrays. If needed, you can have separate custom decoders for the +array and non-array types by defining another custom decoders for the array type itself. + +```ts +{ + const client = new Client({ + database: "some_db", + user: "some_user", + controls: { + decodeStrategy: "string", + decoders: { + // Custom decoder for int4 (OID 23 = int4) + // convert to int and multiply by 100 + 23: (value: string) => parseInt(value, 10) * 100, + }, + }, + }); + + const result = await client.queryObject( + "SELECT ARRAY[ 2, 2, 3, 1 ] AS scores, 8 final_score;", + ); + console.log(result.rows[0]); + // { scores: [ 200, 200, 300, 100 ], final_score: 800 } } ``` From cc87d8c31d4d7e126a79c2b724f31f756825948c Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:43:01 -0400 Subject: [PATCH 3/7] chore: fix format --- docs/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index fc802f37..c4763079 100644 --- a/docs/README.md +++ b/docs/README.md @@ -791,9 +791,10 @@ the strategy and internal decoders. ``` The driver takes care of parsing the related `array` OID types automatically. -For example, if a custom decoder is defined for the `int4` type, it will be applied -when parsing `int4[]` arrays. If needed, you can have separate custom decoders for the -array and non-array types by defining another custom decoders for the array type itself. +For example, if a custom decoder is defined for the `int4` type, it will be +applied when parsing `int4[]` arrays. If needed, you can have separate custom +decoders for the array and non-array types by defining another custom decoders +for the array type itself. ```ts { From 9955bf74dbaa3e9418f91eeb97616ebbcb2b6481 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:45:55 -0400 Subject: [PATCH 4/7] chore: bump version, fix type name --- connection/connection_params.ts | 4 ++-- deno.json | 2 +- query/array_parser.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index cc3971f7..ac4f650e 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -3,7 +3,7 @@ import { ConnectionParamsError } from "../client/error.ts"; import { fromFileUrl, isAbsolute } from "../deps.ts"; import { OidType } from "../query/oid.ts"; import { DebugControls } from "../debug.ts"; -import { ParseArrayFunc } from "../query/array_parser.ts"; +import { ParseArrayFunction } from "../query/array_parser.ts"; /** * The connection string must match the following URI structure. All parameters but database and user are optional @@ -117,7 +117,7 @@ export type Decoders = { export type DecoderFunction = ( value: string, oid: number, - parseArray: ParseArrayFunc, + parseArray: ParseArrayFunction, ) => unknown; /** diff --git a/deno.json b/deno.json index 10162a4f..a95580a3 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "lock": false, "name": "@bartlomieju/postgres", - "version": "0.18.1", + "version": "0.19.1", "exports": "./mod.ts" } diff --git a/query/array_parser.ts b/query/array_parser.ts index 312d0406..b7983b41 100644 --- a/query/array_parser.ts +++ b/query/array_parser.ts @@ -6,7 +6,7 @@ type AllowedSeparators = "," | ";"; type ArrayResult = Array>; type Transformer = (value: string) => T; -export type ParseArrayFunc = typeof parseArray; +export type ParseArrayFunction = typeof parseArray; /** * Parse a string into an array of values using the provided transform function. From 7b87470001c7cf27bdec6ac5bba771b8aba6a311 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:51:27 -0400 Subject: [PATCH 5/7] chore: update test readme --- tests/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 10a1a496..e97e1563 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,11 @@ # Testing -To run tests, first prepare your configuration file by copying +To run tests, we recommend using Docker. With Docker, there is no +need to modify any configuration, just run the build and test commands. + +If running tests on your host, prepare your configuration file by copying `config.example.json` into `config.json` and updating it appropriately based on -your environment. If you use the Docker based configuration below there's no -need to modify the configuration. +your environment. ## Running the Tests From 1e4827dc4225c4c5d3c2516cfc42e69b3396dfc3 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:54:34 -0400 Subject: [PATCH 6/7] chore: format readme --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index e97e1563..c8c3e4e9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,7 +1,7 @@ # Testing -To run tests, we recommend using Docker. With Docker, there is no -need to modify any configuration, just run the build and test commands. +To run tests, we recommend using Docker. With Docker, there is no need to modify +any configuration, just run the build and test commands. If running tests on your host, prepare your configuration file by copying `config.example.json` into `config.json` and updating it appropriately based on From d9bd7ee7010dfbe6815650144e63d24f1af81f94 Mon Sep 17 00:00:00 2001 From: hector Date: Thu, 24 Apr 2025 06:40:47 -0400 Subject: [PATCH 7/7] chore: format files --- connection/connection_params.ts | 122 ++++++++++++++++---------------- query/decode.ts | 18 ++--- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index 80566b9f..a55fb804 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -117,7 +117,7 @@ export type Decoders = { export type DecoderFunction = ( value: string, oid: number, - parseArray: ParseArrayFunction + parseArray: ParseArrayFunction, ) => unknown; /** @@ -196,19 +196,21 @@ export type ClientOptions = { }; /** The configuration options required to set up a Client instance */ -export type ClientConfiguration = Required< - Omit< - ClientOptions, - "password" | "port" | "tls" | "connection" | "options" | "controls" +export type ClientConfiguration = + & Required< + Omit< + ClientOptions, + "password" | "port" | "tls" | "connection" | "options" | "controls" + > > -> & { - connection: ConnectionOptions; - controls?: ClientControls; - options: Record; - password?: string; - port: number; - tls: TLSOptions; -}; + & { + connection: ConnectionOptions; + controls?: ClientControls; + options: Record; + password?: string; + port: number; + tls: TLSOptions; + }; function formatMissingParams(missingParams: string[]) { return `Missing connection parameters: ${missingParams.join(", ")}`; @@ -224,7 +226,7 @@ function formatMissingParams(missingParams: string[]) { function assertRequiredOptions( options: Partial, requiredKeys: (keyof ClientOptions)[], - has_env_access: boolean + has_env_access: boolean, ): asserts options is ClientConfiguration { const missingParams: (keyof ClientOptions)[] = []; for (const key of requiredKeys) { @@ -272,7 +274,7 @@ function parseOptionsArgument(options: string): Record { if (args[x] === "-c") { if (args[x + 1] === undefined) { throw new Error( - `No provided value for "${args[x]}" in options parameter` + `No provided value for "${args[x]}" in options parameter`, ); } @@ -281,7 +283,7 @@ function parseOptionsArgument(options: string): Record { x++; } else { throw new Error( - `Argument "${args[x]}" is not supported in options parameter` + `Argument "${args[x]}" is not supported in options parameter`, ); } } else if (/^--\w/.test(args[x])) { @@ -319,10 +321,9 @@ function parseOptionsFromUri(connection_string: string): ClientOptions { port: uri.port || uri.params.port, // Compatibility with JDBC, not standard // Treat as sslmode=require - sslmode: - uri.params.ssl === "true" - ? "require" - : (uri.params.sslmode as TLSModes), + sslmode: uri.params.ssl === "true" + ? "require" + : (uri.params.sslmode as TLSModes), user: uri.user || uri.params.user, }; } catch (e) { @@ -331,15 +332,13 @@ function parseOptionsFromUri(connection_string: string): ClientOptions { if (!["postgres", "postgresql"].includes(postgres_uri.driver)) { throw new ConnectionParamsError( - `Supplied DSN has invalid driver: ${postgres_uri.driver}.` + `Supplied DSN has invalid driver: ${postgres_uri.driver}.`, ); } // No host by default means socket connection const host_type = postgres_uri.host - ? isAbsolute(postgres_uri.host) - ? "socket" - : "tcp" + ? isAbsolute(postgres_uri.host) ? "socket" : "tcp" : "socket"; const options = postgres_uri.options @@ -367,7 +366,7 @@ function parseOptionsFromUri(connection_string: string): ClientOptions { } default: { throw new ConnectionParamsError( - `Supplied DSN has invalid sslmode '${postgres_uri.sslmode}'` + `Supplied DSN has invalid sslmode '${postgres_uri.sslmode}'`, ); } } @@ -385,29 +384,31 @@ function parseOptionsFromUri(connection_string: string): ClientOptions { }; } -const DEFAULT_OPTIONS: Omit< - ClientConfiguration, - "database" | "user" | "hostname" -> & { host: string; socket: string } = { - applicationName: "deno_postgres", - connection: { - attempts: 1, - interval: (previous_interval) => previous_interval + 500, - }, - host: "127.0.0.1", - socket: "/tmp", - host_type: "socket", - options: {}, - port: 5432, - tls: { - enabled: true, - enforce: false, - caCertificates: [], - }, -}; +const DEFAULT_OPTIONS: + & Omit< + ClientConfiguration, + "database" | "user" | "hostname" + > + & { host: string; socket: string } = { + applicationName: "deno_postgres", + connection: { + attempts: 1, + interval: (previous_interval) => previous_interval + 500, + }, + host: "127.0.0.1", + socket: "/tmp", + host_type: "socket", + options: {}, + port: 5432, + tls: { + enabled: true, + enforce: false, + caCertificates: [], + }, + }; export function createParams( - params: string | ClientOptions = {} + params: string | ClientOptions = {}, ): ClientConfiguration { if (typeof params === "string") { params = parseOptionsFromUri(params); @@ -423,9 +424,9 @@ export function createParams( // 1 and Deno 2 if ( e instanceof - ("NotCapable" in Deno.errors - ? Deno.errors.NotCapable - : Deno.errors.PermissionDenied) + ("NotCapable" in Deno.errors + ? Deno.errors.NotCapable + : Deno.errors.PermissionDenied) ) { has_env_access = false; } else { @@ -436,8 +437,8 @@ export function createParams( const provided_host = params.hostname ?? pgEnv.hostname; // If a host is provided, the default connection type is TCP - const host_type = - params.host_type ?? (provided_host ? "tcp" : DEFAULT_OPTIONS.host_type); + const host_type = params.host_type ?? + (provided_host ? "tcp" : DEFAULT_OPTIONS.host_type); if (!["tcp", "socket"].includes(host_type)) { throw new ConnectionParamsError(`"${host_type}" is not a valid host type`); } @@ -496,13 +497,13 @@ export function createParams( } if (Number.isNaN(port) || port === 0) { throw new ConnectionParamsError( - `"${params.port ?? pgEnv.port}" is not a valid port number` + `"${params.port ?? pgEnv.port}" is not a valid port number`, ); } if (host_type === "socket" && params?.tls) { throw new ConnectionParamsError( - 'No TLS options are allowed when host type is set to "socket"' + 'No TLS options are allowed when host type is set to "socket"', ); } const tls_enabled = !!(params?.tls?.enabled ?? DEFAULT_OPTIONS.tls.enabled); @@ -510,22 +511,21 @@ export function createParams( if (!tls_enabled && tls_enforced) { throw new ConnectionParamsError( - "Can't enforce TLS when client has TLS encryption is disabled" + "Can't enforce TLS when client has TLS encryption is disabled", ); } // TODO // Perhaps username should be taken from the PC user as a default? const connection_options = { - applicationName: - params.applicationName ?? + applicationName: params.applicationName ?? pgEnv.applicationName ?? DEFAULT_OPTIONS.applicationName, connection: { - attempts: - params?.connection?.attempts ?? DEFAULT_OPTIONS.connection.attempts, - interval: - params?.connection?.interval ?? DEFAULT_OPTIONS.connection.interval, + attempts: params?.connection?.attempts ?? + DEFAULT_OPTIONS.connection.attempts, + interval: params?.connection?.interval ?? + DEFAULT_OPTIONS.connection.interval, }, database: params.database ?? pgEnv.database, hostname: host, @@ -545,7 +545,7 @@ export function createParams( assertRequiredOptions( connection_options, ["applicationName", "database", "hostname", "host_type", "port", "user"], - has_env_access + has_env_access, ); return connection_options; diff --git a/query/decode.ts b/query/decode.ts index 6e1ab21d..c0311910 100644 --- a/query/decode.ts +++ b/query/decode.ts @@ -46,7 +46,7 @@ export class Column { public typeOid: number, public columnLength: number, public typeModifier: number, - public format: Format + public format: Format, ) {} } @@ -201,7 +201,7 @@ function decodeText(value: string, typeOid: number) { bold(yellow(`Error decoding type Oid ${typeOid} value`)) + (e instanceof Error ? e.message : e) + "\n" + - bold("Defaulting to null.") + bold("Defaulting to null."), ); // If an error occurred during decoding, return null return null; @@ -211,7 +211,7 @@ function decodeText(value: string, typeOid: number) { export function decode( value: Uint8Array, column: Column, - controls?: ClientControls + controls?: ClientControls, ) { const strValue = decoder.decode(value); @@ -219,8 +219,8 @@ export function decode( if (controls?.decoders) { const oidType = OidTypes[column.typeOid as OidValue]; // check if there is a custom decoder by oid (number) or by type name (string) - const decoderFunc = - controls.decoders?.[column.typeOid] || controls.decoders?.[oidType]; + const decoderFunc = controls.decoders?.[column.typeOid] || + controls.decoders?.[oidType]; if (decoderFunc) { return decoderFunc(strValue, column.typeOid, parseArray); @@ -231,12 +231,12 @@ export function decode( // check if the base type is in the Oid object if (baseOidType in Oid) { // check if there is a custom decoder for the base type by oid (number) or by type name (string) - const decoderFunc = - controls.decoders?.[Oid[baseOidType]] || + const decoderFunc = controls.decoders?.[Oid[baseOidType]] || controls.decoders?.[baseOidType]; if (decoderFunc) { - return parseArray(strValue, (value: string) => - decoderFunc(value, column.typeOid, parseArray) + return parseArray( + strValue, + (value: string) => decoderFunc(value, column.typeOid, parseArray), ); } }