From 3913c6d6753ea5fd1c4fcc18602ddeffcd44555c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:14:42 +0100 Subject: [PATCH 1/3] bdd: use a SQL statement template for index checks Also introduces substring matching. --- tests/bdd/environment.py | 1 + tests/bdd/flex/lua-index-definitions.feature | 121 ++++++++++--------- tests/bdd/steps/steps_db.py | 35 +++++- 3 files changed, 92 insertions(+), 65 deletions(-) diff --git a/tests/bdd/environment.py b/tests/bdd/environment.py index 350b5e1b6..16919ec85 100644 --- a/tests/bdd/environment.py +++ b/tests/bdd/environment.py @@ -122,6 +122,7 @@ def before_scenario(context, scenario): context.geometry_factory = GeometryFactory() context.osm2pgsql_replication.ReplicationServer = ReplicationServerMock() context.urlrequest_responses = {} + context.sql_statements = {} def _mock_urlopen(request): if not request.full_url in context.urlrequest_responses: diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature index eeb92202a..051253fa5 100644 --- a/tests/bdd/flex/lua-index-definitions.feature +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -1,5 +1,16 @@ Feature: Index definitions in Lua file + Background: + Given the SQL statement mytable_indexes + """ + SELECT indexdef, indisprimary as is_primary + FROM pg_catalog.pg_index, pg_catalog.pg_indexes + WHERE schemaname = 'public' + AND tablename = 'mytable' + AND indrelid = tablename::regclass + AND indexrelid = indexname::regclass + """ + Scenario: Indexes field in table definition must be an array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -36,9 +47,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING gist (geom) | Scenario: Empty indexes field in table definition gets you no index Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -56,8 +67,8 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' - | schemaname | tablename | + Then statement mytable_indexes returns exactly + | indexdef | Scenario: Explicitly setting an index column works Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -77,9 +88,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) | Scenario: Explicitly setting multiple indexes Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -101,15 +112,11 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | - And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' - | schemaname | tablename | - | public | mytable | - And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name, tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING btree (name) | + | USING gist (geom) | + | USING btree (name, tags) | Scenario: Method can not be missing Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -244,9 +251,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (lower(name))%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (lower(name)) | @needs-pg-index-includes Scenario: Include field must be a string or array @@ -315,9 +322,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) INCLUDE (tags) | @needs-pg-index-includes Scenario: Include field works with array @@ -338,9 +345,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) INCLUDE (tags) | Scenario: Tablespace needs a string Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -383,9 +390,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) | Scenario: Unique needs a boolean Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -428,9 +435,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%UNIQUE%' - | schemaname | tablename | - | public | mytable | + Then statement mytable_indexes returns + | indexdef@fullmatch | + | .*UNIQUE.*USING btree \(name\).* | Scenario: Where condition needs a string Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -473,9 +480,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes contains - | schemaname | tablename | indexdef@fullmatch | - | public | mytable | .*USING btree \(name\).*WHERE \(name = lower\(name\)\).* | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (name) WHERE (name = lower(name)) | Scenario: Don't create id index if the configuration doesn't mention it @@ -493,9 +500,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes doesn't contain - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING gist (geom) | Scenario: Don't create id index if the configuration doesn't says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -512,9 +519,9 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes doesn't contain - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns exactly + | indexdef@substr | + | USING gist (geom) | Scenario: Always create id index if the configuration says so Given the input file 'liechtenstein-2013-08-03.osm.pbf' @@ -531,16 +538,16 @@ Feature: Index definitions in Lua file }) """ When running osm2pgsql flex - Then table pg_catalog.pg_indexes contains - | schemaname | tablename | indexname@fullmatch | - | public | mytable | .*node_id.* | + Then statement mytable_indexes returns + | indexdef@substr | + | USING btree (node_id) | Scenario: Create a unique id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style """ local t = osm2pgsql.define_table({ - name = 'foo', + name = 'mytable', ids = { type = 'node', id_column = 'node_id', create_index = 'unique' }, columns = {} }) @@ -550,20 +557,17 @@ Feature: Index definitions in Lua file end """ When running osm2pgsql flex - Then table foo has 1562 rows - Then table pg_catalog.pg_indexes contains - | tablename | indexdef@fullmatch | - | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary - | count | - | 0 | + Then table mytable has 1562 rows + Then statement mytable_indexes returns + | indexdef@fullmatch | is_primary | + | CREATE UNIQUE INDEX .* USING .*\(node_id\).* | False | Scenario: Create a primary key id index when requested Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style """ local t = osm2pgsql.define_table({ - name = 'foo', + name = 'mytable', ids = { type = 'node', id_column = 'node_id', create_index = 'primary_key' }, columns = {} }) @@ -573,10 +577,7 @@ Feature: Index definitions in Lua file end """ When running osm2pgsql flex - Then table foo has 1562 rows - Then table pg_catalog.pg_indexes contains - | tablename | indexdef@fullmatch | - | foo | CREATE UNIQUE INDEX .* USING .*\(node_id\) | - And SELECT count(*) FROM pg_catalog.pg_index WHERE indrelid = 'foo'::regclass and indisprimary - | count | - | 1 | + Then table mytable has 1562 rows + Then statement mytable_indexes returns + | indexdef@fullmatch | is_primary | + | CREATE UNIQUE INDEX .* USING .*\(node_id\) | True | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index f83d085af..c91d1e059 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -92,19 +92,30 @@ def db_check_table_absence(context, table): assert not row in actuals, f"Row unexpectedly found: {row}. Full content:\n{actuals}" -@then("(?PSELECT .*)") -def db_check_sql_statement(context, query): +@given("the SQL statement (?P.+)") +def db_define_sql_statement(context, sql): + context.sql_statements[sql] = context.text + +@then("statement (?P.+) returns(?P exactly)?") +def db_check_sql_statement(context, stmt, exact): with context.db.cursor() as cur: - cur.execute(query) + assert stmt in context.sql_statements + cur.execute(context.sql_statements[stmt]) actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur) linenr = 1 for row in context.table.rows: - assert any(r == row for r in actuals),\ - f"{linenr}. entry not found in table. Full content:\n{actuals}" + try: + actuals.remove(row) + except ValueError: + assert False,\ + f"{linenr}. entry not found in result. Full response:\n{actuals}" linenr += 1 + assert not exact or not actuals,\ + f"Unexpected lines in result:\n{actuals}" + ### Helper functions and classes @@ -152,6 +163,8 @@ def __init__(self, row, headings, factory): self.data.append(DBValueGeometry(value, props, factory)) elif props == 'fullmatch': self.data.append(DBValueRegex(value)) + elif props == 'substr': + self.data.append(DBValueSubString(value)) else: self.data.append(str(value)) @@ -310,3 +323,15 @@ def __eq__(self, other): def __repr__(self): return repr(self.value) + + +class DBValueSubString: + + def __init__(self, value): + self.value = str(value) + + def __eq__(self, other): + return str(other) in self.value + + def __repr__(self): + return repr(self.value) From 7d69ce84c75e39aa6bf26adfd45adf1d1c2807db Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:16:10 +0100 Subject: [PATCH 2/3] bdd: remove needs include index check Postgrsql versions without the feature are no longer supported. --- tests/bdd/environment.py | 7 ------- tests/bdd/flex/lua-index-definitions.feature | 4 ---- 2 files changed, 11 deletions(-) diff --git a/tests/bdd/environment.py b/tests/bdd/environment.py index 16919ec85..16afeeae4 100644 --- a/tests/bdd/environment.py +++ b/tests/bdd/environment.py @@ -154,10 +154,3 @@ def test_db(context, **kwargs): def working_directory(context, **kwargs): with tempfile.TemporaryDirectory() as tmpdir: yield Path(tmpdir) - - -def before_tag(context, tag): - if tag == 'needs-pg-index-includes': - if context.config.userdata['PG_VERSION'] < 110000: - context.scenario.skip("No index includes in PostgreSQL < 11") - diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature index 051253fa5..1400a7809 100644 --- a/tests/bdd/flex/lua-index-definitions.feature +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -255,7 +255,6 @@ Feature: Index definitions in Lua file | indexdef@substr | | USING btree (lower(name)) | - @needs-pg-index-includes Scenario: Include field must be a string or array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -279,7 +278,6 @@ Feature: Index definitions in Lua file The 'include' field in an index definition must contain a string or an array. """ - @needs-pg-index-includes Scenario: Include field must contain a valid column Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -303,7 +301,6 @@ Feature: Index definitions in Lua file Unknown column 'foo' in table 'mytable'. """ - @needs-pg-index-includes Scenario: Include field works with string Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style @@ -326,7 +323,6 @@ Feature: Index definitions in Lua file | indexdef@substr | | USING btree (name) INCLUDE (tags) | - @needs-pg-index-includes Scenario: Include field works with array Given the input file 'liechtenstein-2013-08-03.osm.pbf' And the lua style From 69739e8dccce612ce0b9a8834aa839ed78ab130e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 29 Nov 2025 11:52:05 +0100 Subject: [PATCH 3/3] bdd: replace remaining SELECT steps with SQL templates --- tests/bdd/flex/delete-callbacks.feature | 34 ++++++++++++++--------- tests/bdd/regression/multipolygon.feature | 28 +++++++++++++++---- tests/bdd/steps/steps_db.py | 7 +++-- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/tests/bdd/flex/delete-callbacks.feature b/tests/bdd/flex/delete-callbacks.feature index 2d2a3ec02..773c521db 100644 --- a/tests/bdd/flex/delete-callbacks.feature +++ b/tests/bdd/flex/delete-callbacks.feature @@ -27,21 +27,29 @@ Feature: Test for delete callbacks Given the input file '008-ch.osc.gz' + Given the SQL statement grouped_counts + """ + SELECT osm_type, + count(*), + sum(extra) AS sum_extra, + sum(osm_id) AS sum_osm_id + FROM change + GROUP BY osm_type + """ + Scenario: Delete callbacks are called When running osm2pgsql flex with parameters | --slim | -a | - - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | - | N | 16773 | 16779 | - | W | 4 | 9 | - | R | 1 | 3 | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | + | N | 16773 | 16779 | + | W | 4 | 9 | + | R | 1 | 3 | When running osm2pgsql flex with parameters | --slim | -a | - - Then SELECT osm_type, count(*), sum(osm_id) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_osm_id | | N | 16773 | 37856781001834 | | W | 4 | 350933407 | | R | 1 | 2871571 | @@ -72,8 +80,8 @@ Feature: Test for delete callbacks """ When running osm2pgsql flex with parameters | --slim | -a | - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | | N | 16773 | 16773 | | W | 4 | 4 | | R | 1 | 1 | @@ -105,8 +113,8 @@ Feature: Test for delete callbacks """ When running osm2pgsql flex with parameters | --slim | -a | - Then SELECT osm_type, count(*), sum(extra) FROM change GROUP BY osm_type - | osm_type | count | sum | + Then statement grouped_counts returns exactly + | osm_type | count | sum_extra | | N | 16773 | 16773| | W | 4 | 4| | R | 1 | 1| diff --git a/tests/bdd/regression/multipolygon.feature b/tests/bdd/regression/multipolygon.feature index ce4b48ef5..4f5d7af91 100644 --- a/tests/bdd/regression/multipolygon.feature +++ b/tests/bdd/regression/multipolygon.feature @@ -3,6 +3,16 @@ Feature: Import and update of multipolygon areas Background: Given the input file 'test_multipolygon.osm' + Given the SQL statement grouped_polygons + """ + SELECT osm_id, + count(*) AS count, + round(sum(ST_Area(way))) AS area, + round(sum(way_area::numeric)) AS way_area + FROM planet_osm_polygon + GROUP BY osm_id + """ + Scenario Outline: Import and update slim Given lua tagtransform When running osm2pgsql pgsql with parameters @@ -35,7 +45,7 @@ Feature: Import and update of multipolygon areas | osm_id | landuse | name | ST_NumInteriorRing(way) | | -3 | residential | Name_rel11 | 2 | - Then SELECT osm_id, round(sum(ST_Area(way))), round(sum(way_area::numeric)) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | area | way_area | | -13 | 17581 | 17581 | | -7 | 16169 | 16169 | @@ -43,7 +53,7 @@ Feature: Import and update of multipolygon areas | -39 | 10377 | 10378 | | -40 | 12397 | 12397 | - Then SELECT osm_id, count(*) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | count | | -25 | 1 | | 113 | 1 | @@ -63,7 +73,15 @@ Feature: Import and update of multipolygon areas | osm_id | "natural" | | -33 | water | - Then SELECT osm_id, CASE WHEN '' = '-G' THEN min(ST_NumGeometries(way)) ELSE count(*) END FROM planet_osm_polygon GROUP BY osm_id + Given the SQL statement geometries_polygon + """ + SELECT osm_id, + CASE WHEN '' = '-G' THEN min(ST_NumGeometries(way)) + ELSE count(*) END AS count + FROM planet_osm_polygon + GROUP BY osm_id + """ + Then statement geometries_polygon returns | osm_id | count | | -13 | 2 | | -7 | 2 | @@ -99,7 +117,7 @@ Feature: Import and update of multipolygon areas | osm_id | landuse | name | ST_NumInteriorRing(way) | | -3 | residential | Name_rel11 | 2 | - Then SELECT osm_id, round(sum(ST_Area(way))), round(sum(way_area::numeric)) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | area | way_area | | -13 | 17581 | 17581 | | -7 | 16169 | 16169 | @@ -107,7 +125,7 @@ Feature: Import and update of multipolygon areas | -39 | 10377 | 10378 | | -40 | 12397 | 12397 | - Then SELECT osm_id, count(*) FROM planet_osm_polygon GROUP BY osm_id + Then statement grouped_polygons returns | osm_id | count | | 113 | 1 | | 118 | 1 | diff --git a/tests/bdd/steps/steps_db.py b/tests/bdd/steps/steps_db.py index c91d1e059..681b0635d 100644 --- a/tests/bdd/steps/steps_db.py +++ b/tests/bdd/steps/steps_db.py @@ -98,9 +98,12 @@ def db_define_sql_statement(context, sql): @then("statement (?P.+) returns(?P exactly)?") def db_check_sql_statement(context, stmt, exact): + assert stmt in context.sql_statements + rows = sql.SQL(', '.join(h.rsplit('@')[0] for h in context.table.headings)) + with context.db.cursor() as cur: - assert stmt in context.sql_statements - cur.execute(context.sql_statements[stmt]) + cur.execute(sql.SQL("SELECT {} FROM ({}) _sql_statement") + .format(rows, sql.SQL(context.sql_statements[stmt]))) actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur)