diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index 17ceb9b696255..e00dad90df7f4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -488,6 +488,58 @@ public void testDeleteByRangeComparison() throws SQLException { } } + @Test + public void testDropAndAlter() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.test"); + statement.execute( + "CREATE TIMESERIES root.test.g_0.d3.s_10 with datatype=INT32 tags(tag1=v1, tag2=v2)"); + + // time=1 and time=2 are INT32 and deleted by drop column + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(1, 1)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(2, 2)"); + + statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 DROP tag1"); + + // time=3 and time=4 are STRING + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(3, 3)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(4, 4)"); + + statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 ADD TAGS tag1=v1"); + + // time=5 and time=6 are TEXT + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(5, 5)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(6, 6)"); + + try (ResultSet dataSet = + statement.executeQuery("select * from root.test.g_0.d3 order by time")) { + // s1 is dropped but the time should remain + int i = 1; + while (dataSet.next()) { + assertEquals(i, dataSet.getLong(1)); + i++; + } + Assert.assertEquals(6, i - 1); + assertFalse(dataSet.next()); + } + } finally { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE root.test"); + } + } + } + private static void prepareSeries() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 3d61b64e71a64..7abb5d2ca05d1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -33,8 +33,11 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.write.record.Tablet; import org.awaitility.Awaitility; import org.junit.After; import org.junit.AfterClass; @@ -60,6 +63,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Random; @@ -74,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -2268,6 +2273,170 @@ public void testMultiDeviceCompletelyDeleteTable() throws SQLException { cleanData(testNum); } + @Test + public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS delete_by_tag (deviceId STRING TAG, s1 INT32 FIELD)"); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from delete_by_tag order by time"); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + session.executeNonQueryStatement("FLUSH"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + dataSet = session.executeQueryStatement("select * from delete_by_tag order by time"); + + RowRecord rec; + int cnt = 0; + for (int i = 1; i < 5; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + Assert.assertEquals(i, rec.getFields().get(2).getIntV()); + Assert.assertEquals(TSDataType.INT32, rec.getFields().get(2).getDataType()); + cnt++; + } + Assert.assertEquals(4, cnt); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag"); + } + } + } + + @Test + public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); + + // time=1 and time=2 are INT32 and deleted by drop column + Tablet tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + + // time=3 and time=4 are STRING + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 4)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + session.executeNonQueryStatement("ALTER TABLE drop_and_alter ADD COLUMN s1 TEXT"); + + // time=5 and time=6 are TEXT + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 5)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 6)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from drop_and_alter order by time"); + // s1 is dropped but the time should remain + RowRecord rec; + int cnt = 0; + for (int i = 1; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + LOGGER.error( + "time is {}, value is {}, value type is {}", + rec.getFields().get(0).getLongV(), + rec.getFields().get(1), + rec.getFields().get(1).getDataType()); + // assertNull(rec.getFields().get(1).getDataType()); + // Assert.assertEquals(TSDataType.TEXT, rec.getFields().get(1).getDataType()); + cnt++; + } + Assert.assertEquals(6, cnt); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS drop_and_alter"); + } + } + } + private static void prepareDatabase() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 9c02ac94208ae..6ae696ba454d4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -1596,7 +1596,7 @@ private void testOneCastWithRow( } @SuppressWarnings("SameParameterValue") - private Object genValue(TSDataType dataType, int i) { + public static Object genValue(TSDataType dataType, int i) { switch (dataType) { case INT32: return i; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 09c9ee8430415..a4aa20c3de3b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -114,6 +114,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo; +import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; @@ -3030,6 +3031,15 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M Set involvedModificationFiles = new HashSet<>(); List deletedByMods = new ArrayList<>(); List deletedByFiles = new ArrayList<>(); + boolean isDropMeasurementExist = false; + IDPredicate.IDPredicateType idPredicateType = null; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !tableDeletionEntry.getPredicate().getMeasurementNames().isEmpty(); + idPredicateType = tableDeletionEntry.getPredicate().getIdPredicateType(); + } + for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; @@ -3126,7 +3136,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty()) { + if (!deletedByFiles.isEmpty() + && !isDropMeasurementExist + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 7e79e8f580dc5..294999788fa70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -72,6 +72,10 @@ public void setIdPredicate(IDPredicate idPredicate) { this.idPredicate = idPredicate; } + public IDPredicate.IDPredicateType getIdPredicateType() { + return this.idPredicate.type; + } + public String getTableName() { return tableName; }