From ee39ee10811d43567acc33bd24977d3ae1e22ca9 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 15 Dec 2025 14:17:01 +0800 Subject: [PATCH 1/6] deving --- .../storageengine/dataregion/DataRegion.java | 14 +++++++++-- .../rescon/disk/TierManager.java | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 09c9ee843041..639b7e4c214f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2751,8 +2751,8 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException { if (deleted) { return; } - TableDeviceSchemaCache.getInstance() - .invalidateLastCache(getDatabaseName(), modEntries.get(0).getTableName()); + String tableName = modEntries.get(0).getTableName(); + TableDeviceSchemaCache.getInstance().invalidateLastCache(getDatabaseName(), tableName); List walListeners = logDeletionInWAL(node); for (WALFlushListener walFlushListener : walListeners) { @@ -2762,6 +2762,10 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException { } } + if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) { + deleteObjectFiles(tableName, modEntries); + } + List> sealedTsFileResourceLists = new ArrayList<>(modEntries.size()); for (TableDeletionEntry modEntry : modEntries) { List sealedTsFileResource = new ArrayList<>(); @@ -2930,6 +2934,12 @@ private List logDeletionInWAL( return walFlushListeners; } + private void deleteObjectFiles(String tableName, List modEntries) { + for (TableDeletionEntry modEntry : modEntries) { + + } + } + /** * For IoTConsensus sync. See github pull * request for details. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index a5fa8b54e7b1..e173bb0abafb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.rescon.disk.strategy.RandomOnDiskUsableSpaceStrategy; import org.apache.iotdb.metrics.utils.FileStoreUtils; +import com.google.common.io.BaseEncoding; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FSUtils; @@ -36,6 +37,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; import java.nio.file.Path; import java.util.ArrayList; @@ -272,6 +274,28 @@ public Optional getAbsoluteObjectFilePath(String filePath, boolean needTem return Optional.empty(); } + public boolean checkObjectPathExist(String regionIdStr, String... path) { + StringBuilder objectPath = new StringBuilder(); + objectPath.append(regionIdStr); + for (String str : path) { + objectPath + .append(File.separator) + .append( + config.getRestrictObjectLimit() + ? str + : BaseEncoding.base32() + .omitPadding() + .encode(str.getBytes(StandardCharsets.UTF_8))); + } + for (String objectDir : objectDirs) { + File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString()); + if (objectFilePath.exists()) { + return true; + } + } + return false; + } + public int getTiersNum() { return seqTiers.size(); } From e2383e13ad0edac8e8a0f110e67a7e78a02720af Mon Sep 17 00:00:00 2001 From: HTHou Date: Tue, 16 Dec 2025 09:53:55 +0800 Subject: [PATCH 2/6] dev --- .../storageengine/dataregion/DataRegion.java | 65 +++++++++++++++++-- .../rescon/disk/TierManager.java | 10 +-- .../iotdb/db/utils/ObjectTypeUtils.java | 21 ++++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 639b7e4c214f..27a290b60fd8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -158,6 +158,7 @@ import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.db.utils.ObjectWriter; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; @@ -165,11 +166,13 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.io.BaseEncoding; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.external.commons.lang3.tuple.Triple; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; @@ -186,6 +189,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -2762,8 +2766,10 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException { } } - if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) { - deleteObjectFiles(tableName, modEntries); + List matchedObjectDirs = + TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName); + if (!matchedObjectDirs.isEmpty()) { + deleteObjectFiles(matchedObjectDirs, modEntries); } List> sealedTsFileResourceLists = new ArrayList<>(modEntries.size()); @@ -2934,9 +2940,58 @@ private List logDeletionInWAL( return walFlushListeners; } - private void deleteObjectFiles(String tableName, List modEntries) { - for (TableDeletionEntry modEntry : modEntries) { - + private void deleteObjectFiles(List matchedObjectDirs, List modEntries) + throws IOException { + for (File matchedObjectDir : matchedObjectDirs) { + try (Stream paths = Files.walk(matchedObjectDir.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin"); + }) + .forEach( + path -> { + Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); + String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2]; + for (int i = 0; i < ideviceIdSegments.length; i++) { + ideviceIdSegments[i] = + config.getRestrictObjectLimit() + ? relativePath.getName(i).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode(relativePath.getName(i).toString()), + StandardCharsets.UTF_8); + } + IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments); + String measurementId = + config.getRestrictObjectLimit() + ? relativePath.getName(relativePath.getNameCount() - 2).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode( + relativePath + .getName(relativePath.getNameCount() - 2) + .toString()), + StandardCharsets.UTF_8); + String fileName = path.getFileName().toString(); + long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); + logger.info( + "timestamp {}, measurementId {}, ideviceId {}", + timestamp, + measurementId, + iDeviceID); + for (TableDeletionEntry modEntry : modEntries) { + if (modEntry.affects(iDeviceID, timestamp, timestamp) + && modEntry.affects(measurementId)) { + ObjectTypeUtils.deleteObjectPath(path.toFile()); + } + } + }); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index e173bb0abafb..a673e44dff9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -274,7 +274,8 @@ public Optional getAbsoluteObjectFilePath(String filePath, boolean needTem return Optional.empty(); } - public boolean checkObjectPathExist(String regionIdStr, String... path) { + public List getAllMatchedObjectDirs(String regionIdStr, String... path) { + List matchedDirs = new ArrayList<>(); StringBuilder objectPath = new StringBuilder(); objectPath.append(regionIdStr); for (String str : path) { @@ -288,12 +289,13 @@ public boolean checkObjectPathExist(String regionIdStr, String... path) { .encode(str.getBytes(StandardCharsets.UTF_8))); } for (String objectDir : objectDirs) { - File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString()); + File objectFilePath = + FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString()); if (objectFilePath.exists()) { - return true; + matchedDirs.add(objectFilePath); } } - return false; + return matchedDirs; } public int getTiersNum() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java index ec1fd592617a..2a03facbf964 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -294,6 +294,27 @@ public static void deleteObjectPathFromBinary(Binary binary) { } } + public static void deleteObjectPath(File file) { + File tmpFile = new File(file.getPath() + ".tmp"); + File bakFile = new File(file.getPath() + ".back"); + for (int i = 0; i < 2; i++) { + if (file.exists()) { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(file.length()); + } + try { + deleteObjectFile(file); + deleteObjectFile(tmpFile); + deleteObjectFile(bakFile); + } catch (IOException e) { + logger.error("Failed to remove object file {}", file.getAbsolutePath(), e); + } + } + if (file.getParentFile().exists()) { + + } + } + private static void deleteObjectFile(File file) throws IOException { if (file.exists()) { logger.info("Remove object file {}, size is {}(byte)", file.getAbsolutePath(), file.length()); From fdbbd52a12cea171dffe424784da81c9cd8b7f24 Mon Sep 17 00:00:00 2001 From: HTHou Date: Tue, 16 Dec 2025 16:40:28 +0800 Subject: [PATCH 3/6] remove empty object dir --- .../db/storageengine/rescon/disk/TierManager.java | 11 +++++++++++ .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 13 ++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index a673e44dff9c..3aff31a96314 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -174,6 +175,16 @@ public synchronized void initFolders() { } catch (DiskSpaceInsufficientException e) { logger.error("All disks of tier {} are full.", tierLevel, e); } + // try to remove empty objectDirs + for (String dir : objectDirs) { + File dirFile = FSFactoryProducer.getFSFactory().getFile(dir); + if (dirFile.isDirectory() && Objects.requireNonNull(dirFile.list()).length == 0) { + try { + Files.delete(dirFile.toPath()); + } catch (IOException ignore) { + } + } + } } tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java index 2a03facbf964..15964ffaddd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -55,6 +55,7 @@ import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; public class ObjectTypeUtils { @@ -310,8 +311,18 @@ public static void deleteObjectPath(File file) { logger.error("Failed to remove object file {}", file.getAbsolutePath(), e); } } - if (file.getParentFile().exists()) { + deleteEmptyParentDir(file); + } + private static void deleteEmptyParentDir(File file) { + File dir = file.getParentFile(); + if (dir.isDirectory() && Objects.requireNonNull(dir.list()).length == 0) { + try { + Files.deleteIfExists(dir.toPath()); + deleteEmptyParentDir(dir); + } catch (IOException e) { + logger.error("Failed to remove empty object dir {}", dir.getAbsolutePath(), e); + } } } From 6bbf453f62f3804aca11fac272b9413a55d1ca4d Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 18 Dec 2025 12:19:32 +0800 Subject: [PATCH 4/6] optimize drop table --- .../storageengine/dataregion/DataRegion.java | 109 ++++++++++-------- .../modification/DeletionPredicate.java | 4 + .../modification/TableDeletionEntry.java | 9 ++ .../rescon/disk/TierManager.java | 11 ++ 4 files changed, 82 insertions(+), 51 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 27a290b60fd8..8de74e066abc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2766,10 +2766,21 @@ public void deleteByTable(RelationalDeleteDataNode node) throws IOException { } } - List matchedObjectDirs = + List objectTableDirs = TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName); - if (!matchedObjectDirs.isEmpty()) { - deleteObjectFiles(matchedObjectDirs, modEntries); + if (!objectTableDirs.isEmpty()) { + boolean droppingTable = false; + for (TableDeletionEntry entry : modEntries) { + if (entry.isDroppingTable()) { + for (File objectTableDir : objectTableDirs) { + droppingTable = true; + FileUtils.deleteQuietly(objectTableDir); + } + } + } + if (!droppingTable) { + deleteObjectFiles(objectTableDirs, modEntries); + } } List> sealedTsFileResourceLists = new ArrayList<>(modEntries.size()); @@ -2943,54 +2954,50 @@ private List logDeletionInWAL( private void deleteObjectFiles(List matchedObjectDirs, List modEntries) throws IOException { for (File matchedObjectDir : matchedObjectDirs) { - try (Stream paths = Files.walk(matchedObjectDir.toPath())) { - paths - .filter(Files::isRegularFile) - .filter( - path -> { - String name = path.getFileName().toString(); - return name.endsWith(".bin"); - }) - .forEach( - path -> { - Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); - String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2]; - for (int i = 0; i < ideviceIdSegments.length; i++) { - ideviceIdSegments[i] = - config.getRestrictObjectLimit() - ? relativePath.getName(i).toString() - : new String( - BaseEncoding.base32() - .omitPadding() - .decode(relativePath.getName(i).toString()), - StandardCharsets.UTF_8); - } - IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments); - String measurementId = - config.getRestrictObjectLimit() - ? relativePath.getName(relativePath.getNameCount() - 2).toString() - : new String( - BaseEncoding.base32() - .omitPadding() - .decode( - relativePath - .getName(relativePath.getNameCount() - 2) - .toString()), - StandardCharsets.UTF_8); - String fileName = path.getFileName().toString(); - long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); - logger.info( - "timestamp {}, measurementId {}, ideviceId {}", - timestamp, - measurementId, - iDeviceID); - for (TableDeletionEntry modEntry : modEntries) { - if (modEntry.affects(iDeviceID, timestamp, timestamp) - && modEntry.affects(measurementId)) { - ObjectTypeUtils.deleteObjectPath(path.toFile()); - } - } - }); + try (Stream paths = + Files.find( + matchedObjectDir.toPath(), + Integer.MAX_VALUE, + (path, attrs) -> + attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) { + paths.forEach( + path -> { + Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); + String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2]; + for (int i = 0; i < ideviceIdSegments.length; i++) { + ideviceIdSegments[i] = + config.getRestrictObjectLimit() + ? relativePath.getName(i).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode(relativePath.getName(i).toString()), + StandardCharsets.UTF_8); + } + IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments); + String measurementId = + config.getRestrictObjectLimit() + ? relativePath.getName(relativePath.getNameCount() - 2).toString() + : new String( + BaseEncoding.base32() + .omitPadding() + .decode( + relativePath.getName(relativePath.getNameCount() - 2).toString()), + StandardCharsets.UTF_8); + String fileName = path.getFileName().toString(); + long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); + logger.info( + "timestamp {}, measurementId {}, ideviceId {}", + timestamp, + measurementId, + iDeviceID); + for (TableDeletionEntry modEntry : modEntries) { + if (modEntry.affects(iDeviceID, timestamp, timestamp) + && modEntry.affects(measurementId)) { + ObjectTypeUtils.deleteObjectPath(path.toFile()); + } + } + }); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 7e79e8f580dc..0a22da2a90a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -72,6 +72,10 @@ public void setIdPredicate(IDPredicate idPredicate) { this.idPredicate = idPredicate; } + public IDPredicate getIdPredicate() { + return idPredicate; + } + public String getTableName() { return tableName; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java index 858b6645b2f9..61ec2e0d6785 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.IDPredicateType; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.IDeviceID; @@ -136,6 +137,14 @@ public String getTableName() { return predicate.getTableName(); } + public boolean isDroppingTable() { + IDPredicate idPredicate = predicate.getIdPredicate(); + return idPredicate.type == IDPredicateType.NOP + && predicate.getMeasurementNames().isEmpty() + && timeRange.getMin() == Long.MIN_VALUE + && timeRange.getMax() == Long.MAX_VALUE; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index 3aff31a96314..f83bdc36b4b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -287,6 +287,17 @@ public Optional getAbsoluteObjectFilePath(String filePath, boolean needTem public List getAllMatchedObjectDirs(String regionIdStr, String... path) { List matchedDirs = new ArrayList<>(); + boolean hasObjectDir = false; + for (String objectDir : objectDirs) { + File objectDirPath = FSFactoryProducer.getFSFactory().getFile(objectDir); + if (objectDirPath.exists()) { + hasObjectDir = true; + break; + } + } + if (!hasObjectDir) { + return matchedDirs; + } StringBuilder objectPath = new StringBuilder(); objectPath.append(regionIdStr); for (String str : path) { From b5a55494ac8be02197f5159c5af32bacb626130e Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 22 Dec 2025 17:50:36 +0800 Subject: [PATCH 5/6] add some IT --- .../it/session/IoTDBObjectDeleteIT.java | 273 ++++++++++++++++++ .../storageengine/dataregion/DataRegion.java | 6 +- 2 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java new file mode 100644 index 000000000000..8bfec578671e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import com.google.common.io.BaseEncoding; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertNull; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectDeleteIT { + + @BeforeClass + public static void classSetUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + } + } + + @After + public void tearDown() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1"); + } + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void deleteObjectTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject))); + session.insert(tablet); + tablet.reset(); + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues()); + } + session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table"); + } + } + + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin"); + if (objectFile.exists() && objectFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertFalse(success); + } + + @Test + public void deleteObjectSegmentsTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + byte[] objectBytes = Files.readAllBytes(Paths.get(testObject)); + List objectSegments = new ArrayList<>(); + for (int i = 0; i < objectBytes.length; i += 512) { + objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length))); + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + for (int i = 0; i < objectSegments.size() - 1; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i)); + session.insert(tablet); + tablet.reset(); + } + session.executeNonQueryStatement("DELETE FROM object_table where time = 1"); + + try (SessionDataSet dataSet = + session.executeQueryStatement("select file from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + assertNull(iterator.getString(1)); + } + } + } + + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectTmpFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin.tmp"); + if (objectTmpFile.exists() && objectTmpFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertFalse(success); + } + + protected String convertPathString(String path) { + return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 8de74e066abc..e3ebd991cf0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2959,7 +2959,9 @@ private void deleteObjectFiles(List matchedObjectDirs, List - attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) { + attrs.isRegularFile() + && (path.getFileName().toString().endsWith(".bin") + || path.getFileName().toString().endsWith(".tmp")))) { paths.forEach( path -> { Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); @@ -2985,7 +2987,7 @@ private void deleteObjectFiles(List matchedObjectDirs, List Date: Tue, 23 Dec 2025 12:08:33 +0800 Subject: [PATCH 6/6] Fix error --- .../apache/iotdb/db/storageengine/dataregion/DataRegion.java | 4 ++-- .../iotdb/db/storageengine/rescon/disk/TierManager.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index e3ebd991cf0a..14a56e24acbf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2968,7 +2968,7 @@ private void deleteObjectFiles(List matchedObjectDirs, List matchedObjectDirs, List getAllMatchedObjectDirs(String regionIdStr, String... path) { objectPath .append(File.separator) .append( - config.getRestrictObjectLimit() + CommonDescriptor.getInstance().getConfig().isRestrictObjectLimit() ? str : BaseEncoding.base32() .omitPadding()