Skipping subtrees that are not relevant for deletion for much better performance on exact file deletion

Author: tiavision

GitRewrite/Delete/DeleteObjects.cs

@@ -14,21 +14,28 @@ public static void Run(string repositoryPath, IEnumerable<string> filesToDelete,
         {
             var fileDeleteStrategies = new FileDeletionStrategies(filesToDelete);
             var folderDeleteStrategies = new FolderDeletionStrategies(foldersToDelete);
-            var rewrittenCommits = RemoveObjectsFromTree(repositoryPath, fileDeleteStrategies, folderDeleteStrategies);
+
+            var relevantPathes =
+                fileDeleteStrategies.RelevantPaths.Union(folderDeleteStrategies.RelevantPaths).ToList();
+
+            var rewrittenCommits = RemoveObjectsFromTree(repositoryPath, fileDeleteStrategies, folderDeleteStrategies,
+                relevantPathes);
             if (rewrittenCommits.Any())
                 Refs.Update(repositoryPath, rewrittenCommits);
         }
 
         public static Dictionary<ObjectHash, ObjectHash> RemoveObjectsFromTree(string vcsPath,
-            FileDeletionStrategies filesToDelete, FolderDeletionStrategies foldersToDelete)
+            FileDeletionStrategies filesToDelete, FolderDeletionStrategies foldersToDelete,
+            List<byte[]> relevantPaths)
         {
             var rewrittenCommits = new Dictionary<ObjectHash, ObjectHash>();
             var rewrittenTrees = new ConcurrentDictionary<ObjectHash, ObjectHash>();
 
             foreach (var commit in CommitWalker
                 .CommitsInOrder(vcsPath))
             {
-                var newTreeHash = RemoveObjectFromTree(vcsPath, commit.TreeHash, filesToDelete, foldersToDelete, rewrittenTrees, new byte[0]);
+                var newTreeHash = RemoveObjectFromTree(vcsPath, commit.TreeHash, filesToDelete, foldersToDelete,
+                    rewrittenTrees, new byte[0], relevantPaths);
                 if (newTreeHash != commit.TreeHash)
                 {
                     var newCommit = Commit.GetSerializedCommitWithChangedTreeAndParents(commit, newTreeHash,
@@ -51,17 +58,50 @@ public static Dictionary<ObjectHash, ObjectHash> RemoveObjectsFromTree(string vc
         private static readonly ArrayPool<byte> FilePathPool = ArrayPool<byte>.Shared;
         private static readonly TreeLineByHashComparer TreeLineByHashComparer = new TreeLineByHashComparer();
 
+        private static bool IsPathRelevant(in ReadOnlySpan<byte> currentPath, List<byte[]> relevantPathes)
+        {
+            if (currentPath.Length == 0 || !relevantPathes.Any())
+                return true;
+
+            for (int i = relevantPathes.Count - 1; i >= 0; i--)
+            {
+                var path = relevantPathes[i];
+
+                if (currentPath.Length > path.Length)
+                    continue;
+
+                var isRelevant = true;
+                for (int j = currentPath.Length - 1; j >= 0; j--)
+                {
+                    if (currentPath[j] != path[j])
+                    {
+                        isRelevant = false;
+                        break;
+                    }
+                }
+
+                if (isRelevant) 
+                    return true;
+            }
+
+            return false;
+        }
+
         private static ObjectHash RemoveObjectFromTree(
             string vcsPath,
             ObjectHash treeHash,
             FileDeletionStrategies filesToRemove,
             FolderDeletionStrategies foldersToRemove,
             ConcurrentDictionary<ObjectHash, ObjectHash> rewrittenTrees,
-            ReadOnlySpan<byte> currentPath)
+            in ReadOnlySpan<byte> currentPath,
+            List<byte[]> relevantPathes)
         {
             if (rewrittenTrees.TryGetValue(treeHash, out var rewrittenHash))
                 return rewrittenHash;
 
+            if (!IsPathRelevant(currentPath, relevantPathes))
+                return treeHash;
+
             var tree = GitObjectFactory.ReadTree(vcsPath, treeHash);
             var resultingLines = new List<Tree.TreeLine>();
             foreach (var line in tree.Lines)
@@ -89,7 +129,8 @@ private static ObjectHash RemoveObjectFromTree(
                                 filesToRemove,
                                 foldersToRemove,
                                 rewrittenTrees,
-                                path);
+                                path,
+                                relevantPathes);
 
                             FilePathPool.Return(rentedPathBytes);
 

GitRewrite/Delete/FileDeletionStrategies.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Text;
 
 namespace GitRewrite.Delete
 {
@@ -15,14 +16,23 @@ public FileDeletionStrategies(IEnumerable<string> filePatterns)
                 if (objectPattern[0] == '*')
                     _strategies.Add(new FileEndsWithDeletionStrategy(objectPattern));
                 else if (objectPattern[0] == '/')
+                {
                     _strategies.Add(new FileExactDeletionStrategy(objectPattern));
+
+                    var indexToCut = objectPattern.LastIndexOf('/');
+                    var pathString = objectPattern.Substring(0, indexToCut);
+                    var bytes = Encoding.UTF8.GetBytes(pathString);
+                    RelevantPaths.Add(bytes);
+                }
                 else if (objectPattern[objectPattern.Length - 1] == '*')
                     _strategies.Add(new FileStartsWithDeletionStrategy(objectPattern));
                 else 
                     _strategies.Add(new FileSimpleDeleteStrategy(objectPattern));
             }
         }
 
+        public List<byte[]> RelevantPaths { get; } = new List<byte[]>();
+
         public bool DeleteObject(in ReadOnlySpan<byte> fileName, ReadOnlySpan<byte> currentPath)
         {
             foreach (var strategy in _strategies)

GitRewrite/Delete/FolderDeletionStrategies.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Text;
 
 namespace GitRewrite.Delete
 {
@@ -15,14 +16,20 @@ public FolderDeletionStrategies(IEnumerable<string> patterns)
                 if (objectPattern[0] == '*')
                     _strategies.Add(new FolderEndsWithDeletionStrategy(objectPattern));
                 else if (objectPattern[0] == '/')
-                    _strategies.Add(new FolderExactDeletionStrategy(objectPattern));
+                {
+                    var bytes = Encoding.UTF8.GetBytes(objectPattern);
+                    _strategies.Add(new FolderExactDeletionStrategy(bytes));
+                    RelevantPaths.Add(bytes);
+                }
                 else if (objectPattern[objectPattern.Length - 1] == '*')
                     _strategies.Add(new FolderStartsWithDeletionStrategy(objectPattern));
                 else 
                     _strategies.Add(new FolderSimpleDeleteStrategy(objectPattern));
             }
         }
 
+        public List<byte[]> RelevantPaths { get; } = new List<byte[]>();
+
         public bool DeleteObject(ReadOnlySpan<byte> currentPath)
         {
             foreach (var strategy in _strategies)

GitRewrite/Delete/FolderExactDeletionStrategy.cs

@@ -8,7 +8,7 @@ class FolderExactDeletionStrategy : IFolderDeletionStrategy
     {
         private readonly Memory<byte> _folderName;
 
-        public FolderExactDeletionStrategy(string fileName) => _folderName = Encoding.UTF8.GetBytes(fileName);
+        public FolderExactDeletionStrategy(byte[] fileName) => _folderName = fileName;
 
         public bool DeleteObject(in ReadOnlySpan<byte> currentPath) => _folderName.Span.SpanEquals(currentPath);
     }

README.md

@@ -20,6 +20,7 @@ Deleting should be pretty fast, in my tests it even outperformed the bfg repo cl
 Simple wildcards for the beginning and the end of the filename are supported, like *.zip.
 It also lets you specify the complete path to the file instead of only a file name. 
 For this the path has to be prefixed by a forward slash and the path seperator also is a forward slash: /path/to/file.txt
+Specifying only files with complete path will result in much better performance as not all subtrees have to be checked.
 
 ### Deleting folders
 ```