feat!: add support for not tracking single empty files.

Empty files are equivalent to having no content, which also means
such files have no identity to speak off. This definitely helps
with false positives of `.gitignore` for instance, which can be empty
to tell Git to track a directory.

On top of that, Git has a heuristic to do rename tracking of small files
by similarity as the similarity may be off of files just have a couple of
lines to speak about.

Note that empty files that are renamed as part of a whole directory will still
be tracked as renames.

Author: Byron

gix-diff/src/lib.rs

@@ -37,6 +37,11 @@ pub struct Rewrites {
     /// If the limit would not be enough to test the entire set of combinations, the algorithm will trade in precision and not
     /// run the fuzzy version of identity tests at all. That way results are never partial.
     pub limit: usize,
+
+    /// If `true`, empty blobs will be tracked. If `false`, they do not participate in rename tracking.
+    ///
+    /// Leaving this off usually leads to better results as empty files don't have a unique-enough identity.
+    pub track_empty: bool,
 }
 
 /// Contains a [Tracker](rewrites::Tracker) to detect rewrites.

gix-diff/src/rewrites/mod.rs

@@ -70,6 +70,7 @@ impl Default for Rewrites {
             copies: None,
             percentage: Some(0.5),
             limit: 1000,
+            track_empty: false,
         }
     }
 }

gix-diff/src/rewrites/tracker.rs

@@ -286,12 +286,10 @@ impl<T: Change> Tracker<T> {
                 CopySource::FromSetOfModifiedFiles => {}
                 CopySource::FromSetOfModifiedFilesAndAllSources => {
                     push_source_tree(&mut |change, location| {
-                        assert!(
-                            self.try_push_change(change, location).is_none(),
-                            "we must accept every change"
-                        );
-                        // make sure these aren't viable to be emitted anymore.
-                        self.items.last_mut().expect("just pushed").emitted = true;
+                        if self.try_push_change(change, location).is_none() {
+                            // make sure these aren't viable to be emitted anymore.
+                            self.items.last_mut().expect("just pushed").emitted = true;
+                        }
                     })
                     .map_err(|err| emit::Error::GetItemsForExhaustiveCopyDetection(Box::new(err)))?;
                     self.items.sort_by(by_id_and_location);
@@ -404,7 +402,19 @@ impl<T: Change> Tracker<T> {
         while let Some((mut dest_idx, dest)) = self.items[dest_ofs..].iter().enumerate().find_map(|(idx, item)| {
             (!item.emitted
                 && matches!(item.change.kind(), ChangeKind::Addition)
-                && filter.map_or(true, |f| f(&item.change)))
+                && filter.map_or_else(
+                    || {
+                        self.rewrites.track_empty
+                            // We always want to keep track of entries that are involved of a directory rename.
+                            // Note that this may still match them up arbitrarily if empty, but empty is empty.
+                            || matches!(item.change.relation(), Some(Relation::ChildOfParent(_)))
+                            || {
+                                let id = item.change.id();
+                                id != gix_hash::ObjectId::empty_blob(id.kind())
+                            }
+                    },
+                    |f| f(&item.change),
+                ))
             .then_some((idx, item))
         }) {
             dest_idx += dest_ofs;

gix-diff/tests/diff/rewrites/tracker.rs

@@ -27,6 +27,7 @@ fn rename_by_id() -> crate::Result {
             copies: None,
             percentage: None,
             limit,
+            track_empty: false,
         };
         let mut track = util::new_tracker(rewrites);
         assert!(
@@ -80,6 +81,7 @@ fn copy_by_similarity_reports_limit_if_encountered() -> crate::Result {
         }),
         percentage: None,
         limit: 1,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rewrites);
     let odb = util::add_retained_blobs(
@@ -132,6 +134,7 @@ fn copy_by_id() -> crate::Result {
             }),
             percentage: None,
             limit,
+            track_empty: false,
         };
         let mut track = util::new_tracker(rewrites);
         let odb = util::add_retained_blobs(
@@ -206,6 +209,7 @@ fn copy_by_id_search_in_all_sources() -> crate::Result {
             }),
             percentage: None,
             limit,
+            track_empty: false,
         };
         let mut track = util::new_tracker(rewrites);
         let odb = util::add_retained_blobs(
@@ -284,6 +288,7 @@ fn copy_by_50_percent_similarity() -> crate::Result {
         }),
         percentage: None,
         limit: 0,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rewrites);
     let odb = util::add_retained_blobs(
@@ -363,6 +368,7 @@ fn copy_by_id_in_additions_only() -> crate::Result {
         }),
         percentage: None,
         limit: 0,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rewrites);
     let odb = util::add_retained_blobs(
@@ -413,6 +419,7 @@ fn rename_by_similarity_reports_limit_if_encountered() -> crate::Result {
         copies: None,
         percentage: Some(0.5),
         limit: 1,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rewrites);
     let odb = util::add_retained_blobs(
@@ -458,6 +465,7 @@ fn rename_by_50_percent_similarity() -> crate::Result {
         copies: None,
         percentage: Some(0.5),
         limit: 0,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rewrites);
     let odb = util::add_retained_blobs(
@@ -547,6 +555,7 @@ fn directory_renames_by_id_can_fail_gracefully() -> crate::Result {
         copies: None,
         percentage: Some(0.5),
         limit: 0,
+        track_empty: false,
     };
     let mut track = util::new_tracker(rename_by_similarity);
     let tree_dst_id = 1;
@@ -638,6 +647,7 @@ fn simple_directory_rename_by_id() -> crate::Result {
         copies: None,
         percentage: None,
         limit: 0,
+        track_empty: false,
     };
     let mut track = util::new_tracker(renames_by_identity);
     let tree_dst_id = 1;

gix-diff/tests/diff/tree_with_rewrites.rs

@@ -192,12 +192,13 @@ fn changes_against_modified_tree_with_filename_tracking() -> crate::Result {
 
 #[test]
 fn renames_by_identity() -> crate::Result {
-    for (from, to, expected, assert_msg) in [
+    for (from, to, expected, assert_msg, track_empty) in [
         (
             "c3-modification",
             "r1-identity",
             vec![BStr::new("a"), "dir/a-moved".into()],
             "one rename and nothing else",
+            false,
         ),
         (
             "c4 - add identical files",
@@ -211,24 +212,35 @@ fn renames_by_identity() -> crate::Result {
                 "z".into(),
             ],
             "multiple possible sources decide by ordering everything lexicographically",
+            true,
+        ),
+        (
+            "c4 - add identical files",
+            "r2-ambiguous",
+            vec![],
+            "nothing is tracked with `track_empty = false`",
+            false,
         ),
         (
             "c5 - add links",
             "r4-symlinks",
             vec!["link-1".into(), "renamed-link-1".into()],
             "symlinks are only tracked by identity",
+            false,
         ),
         (
             "r1-identity",
             "c4 - add identical files",
             vec![],
             "not having any renames is OK as well",
+            false,
         ),
         (
             "tc1-identity",
             "tc1-identity",
             vec![],
             "copy tracking is off by default",
+            false,
         ),
     ] {
         for percentage in [None, Some(0.5)] {
@@ -239,6 +251,7 @@ fn renames_by_identity() -> crate::Result {
                     location: Some(Location::Path),
                     rewrites: Some(Rewrites {
                         percentage,
+                        track_empty,
                         ..Default::default()
                     }),
                 },
@@ -704,6 +717,7 @@ fn copies_in_entire_tree_by_similarity_with_limit() -> crate::Result {
                     ..Default::default()
                 }),
                 limit: 2, // similarity checks can't be made that way
+                track_empty: false,
                 ..Default::default()
             }),
         },
@@ -833,6 +847,7 @@ fn realistic_renames_by_identity() -> crate::Result {
             rewrites: Some(Rewrites {
                 copies: Some(Copies::default()),
                 limit: 1,
+                track_empty: true,
                 ..Default::default()
             }),
         },
@@ -1324,6 +1339,7 @@ fn realistic_renames_by_identity_3() -> crate::Result {
             rewrites: Some(Rewrites {
                 copies: Some(Copies::default()),
                 limit: 1,
+                track_empty: true,
                 ..Default::default()
             }),
         },
@@ -1402,6 +1418,7 @@ fn realistic_renames_2() -> crate::Result {
             rewrites: Some(Rewrites {
                 copies: Some(Copies::default()),
                 limit: 1,
+                track_empty: false,
                 ..Default::default()
             }),
         },
@@ -1665,6 +1682,7 @@ fn realistic_renames_3_without_identity() -> crate::Result {
                 copies: None,
                 percentage: None,
                 limit: 0,
+                track_empty: false,
             }),
         },
     )?;