FIX fsdp_auto_wrap_policy for some models (#2167)

Some transformers models and custom models would throw an error when
used with PEFT's fsdp_auto_wrap_policy. This is problematatic because
Trainer applies the policy automatically when PEFT and FSDP are
detected. Now there is no error.

Author: csqaiub

src/peft/utils/other.py

@@ -525,6 +525,8 @@ def fsdp_auto_wrap_policy(model):
     ).split(",")
     transformer_cls_to_wrap = {PrefixEncoder, PromptEncoder, PromptEmbedding}
     for layer_class in transformer_cls_names_to_wrap:
+        if len(layer_class) == 0:
+            continue
         transformer_cls = get_module_class_from_name(model, layer_class)
         if transformer_cls is None:
             raise Exception("Could not find the transformer layer class to wrap in the model.")

tests/test_gpu_examples.py

@@ -3774,6 +3774,11 @@ def test_bnb_4bit_wrap_fsdp(self):
         # check that this does not raise:
         FSDP(model, auto_wrap_policy=fsdp_auto_wrap_policy(model), use_orig_params=False, sync_module_states=True)
 
+    def test_fsdp_auto_wrap_policy_does_not_raise_on_custom_model(self):
+        # See #2167
+        # Avoid raising on custom models since Trainer uses fsdp_auto_wrap_policy automatically for PEFT + FSDP
+        fsdp_auto_wrap_policy(SimpleModel())  # does not raise
+
 
 class TestBOFT:
     """