for diffgrad, clone gradient before saving as previous gradient (#57)

Author: lucidrains

torch_optimizer/diffgrad.py

@@ -114,7 +114,7 @@ def step(self, closure: OptLossClosure = None) -> OptFloat:
                 # compute diffgrad coefficient (dfc)
                 diff = torch.abs(previous_grad - grad)
                 dfc = torch.div(1.0, (1.0 + torch.exp(-diff)))
-                state['previous_grad'] = grad
+                state['previous_grad'] = grad.clone()
 
                 # update momentum with dfc
                 exp_avg1 = exp_avg * dfc