fix tied_pointers_to_remove (#2439)

src/accelerate/hooks.py

@@ -374,7 +374,7 @@ def post_forward(self, module, output):
             # this dictionary to allow the garbage collector to do its job.
             for value_pointer, device in self.tied_pointers_to_remove:
                 del self.tied_params_map[value_pointer][device]
-            self.tied_pointers_to_remove = None
+            self.tied_pointers_to_remove = set()
 
         if self.io_same_device and self.input_device is not None:
             output = send_to_device(output, self.input_device, skip_keys=self.skip_keys)