diff --git a/gobject/gsignal.c b/gobject/gsignal.c
index ebf5b3c1c..41599eb0d 100644
--- a/gobject/gsignal.c
+++ b/gobject/gsignal.c
@@ -3916,6 +3916,7 @@ invalid_closure_notify (gpointer  instance,
   g_assert (handler != NULL);
   g_assert (handler->closure == closure);
 
+  g_hash_table_remove (g_handlers, handler);
   handler->sequential_number = 0;
   handler->block_count = 1;
   handler_unref_R (signal_id, instance, handler);