diff --git a/gobject/gobject.c b/gobject/gobject.c
index df908984b..2217e5c4b 100644
--- a/gobject/gobject.c
+++ b/gobject/gobject.c
@@ -5059,16 +5059,22 @@ g_weak_ref_set (GWeakRef *weak_ref,
       /* Remove the weak ref from the old object */
       if (old_object != NULL)
         {
+          gboolean in_weak_refs_notify;
+
           weak_locations = g_datalist_id_get_data (&old_object->qdata, quark_weak_locations);
+          in_weak_refs_notify = g_datalist_id_get_data (&old_object->qdata, quark_weak_refs) == NULL;
           /* for it to point to an object, the object must have had it added once */
-          g_assert (weak_locations != NULL);
+          g_assert (weak_locations != NULL || in_weak_refs_notify);
 
-          *weak_locations = g_slist_remove (*weak_locations, weak_ref);
-
-          if (!*weak_locations)
+          if (weak_locations != NULL)
             {
-              weak_locations_free_unlocked (weak_locations);
-              g_datalist_id_remove_no_notify (&old_object->qdata, quark_weak_locations);
+              *weak_locations = g_slist_remove (*weak_locations, weak_ref);
+
+              if (!*weak_locations)
+                {
+                  weak_locations_free_unlocked (weak_locations);
+                  g_datalist_id_remove_no_notify (&old_object->qdata, quark_weak_locations);
+                }
             }
         }