diff --git a/gobject/gobject.c b/gobject/gobject.c
index 8c8e94e1e..9425bff61 100644
--- a/gobject/gobject.c
+++ b/gobject/gobject.c
@@ -4315,6 +4315,42 @@ g_object_add_toggle_ref (GObject *object,
   object_bit_unlock (object, OPTIONAL_BIT_LOCK_TOGGLE_REFS);
 }
 
+static gpointer
+toggle_refs_unref_cb (gpointer *data,
+                      GDestroyNotify *destroy_notify,
+                      gpointer user_data)
+{
+  ToggleRefCallbackData *trdata = user_data;
+  ToggleRefStack *tstack = *data;
+  gboolean found_one = FALSE;
+  guint i;
+
+  if (!tstack)
+    goto out;
+
+  for (i = 0; i < tstack->n_toggle_refs; i++)
+    {
+      if (tstack->toggle_refs[i].notify == trdata->tuple.notify &&
+          (tstack->toggle_refs[i].data == trdata->tuple.data || trdata->tuple.data == NULL))
+        {
+          found_one = TRUE;
+          tstack->n_toggle_refs -= 1;
+          if (tstack->n_toggle_refs == 0)
+            {
+              g_datalist_unset_flags (&trdata->object->qdata, OBJECT_HAS_TOGGLE_REF_FLAG);
+              g_free (tstack);
+              *data = NULL;
+            }
+          else if (i != tstack->n_toggle_refs)
+            tstack->toggle_refs[i] = tstack->toggle_refs[tstack->n_toggle_refs];
+          break;
+        }
+    }
+
+out:
+  return GINT_TO_POINTER (found_one);
+}
+
 /**
  * g_object_remove_toggle_ref: (skip)
  * @object: a #GObject
@@ -4335,46 +4371,35 @@ g_object_add_toggle_ref (GObject *object,
  * Since: 2.8
  */
 void
-g_object_remove_toggle_ref (GObject       *object,
-			    GToggleNotify  notify,
-			    gpointer       data)
+g_object_remove_toggle_ref (GObject *object,
+                            GToggleNotify notify,
+                            gpointer data)
 {
-  ToggleRefStack *tstack;
-  gboolean found_one = FALSE;
+  gpointer result;
 
   g_return_if_fail (G_IS_OBJECT (object));
   g_return_if_fail (notify != NULL);
 
   object_bit_lock (object, OPTIONAL_BIT_LOCK_TOGGLE_REFS);
-  tstack = g_datalist_id_get_data (&object->qdata, quark_toggle_refs);
-  if (tstack)
-    {
-      guint i;
 
-      for (i = 0; i < tstack->n_toggle_refs; i++)
-	if (tstack->toggle_refs[i].notify == notify &&
-	    (tstack->toggle_refs[i].data == data || data == NULL))
-	  {
-	    found_one = TRUE;
-	    tstack->n_toggle_refs -= 1;
-	    if (i != tstack->n_toggle_refs)
-	      tstack->toggle_refs[i] = tstack->toggle_refs[tstack->n_toggle_refs];
+  result = _g_datalist_id_update_atomic (&object->qdata,
+                                         quark_toggle_refs,
+                                         toggle_refs_unref_cb,
+                                         &((ToggleRefCallbackData){
+                                             .object = object,
+                                             .tuple.notify = notify,
+                                             .tuple.data = data,
+                                         }));
 
-	    if (tstack->n_toggle_refs == 0)
-	      {
-	        g_datalist_unset_flags (&object->qdata, OBJECT_HAS_TOGGLE_REF_FLAG);
-	        g_datalist_id_set_data_full (&object->qdata, quark_toggle_refs, NULL, NULL);
-	      }
-
-	    break;
-	  }
-    }
   object_bit_unlock (object, OPTIONAL_BIT_LOCK_TOGGLE_REFS);
 
-  if (found_one)
-    g_object_unref (object);
-  else
-    g_critical ("%s: couldn't find toggle ref %p(%p)", G_STRFUNC, notify, data);
+  if (!GPOINTER_TO_INT (result))
+    {
+      g_critical ("%s: couldn't find toggle ref %p(%p)", G_STRFUNC, notify, data);
+      return;
+    }
+
+  g_object_unref (object);
 }
 
 /* Internal implementation of g_object_ref() which doesn't call out to user code.