--- libvisual/lv_object.c
+++ libvisual/lv_object.c
@@ -236,14 +236,12 @@
 {
 	visual_log_return_val_if_fail (object != NULL, -VISUAL_ERROR_OBJECT_NULL);
 
-	object->refcount--;
+	if (object->refcount > 0)
+		object->refcount--;
 
 	/* No reference left, start dtoring of this VisObject */
-	if (object->refcount <= 0) {
-		object->refcount = 0;
-
+	if (object->refcount == 0)
 		return visual_object_destroy (object);
-	}
 	return VISUAL_OK;
 }