diff --git a/libjxl.spec b/libjxl.spec
index 36f4506..f6fa457 100644
--- a/libjxl.spec
+++ b/libjxl.spec
@@ -120,7 +120,11 @@ mv skcms third_party/
 
 %install
 %cmake_install
-rm -fv %buildroot/%_libdir/*.a
+b="%buildroot"
+rm -fv "$b/%_libdir"/*.a
+%if %{with gtk}
+rm -Rf "$b/%_libdir"/libjxl* "$b/%_bindir" "$b/%_includedir" "$b/%_libdir/pkgconfig"
+%endif
 
 %ldconfig_scriptlets -n %lname
 
@@ -130,6 +134,8 @@ rm -fv %buildroot/%_libdir/*.a
 %postun -n gdk-pixbuf-loader-jxl
 %gdk_pixbuf_loader_postun
 
+%if %{without gtk}
+
 %files -n %lname
 %license LICENSE
 %_libdir/libjxl*.so.*
@@ -144,7 +150,8 @@ rm -fv %buildroot/%_libdir/*.a
 %_libdir/libjxl_threads.so
 %_libdir/pkgconfig/*.pc
 
-%if %{with gtk}
+%else
+
 %files -n gdk-pixbuf-loader-jxl
 %_libdir/gdk-pixbuf-2.0/*/loaders/libpixbufloader-jxl.so
 
@@ -155,6 +162,7 @@ rm -fv %buildroot/%_libdir/*.a
 %dir %_datadir/thumbnailers
 %_datadir/thumbnailers/*.thumbnailer
 %_datadir/mime/packages/*.xml
+
 %endif
 
 %changelog