--- configure.ac
+++ configure.ac
@@ -107,7 +107,8 @@
   AC_CHECK_LIB(X11, XOpenDisplay,
    [ac_cv_my_have_x11="yes"
     if test -n "${x_includes}"; then X_CFLAGS="-I${x_includes}"; fi
-    if test -n "${x_libraries}"; then X_LIBS="-lX11 -L${x_libraries}"; fi
+    if test -n "${x_libraries}"; then X_LIBS="-lX11"; fi
+    if test "${x_libraries}" != "/usr"; then X_LIBS="$X_LIBS -L${x_libraries}"; fi
     AC_DEFINE(USE_X11, 1, Define to activate the X11 backend driver)
     CPPFLAGS="${CPPFLAGS} ${X_CFLAGS}"
     CACA_LIBS="${CACA_LIBS} ${X_LIBS}"