diff --git a/fix-faildiff-test-for-diffutils-3.4.patch b/fix-faildiff-test-for-diffutils-3.4.patch
deleted file mode 100644
index 6fae038..0000000
--- a/fix-faildiff-test-for-diffutils-3.4.patch
+++ /dev/null
@@ -1,26 +0,0 @@
-From: Jean Delvare <jdelvare@suse.de>
-Subject: faildiff.test: Fix for diffutils 3.4 and later
-
-Since diffutils version 3.4, diff no longer returns an error code for
-binary files. Remove this test from faildiff.test.
-
-Signed-off-by: Jean Delvare <jdelvare@suse.de>
----
- test/faildiff.test |    9 ++-------
- 1 file changed, 2 insertions(+), 7 deletions(-)
-
---- quilt.orig/test/faildiff.test	2016-09-09 10:16:05.709530299 +0200
-+++ quilt/test/faildiff.test	2016-09-09 10:39:16.914180640 +0200
-@@ -29,10 +29,5 @@ What happens on binary files?
- 	$ printf "\\003\\000\\001" > test.bin
- 	$ quilt diff -pab --no-index
- 	>~ (Files|Binary files) a/test\.bin and b/test\.bin differ
--	$ echo %{?}
--	> 1
--
--	$ quilt refresh
--	> Diff failed on file 'test.bin', aborting
--	$ echo %{?}
--	> 1
-+	# Since diffutils version 3.4, diff no longer returns an error
-+	# code for binary files, so we don't test it here.
diff --git a/quilt.changes b/quilt.changes
index 7b0999d..5d95acb 100644
--- a/quilt.changes
+++ b/quilt.changes
@@ -1,3 +1,13 @@
+-------------------------------------------------------------------
+Fri Sep 16 09:24:13 UTC 2016 - jdelvare@suse.de
+
+- reject-binary-files-in-patches.patch: Since diffutils version
+  3.4, diff no longer returns an error code for binary files. Parse
+  the first line of the output to detect this case and raise our
+  own error.
+- Remove fix-faildiff-test-for-diffutils-3.4.patch, use the
+  upstream fix instead.
+
 -------------------------------------------------------------------
 Fri Sep  9 11:05:04 CEST 2016 - jdelvare@suse.de
 
diff --git a/quilt.spec b/quilt.spec
index 0287029..acb1113 100644
--- a/quilt.spec
+++ b/quilt.spec
@@ -50,7 +50,7 @@ Patch8:         translation-fixes-04.patch
 Patch9:         edit-01-test-file-creation.patch
 Patch10:        edit-02-call-remove-not-revert.patch
 Patch11:        edit-03-fix-corner-case.patch
-Patch12:        fix-faildiff-test-for-diffutils-3.4.patch
+Patch12:        reject-binary-files-in-patches.patch
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildArch:      noarch
 %if 0%{?suse_version}
diff --git a/reject-binary-files-in-patches.patch b/reject-binary-files-in-patches.patch
new file mode 100644
index 0000000..eabaa6f
--- /dev/null
+++ b/reject-binary-files-in-patches.patch
@@ -0,0 +1,36 @@
+From: Jean Delvare <jdelvare@suse.de>
+Date: Thu, 15 Sep 2016 14:05:29 +0200
+Subject: Reject binary files in patches
+Git-commit: 20f06212baced666027131555ad4c834d8e4b232
+Patch-mainline: yes
+
+Since diffutils version 3.4, diff no longer returns an error code for
+binary files. Parse the first line of the output to detect this case
+and raise our own error.
+
+diff --git a/quilt/scripts/patchfns.in b/quilt/scripts/patchfns.in
+index 521e92d18b53..a496f3b07390 100644
+--- a/quilt/scripts/patchfns.in
++++ b/quilt/scripts/patchfns.in
+@@ -741,6 +741,12 @@ diff_file()
+ 	     "$old_file" "$new_file" \
+ 	| if read line
+ 	then
++		if [[ "$line" =~ ^Binary\ files\ .*\ differ$ ]]
++		then
++			echo "$line"
++			return 1
++		fi
++
+ 		if [ -z "$QUILT_NO_DIFF_INDEX" ]
+ 		then
+ 			echo "Index: $index"
+@@ -751,7 +757,7 @@ diff_file()
+ 	fi
+ 
+ 	# Test the return value of diff, and propagate the error retcode if any
+-	if [ ${PIPESTATUS[0]} == 2 ]
++	if [ ${PIPESTATUS[0]} == 2 -o ${PIPESTATUS[1]} == 1 ]
+ 	then
+ 		return 1
+ 	fi