diff --git a/junit.changes b/junit.changes
index 3143a3f..b1ac3eb 100644
--- a/junit.changes
+++ b/junit.changes
@@ -1,3 +1,12 @@
+-------------------------------------------------------------------
+Mon Dec 18 15:35:28 UTC 2017 - dimstar@opensuse.org
+
+- Harden and fix the test suite:
+  + org.framework.junit is deprecated since junit 4.0
+  + Ensure we fail check when junit no longer returns what we
+    expect it to (error code checking is useless, since Assert
+    returns !0, like any other failure to start the test).
+
 -------------------------------------------------------------------
 Mon Dec 18 09:34:48 UTC 2017 - fstrba@suse.com
 
diff --git a/junit.spec b/junit.spec
index 5fe6a2a..3db046c 100644
--- a/junit.spec
+++ b/junit.spec
@@ -120,7 +120,7 @@ cp -pr junit%{version}/junit/* %{buildroot}%{_datadir}/%{name}/demo/junit
 %check
 
 cat > test.java <<EOF
-import junit.framework.Assert;
+import org.junit.Assert;
 class test {
 
     public static void main(String[] args) {
@@ -130,7 +130,8 @@ class test {
 }
 EOF
 javac -cp %{buildroot}/%{_javadir}/%{name}.jar test.java
-java -cp %{buildroot}/%{_javadir}/%{name}.jar: test  && exit 42
+java -cp %{buildroot}/%{_javadir}/%{name}.jar: test 2>&1 | \
+   grep 'Exception in thread "main" java.lang.AssertionError: Hello world from junit'
 
 %files
 %defattr(0644,root,root,0755)