diff --git a/no-net-tests.patch b/no-net-tests.patch
deleted file mode 100644
index d89bb79..0000000
--- a/no-net-tests.patch
+++ /dev/null
@@ -1,36 +0,0 @@
----
- setup.cfg                              |    2 ++
- tests/test_sphinx_autodoc_typehints.py |    4 ++++
- 2 files changed, 6 insertions(+)
-
---- a/setup.cfg
-+++ b/setup.cfg
-@@ -43,6 +43,8 @@ max-line-length = 99
- [tool:pytest]
- addopts = -rsx --tb=short
- testpaths = tests
-+markers =
-+    network: marks tests which require network connection
- 
- [egg_info]
- tag_build = 
---- a/tests/test_sphinx_autodoc_typehints.py
-+++ b/tests/test_sphinx_autodoc_typehints.py
-@@ -95,6 +95,8 @@ def test_parse_annotation(annotation, mo
-     assert get_annotation_args(annotation, module, class_name) == args
- 
- 
-+# requires inv fixtures which doesn't work without network connection
-+@pytest.mark.network
- @pytest.mark.parametrize('annotation, expected_result', [
-     (str,                           ':py:class:`str`'),
-     (int,                           ':py:class:`int`'),
-@@ -177,6 +179,8 @@ def test_format_annotation(inv, annotati
-             assert m.group('role') == expected_role
- 
- 
-+# requires inv fixtures which doesn't work without network connection
-+@pytest.mark.network
- @pytest.mark.parametrize('library', [typing, typing_extensions],
-                          ids=['typing', 'typing_extensions'])
- @pytest.mark.parametrize('annotation, params, expected_result', [
diff --git a/python-sphinx-autodoc-typehints-system-object.inv.patch b/python-sphinx-autodoc-typehints-system-object.inv.patch
new file mode 100644
index 0000000..50fbd5e
--- /dev/null
+++ b/python-sphinx-autodoc-typehints-system-object.inv.patch
@@ -0,0 +1,31 @@
+---
+ tests/conftest.py |   13 ++-----------
+ 1 file changed, 2 insertions(+), 11 deletions(-)
+
+--- a/tests/conftest.py
++++ b/tests/conftest.py
+@@ -1,5 +1,4 @@
+ import os
+-import sys
+ import pathlib
+ import shutil
+ 
+@@ -13,16 +12,8 @@ collect_ignore = ['roots']
+ 
+ @pytest.fixture(scope='session')
+ def inv(pytestconfig):
+-    cache_path = 'python{v.major}.{v.minor}/objects.inv'.format(v=sys.version_info)
+-    inv_dict = pytestconfig.cache.get(cache_path, None)
+-    if inv_dict is not None:
+-        return Inventory(inv_dict)
+-
+-    print("Downloading objects.inv")
+-    url = 'https://docs.python.org/{v.major}.{v.minor}/objects.inv'.format(v=sys.version_info)
+-    inv = Inventory(url=url)
+-    pytestconfig.cache.set(cache_path, inv.json_dict())
+-    return inv
++    inv_dict = '/usr/share/doc/packages/python@PYTHON_VERSION@/html/objects.inv'
++    return Inventory(inv_dict)
+ 
+ 
+ @pytest.fixture(autouse=True)
diff --git a/python-sphinx-autodoc-typehints.changes b/python-sphinx-autodoc-typehints.changes
index 52e1880..68d98bc 100644
--- a/python-sphinx-autodoc-typehints.changes
+++ b/python-sphinx-autodoc-typehints.changes
@@ -1,3 +1,11 @@
+-------------------------------------------------------------------
+Sat Apr 24 14:11:17 UTC 2021 - Matej Cepl <mcepl@suse.com>
+
+- Remove no-net-tests.patch as adding the fixed
+  python-sphinx-autodoc-typehints-system-object.inv.patch 
+  makes tests requiring network working
+  (gh#agronholm/sphinx-autodoc-typehints#174).
+
 -------------------------------------------------------------------
 Fri Apr 23 14:24:23 UTC 2021 - Matej Cepl <mcepl@suse.com>
 
diff --git a/python-sphinx-autodoc-typehints.spec b/python-sphinx-autodoc-typehints.spec
index cb17239..f2a30dc 100644
--- a/python-sphinx-autodoc-typehints.spec
+++ b/python-sphinx-autodoc-typehints.spec
@@ -26,9 +26,9 @@ License:        MIT
 Group:          Development/Languages/Python
 URL:            https://github.com/agronholm/sphinx-autodoc-typehints
 Source:         https://files.pythonhosted.org/packages/source/s/sphinx-autodoc-typehints/sphinx-autodoc-typehints-%{version}.tar.gz
-# PATCH-FIX-UPSTREAM no-net-tests.patch gh#agronholm/sphinx-autodoc-typehints#174 mcepl@suse.com
+# PATCH-FIX-UPSTREAM python-sphinx-autodoc-typehints-system-object.inv.patch gh#agronholm/sphinx-autodoc-typehints#174 mcepl@suse.com
 # skip network tests
-Patch0:         no-net-tests.patch
+Patch0:         python-sphinx-autodoc-typehints-system-object.inv.patch
 BuildRequires:  %{python_module setuptools >= 36.2.7}
 BuildRequires:  %{python_module setuptools_scm >= 1.7.0}
 BuildRequires:  fdupes
@@ -55,6 +55,7 @@ and return value types of functions.
 
 %build
 %python_build
+%python_expand sed -i -e 's/@PYTHON_VERSION@/%{$python_version}/' tests/conftest.py
 
 %install
 %python_install
@@ -62,7 +63,7 @@ and return value types of functions.
 
 %check
 # test_sphinx_output -- too depenedent on sphinx version available
-%pytest -k 'not (test_sphinx_output or network)'
+%pytest -k 'not test_sphinx_output'
 
 %files %{python_files}
 %{python_sitelib}/*