From 0509f0b0f1e880e7651e2a33cf5b70ef1930a3ff Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:22:33 +0200 Subject: [PATCH] Fix IPv6 scope (bsc#1108557) Fix ipaddress imports Remove unused import Fix ipaddress import Fix unicode imports in compat Override standard IPv6Address class Check version via object Isolate Py2 and Py3 mode Add logging Add debugging to the ip_address method (py2 and py3) Remove multiple returns and add check for address syntax Remove unnecessary variable for import detection Remove duplicated code Remove unnecessary operator Remove multiple returns Use ternary operator instead Remove duplicated code Move docstrings to their native places Add real exception message Add logging to the ip_interface Add scope on str Lintfix: mute not called constructors Add extra detection for hexadecimal packed bytes on Python2. This cannot be detected with type comparison, because bytes == str and at the same time bytes != str if compatibility is not around Fix py2 case where the same class cannot initialise itself on Python2 via super. Simplify checking clause Do not use introspection for method swap Fix wrong type swap Add Py3.4 old implementation's fix Lintfix Lintfix refactor: remove duplicate returns as not needed Revert method remapping with pylint updates Remove unnecessary manipulation with IPv6 scope outside of the IPv6Address object instance Lintfix: W0611 Reverse skipping tests: if no ipaddress --- salt/_compat.py | 287 +++++++++++++++++++++++------ salt/cloud/clouds/saltify.py | 5 +- salt/cloud/clouds/vagrant.py | 9 +- salt/ext/win_inet_pton.py | 2 +- salt/minion.py | 5 +- salt/modules/ipset.py | 5 +- salt/modules/network.py | 5 +- salt/modules/vagrant.py | 6 +- salt/utils/dns.py | 11 +- salt/utils/minions.py | 5 +- tests/unit/grains/test_core.py | 5 +- tests/unit/modules/test_network.py | 15 +- 12 files changed, 245 insertions(+), 115 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 9b10646ace..0576210afc 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -2,18 +2,21 @@ ''' Salt compatibility code ''' -# pylint: disable=import-error,unused-import,invalid-name +# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233 # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function import sys import types +import logging # Import 3rd-party libs -from salt.ext.six import binary_type, string_types, text_type +from salt.exceptions import SaltException +from salt.ext.six import binary_type, string_types, text_type, integer_types from salt.ext.six.moves import cStringIO, StringIO -HAS_XML = True +log = logging.getLogger(__name__) + try: # Python >2.5 import xml.etree.cElementTree as ElementTree @@ -31,11 +34,10 @@ except Exception: import elementtree.ElementTree as ElementTree except Exception: ElementTree = None - HAS_XML = False # True if we are running on Python 3. -PY3 = sys.version_info[0] == 3 +PY3 = sys.version_info.major == 3 if PY3: @@ -45,13 +47,12 @@ else: import exceptions -if HAS_XML: +if ElementTree is not None: if not hasattr(ElementTree, 'ParseError'): class ParseError(Exception): ''' older versions of ElementTree do not have ParseError ''' - pass ElementTree.ParseError = ParseError @@ -61,9 +62,7 @@ def text_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``binary_type``, return ``s.decode(encoding, errors)``, otherwise return ``s`` ''' - if isinstance(s, binary_type): - return s.decode(encoding, errors) - return s + return s.decode(encoding, errors) if isinstance(s, binary_type) else s def bytes_(s, encoding='latin-1', errors='strict'): @@ -71,57 +70,37 @@ def bytes_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``s`` ''' - if isinstance(s, text_type): - return s.encode(encoding, errors) - return s + return s.encode(encoding, errors) if isinstance(s, text_type) else s -if PY3: - def ascii_native_(s): - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s, 'ascii', 'strict') -else: - def ascii_native_(s): - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s) +def ascii_native_(s): + ''' + Python 3: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` -ascii_native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s)`` + ''' + if isinstance(s, text_type): + s = s.encode('ascii') -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s)`` -''' + return str(s, 'ascii', 'strict') if PY3 else s -if PY3: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s``, otherwise return ``str(s, encoding, errors)`` - ''' - if isinstance(s, text_type): - return s - return str(s, encoding, errors) -else: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s.encode(encoding, errors)``, otherwise return ``str(s)`` - ''' - if isinstance(s, text_type): - return s.encode(encoding, errors) - return str(s) +def native_(s, encoding='latin-1', errors='strict'): + ''' + Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise + return ``str(s, encoding, errors)`` -native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise -return ``str(s, encoding, errors)`` + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode(encoding, errors)``, otherwise return ``str(s)`` + ''' + if PY3: + out = s if isinstance(s, text_type) else str(s, encoding, errors) + else: + out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s) -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode(encoding, errors)``, otherwise return ``str(s)`` -''' + return out def string_io(data=None): # cStringIO can't handle unicode @@ -133,7 +112,199 @@ def string_io(data=None): # cStringIO can't handle unicode except (UnicodeEncodeError, TypeError): return StringIO(data) -if PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress + +try: + if PY3: + import ipaddress + else: + import salt.ext.ipaddress as ipaddress +except ImportError: + ipaddress = None + + +class IPv6AddressScoped(ipaddress.IPv6Address): + ''' + Represent and manipulate single IPv6 Addresses. + Scope-aware version + ''' + def __init__(self, address): + ''' + Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'. + + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560) + or, more generally + IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::') + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + + :param address: + ''' + # pylint: disable-all + if not hasattr(self, '_is_packed_binary'): + # This method (below) won't be around for some Python 3 versions + # and we need check this differently anyway + self._is_packed_binary = lambda p: isinstance(p, bytes) + # pylint: enable-all + + if isinstance(address, string_types) and '%' in address: + buff = address.split('%') + if len(buff) != 2: + raise SaltException('Invalid IPv6 address: "{}"'.format(address)) + address, self.__scope = buff + else: + self.__scope = None + + if sys.version_info.major == 2: + ipaddress._BaseAddress.__init__(self, address) + ipaddress._BaseV6.__init__(self, address) + else: + # Python 3.4 fix. Versions higher are simply not affected + # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 + self._version = 6 + self._max_prefixlen = ipaddress.IPV6LENGTH + + # Efficient constructor from integer. + if isinstance(address, integer_types): + self._check_int_address(address) + self._ip = address + elif self._is_packed_binary(address): + self._check_packed_address(address, 16) + self._ip = ipaddress._int_from_bytes(address, 'big') + else: + address = str(address) + if '/' in address: + raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address)) + self._ip = self._ip_int_from_string(address) + + def _is_packed_binary(self, data): + ''' + Check if data is hexadecimal packed + + :param data: + :return: + ''' + packed = False + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) + except ValueError: + pass + + return packed + + @property + def scope(self): + ''' + Return scope of IPv6 address. + + :return: + ''' + return self.__scope + + def __str__(self): + return text_type(self._string_from_ip_int(self._ip) + + ('%' + self.scope if self.scope is not None else '')) + + +class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped): + ''' + Update + ''' + def __init__(self, address): + if isinstance(address, (bytes, int)): + IPv6AddressScoped.__init__(self, address) + self.network = ipaddress.IPv6Network(self._ip) + self._prefixlen = self._max_prefixlen + return + + addr = ipaddress._split_optional_netmask(address) + IPv6AddressScoped.__init__(self, addr[0]) + self.network = ipaddress.IPv6Network(address, strict=False) + self.netmask = self.network.netmask + self._prefixlen = self.network._prefixlen + self.hostmask = self.network.hostmask + + +def ip_address(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Address or IPv6Address object. + + Raises: + ValueError: if the *address* passed isn't either a v4 or a v6 + address + + """ + try: + return ipaddress.IPv4Address(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv4 address: %s', address) + log.debug(err) + + try: + return IPv6AddressScoped(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv6 address: %s', address) + log.debug(err) + + if isinstance(address, bytes): + raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. ' + 'Did you pass in a bytes (str in Python 2) instead ' + 'of a unicode object?'.format(repr(address))) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address))) + + +def ip_interface(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Interface or IPv6Interface object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. + + Notes: + The IPv?Interface classes describe an Address on a particular + Network, so they're basically a combination of both the Address + and Network classes. + + """ + try: + return ipaddress.IPv4Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv4 interface for address %s', address) + log.debug(err) + + try: + return ipaddress.IPv6Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv6 interface for address %s', address) + log.debug(err) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 interface'.format(address)) + + +if ipaddress: + ipaddress.IPv6Address = IPv6AddressScoped + if sys.version_info.major == 2: + ipaddress.IPv6Interface = IPv6InterfaceScoped + ipaddress.ip_address = ip_address + ipaddress.ip_interface = ip_interface diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index c9cc281b42..e0e56349a0 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -27,10 +27,7 @@ import salt.utils.cloud import salt.config as config import salt.client import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress from salt.exceptions import SaltCloudException, SaltCloudSystemExit diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index a24170c78a..0fe410eb91 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -25,13 +25,8 @@ import tempfile import salt.utils import salt.config as config import salt.client -import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress -from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ - SaltInvocationError +from salt._compat import ipaddress +from salt.exceptions import SaltCloudException, SaltCloudSystemExit, SaltInvocationError # Get logging started log = logging.getLogger(__name__) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 1204bede10..89aba14ce9 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os -import ipaddress +from salt._compat import ipaddress import salt.ext.six as six diff --git a/salt/minion.py b/salt/minion.py index 17e11c0ebe..9c05a646ea 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -26,10 +26,7 @@ from binascii import crc32 # Import Salt Libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress from salt.ext.six.moves import range from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py index 7047e84c29..1a0fa0044d 100644 --- a/salt/modules/ipset.py +++ b/salt/modules/ipset.py @@ -13,10 +13,7 @@ from salt.ext.six.moves import map, range import salt.utils.path # Import third-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress # Set up logging log = logging.getLogger(__name__) diff --git a/salt/modules/network.py b/salt/modules/network.py index 92893572a6..60f586f6bc 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -26,10 +26,7 @@ from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py index 0592dede55..0f518c2602 100644 --- a/salt/modules/vagrant.py +++ b/salt/modules/vagrant.py @@ -39,11 +39,7 @@ import salt.utils.path import salt.utils.stringutils from salt.exceptions import CommandExecutionError, SaltInvocationError import salt.ext.six as six - -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/utils/dns.py b/salt/utils/dns.py index db08bcb7ac..40011016fd 100644 --- a/salt/utils/dns.py +++ b/salt/utils/dns.py @@ -1029,18 +1029,13 @@ def parse_resolv(src='/etc/resolv.conf'): try: (directive, arg) = (line[0].lower(), line[1:]) # Drop everything after # or ; (comments) - arg = list(itertools.takewhile( - lambda x: x[0] not in ('#', ';'), arg)) - + arg = list(itertools.takewhile(lambda x: x[0] not in ('#', ';'), arg)) if directive == 'nameserver': - # Split the scope (interface) if it is present - addr, scope = arg[0].split('%', 1) if '%' in arg[0] else (arg[0], '') + addr = arg[0] try: ip_addr = ipaddress.ip_address(addr) version = ip_addr.version - # Rejoin scope after address validation - if scope: - ip_addr = '%'.join((str(ip_addr), scope)) + ip_addr = str(ip_addr) if ip_addr not in nameservers: nameservers.append(ip_addr) if version == 4 and ip_addr not in ip4_nameservers: diff --git a/salt/utils/minions.py b/salt/utils/minions.py index bb0cbaa589..f282464eee 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -26,10 +26,7 @@ import salt.cache from salt.ext import six # Import 3rd-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress HAS_RANGE = False try: import seco.range # pylint: disable=import-error diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index dd7d5b06f8..e973428add 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -32,10 +32,7 @@ import salt.grains.core as core # Import 3rd-party libs from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index 865f15f3e3..50fa629276 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -20,20 +20,11 @@ from tests.support.mock import ( ) # Import Salt Libs -from salt.ext import six import salt.utils.network import salt.utils.path import salt.modules.network as network from salt.exceptions import CommandExecutionError -if six.PY2: - import salt.ext.ipaddress as ipaddress - HAS_IPADDRESS = True -else: - try: - import ipaddress - HAS_IPADDRESS = True - except ImportError: - HAS_IPADDRESS = False +from salt._compat import ipaddress @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -278,7 +269,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(network.connect('host', 'port'), {'comment': ret, 'result': True}) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_private(self): ''' Test for Check if the given IP address is a private address @@ -290,7 +281,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): return_value=True): self.assertTrue(network.is_private('::1')) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_loopback(self): ''' Test for Check if the given IP address is a loopback address -- 2.19.0