From 65038f31ef9ce5d342a4d4e35f80e6a139aa225b216fe6f870931a34191e8ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Tue, 8 Sep 2020 13:26:08 +0000 Subject: [PATCH] osc copypac from project:systemsmanagement:saltstack:testing package:salt revision:352 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=173 --- _lastrevision | 2 +- ...-care-of-failed-skipped-and-unreacha.patch | 1824 ++ backport-virt-patches-from-3001-256.patch | 7112 ++++++ ...eamclosederror-traceback-but-only-lo.patch | 27 + opensuse-3000.2-virt-backports-236-257.patch | 21134 ++++++++++++++++ salt.changes | 33 + salt.spec | 12 + 7 files changed, 30143 insertions(+), 1 deletion(-) create mode 100644 ansiblegate-take-care-of-failed-skipped-and-unreacha.patch create mode 100644 backport-virt-patches-from-3001-256.patch create mode 100644 do-not-raise-streamclosederror-traceback-but-only-lo.patch create mode 100644 opensuse-3000.2-virt-backports-236-257.patch diff --git a/_lastrevision b/_lastrevision index 4e97f64..2ef0b09 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -3c85bd3a365dd15aae8f08c2cb95f16db987fe7b \ No newline at end of file +b193a24466b79150ec137a7e4cc144362b588ef0 \ No newline at end of file diff --git a/ansiblegate-take-care-of-failed-skipped-and-unreacha.patch b/ansiblegate-take-care-of-failed-skipped-and-unreacha.patch new file mode 100644 index 0000000..d5fd856 --- /dev/null +++ b/ansiblegate-take-care-of-failed-skipped-and-unreacha.patch @@ -0,0 +1,1824 @@ +From e1b4dda1eed90b4c6495b7a1fb047052f2cc5d5c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 13 Aug 2020 13:49:16 +0100 +Subject: [PATCH] ansiblegate: take care of failed, skipped and + unreachable tasks (bsc#1173911) + +Add 'retcode' from ansible-playbook execution to the returned data (bsc#1173909) + +Always add retcode to ansible.playbooks output + +Adjust ansible.playbooks output comment properly + +Add new unit test for ansible.playbooks + +Add unit tests for ansible.playbooks state +--- + salt/modules/ansiblegate.py | 10 +- + salt/states/ansiblegate.py | 51 +- + .../unit/files/playbooks/failed_example.json | 748 ++++++++++++++++ + .../unit/files/playbooks/success_example.json | 803 ++++++++++++++++++ + tests/unit/modules/test_ansiblegate.py | 15 + + tests/unit/states/test_ansiblegate.py | 113 +++ + 6 files changed, 1717 insertions(+), 23 deletions(-) + create mode 100644 tests/unit/files/playbooks/failed_example.json + create mode 100644 tests/unit/files/playbooks/success_example.json + create mode 100644 tests/unit/states/test_ansiblegate.py + +diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py +index 8e28fcafa3..e76809d4ba 100644 +--- a/salt/modules/ansiblegate.py ++++ b/salt/modules/ansiblegate.py +@@ -381,9 +381,9 @@ def playbooks(playbook, rundir=None, check=False, diff=False, extra_vars=None, + 'cwd': rundir, + 'cmd': ' '.join(command) + } +- ret = __salt__['cmd.run_all'](**cmd_kwargs) +- log.debug('Ansible Playbook Return: %s', ret) +- retdata = json.loads(ret['stdout']) +- if ret['retcode']: +- __context__['retcode'] = ret['retcode'] ++ ret = __salt__["cmd.run_all"](**cmd_kwargs) ++ log.debug("Ansible Playbook Return: %s", ret) ++ retdata = json.loads(ret["stdout"]) ++ if 'retcode' in ret: ++ __context__["retcode"] = retdata["retcode"] = ret["retcode"] + return retdata +diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py +index b42dc02938..d268e492e2 100644 +--- a/salt/states/ansiblegate.py ++++ b/salt/states/ansiblegate.py +@@ -120,9 +120,11 @@ def _changes(plays): + task_changes = {} + for task in play['tasks']: + host_changes = {} +- for host, data in six.iteritems(task['hosts']): +- if data['changed'] is True: +- host_changes[host] = data.get('diff', data.get('changes', {})) ++ for host, data in six.iteritems(task["hosts"]): ++ if data["changed"] is True: ++ host_changes[host] = data.get("diff", data.get("changes", {})) ++ elif any(x in data for x in ["failed", "skipped", "unreachable"]): ++ host_changes[host] = data.get("results", data.get("msg", {})) + if host_changes: + task_changes[task['task']['name']] = host_changes + if task_changes: +@@ -177,20 +179,33 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs= + if not isinstance(ansible_kwargs, dict): + log.debug('Setting ansible_kwargs to empty dict: %s', ansible_kwargs) + ansible_kwargs = {} +- checks = __salt__['ansible.playbooks'](name, rundir=rundir, check=True, diff=True, **ansible_kwargs) +- if all(not check['changed'] for check in six.itervalues(checks['stats'])): +- ret['comment'] = 'No changes to be made from playbook {0}'.format(name) +- ret['result'] = True +- elif __opts__['test']: +- ret['comment'] = 'Changes will be made from playbook {0}'.format(name) +- ret['result'] = None +- ret['changes'] = _changes(checks) ++ if __opts__["test"]: ++ checks = __salt__["ansible.playbooks"](name, rundir=rundir, check=True, diff=True, **ansible_kwargs) ++ if all(not check["changed"] and not check["failures"] and not check["unreachable"] and not check["skipped"] for check in six.itervalues(checks["stats"])): ++ ret["comment"] = "No changes to be made from playbook {0}".format(name) ++ ret["result"] = True ++ elif any(check["changed"] and not check["failures"] and not check["unreachable"] and not check["skipped"] for check in six.itervalues(checks["stats"])): ++ ret["comment"] = "Changes will be made from playbook {0}".format(name) ++ ret["result"] = None ++ ret["changes"] = _changes(checks) ++ else: ++ ret["comment"] = "There were some issues running the playbook {0}".format(name) ++ ret["result"] = False ++ ret["changes"] = _changes(checks) + else: +- results = __salt__['ansible.playbooks'](name, rundir=rundir, diff=True, **ansible_kwargs) +- ret['comment'] = 'Changes were made by playbook {0}'.format(name) +- ret['changes'] = _changes(results) +- ret['result'] = all( +- not check['failures'] and not check['unreachable'] +- for check in six.itervalues(checks['stats']) +- ) ++ results = __salt__["ansible.playbooks"](name, rundir=rundir, diff=True, **ansible_kwargs) ++ if all(not check["changed"] and not check["failures"] and not check["unreachable"] and not check["skipped"] for check in six.itervalues(results["stats"])): ++ ret["comment"] = "No changes to be made from playbook {0}".format(name) ++ ret["result"] = True ++ ret["changes"] = _changes(results) ++ else: ++ ret["changes"] = _changes(results) ++ ret["result"] = all( ++ not check["failures"] and not check["unreachable"] and not check["skipped"] ++ for check in six.itervalues(results["stats"]) ++ ) ++ if ret["result"]: ++ ret["comment"] = "Changes were made by playbook {0}".format(name) ++ else: ++ ret["comment"] = "There were some issues running the playbook {0}".format(name) + return ret +diff --git a/tests/unit/files/playbooks/failed_example.json b/tests/unit/files/playbooks/failed_example.json +new file mode 100644 +index 0000000000..9ee8ba25b7 +--- /dev/null ++++ b/tests/unit/files/playbooks/failed_example.json +@@ -0,0 +1,748 @@ ++{ ++ "custom_stats": {}, ++ "global_custom_stats": {}, ++ "plays": [ ++ { ++ "play": { ++ "duration": { ++ "end": "2020-08-14T11:55:33.889442Z", ++ "start": "2020-08-14T11:55:30.460145Z" ++ }, ++ "id": "5254001e-9fce-297d-21cd-000000000007", ++ "name": "py2hosts" ++ }, ++ "tasks": [ ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "_ansible_verbose_override": true, ++ "action": "gather_facts", ++ "ansible_facts": { ++ "ansible_all_ipv4_addresses": [ ++ "192.168.122.29" ++ ], ++ "ansible_all_ipv6_addresses": [ ++ "fe80::5054:ff:fe3e:4ce" ++ ], ++ "ansible_apparmor": { ++ "status": "disabled" ++ }, ++ "ansible_architecture": "x86_64", ++ "ansible_bios_date": "04/01/2014", ++ "ansible_bios_version": "rel-1.13.0-0-gf21b5a4-rebuilt.opensuse.org", ++ "ansible_cmdline": { ++ "BOOT_IMAGE": "/vmlinuz-3.10.0-862.el7.x86_64", ++ "LANG": "en_US.UTF-8", ++ "console": "ttyS0,115200", ++ "crashkernel": "auto", ++ "quiet": true, ++ "rhgb": true, ++ "ro": true, ++ "root": "UUID=2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "ansible_date_time": { ++ "date": "2020-08-14", ++ "day": "14", ++ "epoch": "1597406131", ++ "hour": "13", ++ "iso8601": "2020-08-14T11:55:31Z", ++ "iso8601_basic": "20200814T135531991936", ++ "iso8601_basic_short": "20200814T135531", ++ "iso8601_micro": "2020-08-14T11:55:31.992035Z", ++ "minute": "55", ++ "month": "08", ++ "second": "31", ++ "time": "13:55:31", ++ "tz": "CEST", ++ "tz_offset": "+0200", ++ "weekday": "Friday", ++ "weekday_number": "5", ++ "weeknumber": "32", ++ "year": "2020" ++ }, ++ "ansible_default_ipv4": { ++ "address": "192.168.122.29", ++ "alias": "eth0", ++ "broadcast": "192.168.122.255", ++ "gateway": "192.168.122.1", ++ "interface": "eth0", ++ "macaddress": "52:54:00:3e:04:ce", ++ "mtu": 1500, ++ "netmask": "255.255.255.0", ++ "network": "192.168.122.0", ++ "type": "ether" ++ }, ++ "ansible_default_ipv6": {}, ++ "ansible_device_links": { ++ "ids": {}, ++ "labels": {}, ++ "masters": {}, ++ "uuids": { ++ "vda1": [ ++ "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ ], ++ "vda2": [ ++ "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ ], ++ "vda3": [ ++ "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ ], ++ "vda5": [ ++ "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ ] ++ } ++ }, ++ "ansible_devices": { ++ "vda": { ++ "holders": [], ++ "host": "", ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [] ++ }, ++ "model": null, ++ "partitions": { ++ "vda1": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ ] ++ }, ++ "sectors": "2097152", ++ "sectorsize": 512, ++ "size": "1.00 GB", ++ "start": "2048", ++ "uuid": "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ }, ++ "vda2": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ ] ++ }, ++ "sectors": "4196352", ++ "sectorsize": 512, ++ "size": "2.00 GB", ++ "start": "2099200", ++ "uuid": "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ }, ++ "vda3": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ ] ++ }, ++ "sectors": "104857600", ++ "sectorsize": 512, ++ "size": "50.00 GB", ++ "start": "6295552", ++ "uuid": "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "vda4": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [] ++ }, ++ "sectors": "2", ++ "sectorsize": 512, ++ "size": "1.00 KB", ++ "start": "111153152", ++ "uuid": null ++ }, ++ "vda5": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ ] ++ }, ++ "sectors": "308275200", ++ "sectorsize": 512, ++ "size": "147.00 GB", ++ "start": "111155200", ++ "uuid": "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ } ++ }, ++ "removable": "0", ++ "rotational": "1", ++ "sas_address": null, ++ "sas_device_handle": null, ++ "scheduler_mode": "mq-deadline", ++ "sectors": "419430400", ++ "sectorsize": "512", ++ "size": "200.00 GB", ++ "support_discard": "0", ++ "vendor": "0x1af4", ++ "virtual": 1 ++ } ++ }, ++ "ansible_distribution": "CentOS", ++ "ansible_distribution_file_parsed": true, ++ "ansible_distribution_file_path": "/etc/redhat-release", ++ "ansible_distribution_file_variety": "RedHat", ++ "ansible_distribution_major_version": "7", ++ "ansible_distribution_release": "Core", ++ "ansible_distribution_version": "7.5", ++ "ansible_dns": { ++ "nameservers": [ ++ "192.168.122.1" ++ ] ++ }, ++ "ansible_domain": "tf.local", ++ "ansible_effective_group_id": 0, ++ "ansible_effective_user_id": 0, ++ "ansible_env": { ++ "HOME": "/root", ++ "LANG": "es_ES.utf8", ++ "LC_ADDRESS": "C", ++ "LC_COLLATE": "C", ++ "LC_CTYPE": "C", ++ "LC_IDENTIFICATION": "C", ++ "LC_MEASUREMENT": "C", ++ "LC_MESSAGES": "C", ++ "LC_MONETARY": "C", ++ "LC_NAME": "C", ++ "LC_NUMERIC": "C", ++ "LC_PAPER": "C", ++ "LC_TELEPHONE": "C", ++ "LC_TIME": "C", ++ "LESSOPEN": "||/usr/bin/lesspipe.sh %s", ++ "LOGNAME": "root", ++ "LS_COLORS": "rs=0:di=38;5;27:ln=38;5;51:mh=44;38;5;15:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=05;48;5;232;38;5;15:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;34:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.Z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.axv=38;5;13:*.anx=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.axa=38;5;45:*.oga=38;5;45:*.spx=38;5;45:*.xspf=38;5;45:", ++ "MAIL": "/var/mail/root", ++ "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", ++ "PWD": "/root", ++ "SHELL": "/bin/bash", ++ "SHLVL": "2", ++ "SSH_CLIENT": "192.168.122.179 55766 22", ++ "SSH_CONNECTION": "192.168.122.179 55766 192.168.122.29 22", ++ "SSH_TTY": "/dev/pts/0", ++ "TERM": "xterm-256color", ++ "USER": "root", ++ "XDG_RUNTIME_DIR": "/run/user/0", ++ "XDG_SESSION_ID": "110", ++ "_": "/usr/bin/python" ++ }, ++ "ansible_eth0": { ++ "active": true, ++ "device": "eth0", ++ "features": { ++ "busy_poll": "off [fixed]", ++ "fcoe_mtu": "off [fixed]", ++ "generic_receive_offload": "on", ++ "generic_segmentation_offload": "on", ++ "highdma": "on [fixed]", ++ "hw_tc_offload": "off [fixed]", ++ "l2_fwd_offload": "off [fixed]", ++ "large_receive_offload": "off [fixed]", ++ "loopback": "off [fixed]", ++ "netns_local": "off [fixed]", ++ "ntuple_filters": "off [fixed]", ++ "receive_hashing": "off [fixed]", ++ "rx_all": "off [fixed]", ++ "rx_checksumming": "on [fixed]", ++ "rx_fcs": "off [fixed]", ++ "rx_udp_tunnel_port_offload": "off [fixed]", ++ "rx_vlan_filter": "on [fixed]", ++ "rx_vlan_offload": "off [fixed]", ++ "rx_vlan_stag_filter": "off [fixed]", ++ "rx_vlan_stag_hw_parse": "off [fixed]", ++ "scatter_gather": "on", ++ "tcp_segmentation_offload": "on", ++ "tx_checksum_fcoe_crc": "off [fixed]", ++ "tx_checksum_ip_generic": "on", ++ "tx_checksum_ipv4": "off [fixed]", ++ "tx_checksum_ipv6": "off [fixed]", ++ "tx_checksum_sctp": "off [fixed]", ++ "tx_checksumming": "on", ++ "tx_fcoe_segmentation": "off [fixed]", ++ "tx_gre_csum_segmentation": "off [fixed]", ++ "tx_gre_segmentation": "off [fixed]", ++ "tx_gso_partial": "off [fixed]", ++ "tx_gso_robust": "off [fixed]", ++ "tx_ipip_segmentation": "off [fixed]", ++ "tx_lockless": "off [fixed]", ++ "tx_nocache_copy": "off", ++ "tx_scatter_gather": "on", ++ "tx_scatter_gather_fraglist": "off [fixed]", ++ "tx_sctp_segmentation": "off [fixed]", ++ "tx_sit_segmentation": "off [fixed]", ++ "tx_tcp6_segmentation": "on", ++ "tx_tcp_ecn_segmentation": "on", ++ "tx_tcp_mangleid_segmentation": "off", ++ "tx_tcp_segmentation": "on", ++ "tx_udp_tnl_csum_segmentation": "off [fixed]", ++ "tx_udp_tnl_segmentation": "off [fixed]", ++ "tx_vlan_offload": "off [fixed]", ++ "tx_vlan_stag_hw_insert": "off [fixed]", ++ "udp_fragmentation_offload": "on", ++ "vlan_challenged": "off [fixed]" ++ }, ++ "hw_timestamp_filters": [], ++ "ipv4": { ++ "address": "192.168.122.29", ++ "broadcast": "192.168.122.255", ++ "netmask": "255.255.255.0", ++ "network": "192.168.122.0" ++ }, ++ "ipv6": [ ++ { ++ "address": "fe80::5054:ff:fe3e:4ce", ++ "prefix": "64", ++ "scope": "link" ++ } ++ ], ++ "macaddress": "52:54:00:3e:04:ce", ++ "module": "virtio_net", ++ "mtu": 1500, ++ "pciid": "virtio0", ++ "promisc": false, ++ "timestamping": [ ++ "rx_software", ++ "software" ++ ], ++ "type": "ether" ++ }, ++ "ansible_fibre_channel_wwn": [], ++ "ansible_fips": false, ++ "ansible_form_factor": "Other", ++ "ansible_fqdn": "centos7-host1.tf.local", ++ "ansible_hostname": "centos7-host1", ++ "ansible_hostnqn": "", ++ "ansible_interfaces": [ ++ "lo", ++ "eth0" ++ ], ++ "ansible_is_chroot": false, ++ "ansible_iscsi_iqn": "", ++ "ansible_kernel": "3.10.0-862.el7.x86_64", ++ "ansible_kernel_version": "#1 SMP Fri Apr 20 16:44:24 UTC 2018", ++ "ansible_lo": { ++ "active": true, ++ "device": "lo", ++ "features": { ++ "busy_poll": "off [fixed]", ++ "fcoe_mtu": "off [fixed]", ++ "generic_receive_offload": "on", ++ "generic_segmentation_offload": "on", ++ "highdma": "on [fixed]", ++ "hw_tc_offload": "off [fixed]", ++ "l2_fwd_offload": "off [fixed]", ++ "large_receive_offload": "off [fixed]", ++ "loopback": "on [fixed]", ++ "netns_local": "on [fixed]", ++ "ntuple_filters": "off [fixed]", ++ "receive_hashing": "off [fixed]", ++ "rx_all": "off [fixed]", ++ "rx_checksumming": "on [fixed]", ++ "rx_fcs": "off [fixed]", ++ "rx_udp_tunnel_port_offload": "off [fixed]", ++ "rx_vlan_filter": "off [fixed]", ++ "rx_vlan_offload": "off [fixed]", ++ "rx_vlan_stag_filter": "off [fixed]", ++ "rx_vlan_stag_hw_parse": "off [fixed]", ++ "scatter_gather": "on", ++ "tcp_segmentation_offload": "on", ++ "tx_checksum_fcoe_crc": "off [fixed]", ++ "tx_checksum_ip_generic": "on [fixed]", ++ "tx_checksum_ipv4": "off [fixed]", ++ "tx_checksum_ipv6": "off [fixed]", ++ "tx_checksum_sctp": "on [fixed]", ++ "tx_checksumming": "on", ++ "tx_fcoe_segmentation": "off [fixed]", ++ "tx_gre_csum_segmentation": "off [fixed]", ++ "tx_gre_segmentation": "off [fixed]", ++ "tx_gso_partial": "off [fixed]", ++ "tx_gso_robust": "off [fixed]", ++ "tx_ipip_segmentation": "off [fixed]", ++ "tx_lockless": "on [fixed]", ++ "tx_nocache_copy": "off [fixed]", ++ "tx_scatter_gather": "on [fixed]", ++ "tx_scatter_gather_fraglist": "on [fixed]", ++ "tx_sctp_segmentation": "on", ++ "tx_sit_segmentation": "off [fixed]", ++ "tx_tcp6_segmentation": "on", ++ "tx_tcp_ecn_segmentation": "on", ++ "tx_tcp_mangleid_segmentation": "on", ++ "tx_tcp_segmentation": "on", ++ "tx_udp_tnl_csum_segmentation": "off [fixed]", ++ "tx_udp_tnl_segmentation": "off [fixed]", ++ "tx_vlan_offload": "off [fixed]", ++ "tx_vlan_stag_hw_insert": "off [fixed]", ++ "udp_fragmentation_offload": "on", ++ "vlan_challenged": "on [fixed]" ++ }, ++ "hw_timestamp_filters": [], ++ "ipv4": { ++ "address": "127.0.0.1", ++ "broadcast": "host", ++ "netmask": "255.0.0.0", ++ "network": "127.0.0.0" ++ }, ++ "ipv6": [ ++ { ++ "address": "::1", ++ "prefix": "128", ++ "scope": "host" ++ } ++ ], ++ "mtu": 65536, ++ "promisc": false, ++ "timestamping": [ ++ "rx_software", ++ "software" ++ ], ++ "type": "loopback" ++ }, ++ "ansible_local": {}, ++ "ansible_lsb": {}, ++ "ansible_machine": "x86_64", ++ "ansible_machine_id": "d5f025e24919a00e864180785ebaa8c9", ++ "ansible_memfree_mb": 717, ++ "ansible_memory_mb": { ++ "nocache": { ++ "free": 893, ++ "used": 98 ++ }, ++ "real": { ++ "free": 717, ++ "total": 991, ++ "used": 274 ++ }, ++ "swap": { ++ "cached": 0, ++ "free": 2048, ++ "total": 2048, ++ "used": 0 ++ } ++ }, ++ "ansible_memtotal_mb": 991, ++ "ansible_mounts": [ ++ { ++ "block_available": 243103, ++ "block_size": 4096, ++ "block_total": 259584, ++ "block_used": 16481, ++ "device": "/dev/vda1", ++ "fstype": "xfs", ++ "inode_available": 523998, ++ "inode_total": 524288, ++ "inode_used": 290, ++ "mount": "/boot", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 995749888, ++ "size_total": 1063256064, ++ "uuid": "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ }, ++ { ++ "block_available": 12902656, ++ "block_size": 4096, ++ "block_total": 13100800, ++ "block_used": 198144, ++ "device": "/dev/vda3", ++ "fstype": "xfs", ++ "inode_available": 26189994, ++ "inode_total": 26214400, ++ "inode_used": 24406, ++ "mount": "/", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 52849278976, ++ "size_total": 53660876800, ++ "uuid": "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ { ++ "block_available": 38507349, ++ "block_size": 4096, ++ "block_total": 38515585, ++ "block_used": 8236, ++ "device": "/dev/vda5", ++ "fstype": "xfs", ++ "inode_available": 77068797, ++ "inode_total": 77068800, ++ "inode_used": 3, ++ "mount": "/home", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 157726101504, ++ "size_total": 157759836160, ++ "uuid": "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ } ++ ], ++ "ansible_nodename": "centos7-host1", ++ "ansible_os_family": "RedHat", ++ "ansible_pkg_mgr": "yum", ++ "ansible_proc_cmdline": { ++ "BOOT_IMAGE": "/vmlinuz-3.10.0-862.el7.x86_64", ++ "LANG": "en_US.UTF-8", ++ "console": "ttyS0,115200", ++ "crashkernel": "auto", ++ "quiet": true, ++ "rhgb": true, ++ "ro": true, ++ "root": "UUID=2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "ansible_processor": [ ++ "0", ++ "GenuineIntel", ++ "QEMU Virtual CPU version 2.5+" ++ ], ++ "ansible_processor_cores": 1, ++ "ansible_processor_count": 1, ++ "ansible_processor_threads_per_core": 1, ++ "ansible_processor_vcpus": 1, ++ "ansible_product_name": "Standard PC (i440FX + PIIX, 1996)", ++ "ansible_product_serial": "NA", ++ "ansible_product_uuid": "18FEBA4D-2060-45E8-87AF-AD6574F522CC", ++ "ansible_product_version": "pc-i440fx-4.2", ++ "ansible_python": { ++ "executable": "/usr/bin/python", ++ "has_sslcontext": true, ++ "type": "CPython", ++ "version": { ++ "major": 2, ++ "micro": 5, ++ "minor": 7, ++ "releaselevel": "final", ++ "serial": 0 ++ }, ++ "version_info": [ ++ 2, ++ 7, ++ 5, ++ "final", ++ 0 ++ ] ++ }, ++ "ansible_python_version": "2.7.5", ++ "ansible_real_group_id": 0, ++ "ansible_real_user_id": 0, ++ "ansible_selinux": { ++ "status": "disabled" ++ }, ++ "ansible_selinux_python_present": true, ++ "ansible_service_mgr": "systemd", ++ "ansible_ssh_host_key_ecdsa_public": "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE3bXHUHyjmlbxE6LCP2ohRTr0pTX7sq89g0yKvovFK1qhP1rsBvy2jW8wjo2P8mlBWhL7obRGl8B+i3cMxZdrc=", ++ "ansible_ssh_host_key_ed25519_public": "AAAAC3NzaC1lZDI1NTE5AAAAIHv4wovK7u1Est8e1rMvQifupxLPpxtNEJIvKHq/iIVF", ++ "ansible_ssh_host_key_rsa_public": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDPW4spvldGYXFraJCWJAqkuyQQRogSL+aECRU0hAG+IwESq3ceVkUZrvMVnhxmVImcRGWLCP24wmiMC2G/sDMHfBIhQIc4ySvLLyVd20VIsQHWiODQsSZTKCWkIwNmWuUD/8FcIpHm4YKlzZdHRVPwx9oIkdzoxgGyGZ3em7QwhryPZ+GiK8P9dEE2xy2lfAMXCFEL6Eyw/WF1AS0KLZiKl5ct9aYedUZN1rWkWW1Kb9S+OsZ+qzjdZbU2EfQI8SnP8kkvKt1E/B1UnsfZ5R0nlsyIX6Bh8oCluqJrxXrsTBf/s4Pe76/Q7JH/QHp2Yw+sQb+l7wXhlNmDRTpqXDdR", ++ "ansible_swapfree_mb": 2048, ++ "ansible_swaptotal_mb": 2048, ++ "ansible_system": "Linux", ++ "ansible_system_capabilities": [ ++ "cap_chown", ++ "cap_dac_override", ++ "cap_dac_read_search", ++ "cap_fowner", ++ "cap_fsetid", ++ "cap_kill", ++ "cap_setgid", ++ "cap_setuid", ++ "cap_setpcap", ++ "cap_linux_immutable", ++ "cap_net_bind_service", ++ "cap_net_broadcast", ++ "cap_net_admin", ++ "cap_net_raw", ++ "cap_ipc_lock", ++ "cap_ipc_owner", ++ "cap_sys_module", ++ "cap_sys_rawio", ++ "cap_sys_chroot", ++ "cap_sys_ptrace", ++ "cap_sys_pacct", ++ "cap_sys_admin", ++ "cap_sys_boot", ++ "cap_sys_nice", ++ "cap_sys_resource", ++ "cap_sys_time", ++ "cap_sys_tty_config", ++ "cap_mknod", ++ "cap_lease", ++ "cap_audit_write", ++ "cap_audit_control", ++ "cap_setfcap", ++ "cap_mac_override", ++ "cap_mac_admin", ++ "cap_syslog", ++ "35", ++ "36+ep" ++ ], ++ "ansible_system_capabilities_enforced": "True", ++ "ansible_system_vendor": "QEMU", ++ "ansible_uptime_seconds": 178555, ++ "ansible_user_dir": "/root", ++ "ansible_user_gecos": "root", ++ "ansible_user_gid": 0, ++ "ansible_user_id": "root", ++ "ansible_user_shell": "/bin/bash", ++ "ansible_user_uid": 0, ++ "ansible_userspace_architecture": "x86_64", ++ "ansible_userspace_bits": "64", ++ "ansible_virtualization_role": "guest", ++ "ansible_virtualization_type": "kvm", ++ "discovered_interpreter_python": "/usr/bin/python", ++ "gather_subset": [ ++ "all" ++ ], ++ "module_setup": true ++ }, ++ "changed": false, ++ "deprecations": [], ++ "warnings": [] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:31.760375Z", ++ "start": "2020-08-14T11:55:30.470536Z" ++ }, ++ "id": "5254001e-9fce-297d-21cd-00000000000f", ++ "name": "Gathering Facts" ++ } ++ }, ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "action": "yum", ++ "changed": false, ++ "invocation": { ++ "module_args": { ++ "allow_downgrade": false, ++ "autoremove": false, ++ "bugfix": false, ++ "conf_file": null, ++ "disable_excludes": null, ++ "disable_gpg_check": false, ++ "disable_plugin": [], ++ "disablerepo": [], ++ "download_dir": null, ++ "download_only": false, ++ "enable_plugin": [], ++ "enablerepo": [], ++ "exclude": [], ++ "install_repoquery": true, ++ "install_weak_deps": true, ++ "installroot": "/", ++ "list": null, ++ "lock_timeout": 30, ++ "name": [ ++ "httpd" ++ ], ++ "releasever": null, ++ "security": false, ++ "skip_broken": false, ++ "state": "present", ++ "update_cache": false, ++ "update_only": false, ++ "use_backend": "auto", ++ "validate_certs": true ++ } ++ }, ++ "msg": "", ++ "rc": 0, ++ "results": [ ++ "httpd-2.4.6-93.el7.centos.x86_64 providing httpd is already installed" ++ ] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:32.952644Z", ++ "start": "2020-08-14T11:55:31.776073Z" ++ }, ++ "id": "5254001e-9fce-297d-21cd-000000000009", ++ "name": "yum" ++ } ++ }, ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "action": "yum", ++ "changed": false, ++ "failed": true, ++ "invocation": { ++ "module_args": { ++ "allow_downgrade": false, ++ "autoremove": false, ++ "bugfix": false, ++ "conf_file": null, ++ "disable_excludes": null, ++ "disable_gpg_check": false, ++ "disable_plugin": [], ++ "disablerepo": [], ++ "download_dir": null, ++ "download_only": false, ++ "enable_plugin": [], ++ "enablerepo": [], ++ "exclude": [], ++ "install_repoquery": true, ++ "install_weak_deps": true, ++ "installroot": "/", ++ "list": null, ++ "lock_timeout": 30, ++ "name": [ ++ "rsyndc" ++ ], ++ "releasever": null, ++ "security": false, ++ "skip_broken": false, ++ "state": "present", ++ "update_cache": false, ++ "update_only": false, ++ "use_backend": "auto", ++ "validate_certs": true ++ } ++ }, ++ "msg": "No package matching 'rsyndc' found available, installed or updated", ++ "rc": 126, ++ "results": [ ++ "No package matching 'rsyndc' found available, installed or updated" ++ ] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:33.889442Z", ++ "start": "2020-08-14T11:55:32.969762Z" ++ }, ++ "id": "5254001e-9fce-297d-21cd-00000000000a", ++ "name": "yum" ++ } ++ } ++ ] ++ } ++ ], ++ "stats": { ++ "centos7-host1.tf.local": { ++ "changed": 0, ++ "failures": 1, ++ "ignored": 0, ++ "ok": 2, ++ "rescued": 0, ++ "skipped": 0, ++ "unreachable": 0 ++ } ++ }, ++ "retcode": 2 ++} +diff --git a/tests/unit/files/playbooks/success_example.json b/tests/unit/files/playbooks/success_example.json +new file mode 100644 +index 0000000000..8a9f3ad868 +--- /dev/null ++++ b/tests/unit/files/playbooks/success_example.json +@@ -0,0 +1,803 @@ ++{ ++ "custom_stats": {}, ++ "global_custom_stats": {}, ++ "plays": [ ++ { ++ "play": { ++ "duration": { ++ "end": "2020-08-14T11:55:58.334076Z", ++ "start": "2020-08-14T11:55:54.295001Z" ++ }, ++ "id": "5254001e-9fce-f8b5-c66a-000000000007", ++ "name": "py2hosts" ++ }, ++ "tasks": [ ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "_ansible_verbose_override": true, ++ "action": "gather_facts", ++ "ansible_facts": { ++ "ansible_all_ipv4_addresses": [ ++ "192.168.122.29" ++ ], ++ "ansible_all_ipv6_addresses": [ ++ "fe80::5054:ff:fe3e:4ce" ++ ], ++ "ansible_apparmor": { ++ "status": "disabled" ++ }, ++ "ansible_architecture": "x86_64", ++ "ansible_bios_date": "04/01/2014", ++ "ansible_bios_version": "rel-1.13.0-0-gf21b5a4-rebuilt.opensuse.org", ++ "ansible_cmdline": { ++ "BOOT_IMAGE": "/vmlinuz-3.10.0-862.el7.x86_64", ++ "LANG": "en_US.UTF-8", ++ "console": "ttyS0,115200", ++ "crashkernel": "auto", ++ "quiet": true, ++ "rhgb": true, ++ "ro": true, ++ "root": "UUID=2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "ansible_date_time": { ++ "date": "2020-08-14", ++ "day": "14", ++ "epoch": "1597406155", ++ "hour": "13", ++ "iso8601": "2020-08-14T11:55:55Z", ++ "iso8601_basic": "20200814T135555808955", ++ "iso8601_basic_short": "20200814T135555", ++ "iso8601_micro": "2020-08-14T11:55:55.809048Z", ++ "minute": "55", ++ "month": "08", ++ "second": "55", ++ "time": "13:55:55", ++ "tz": "CEST", ++ "tz_offset": "+0200", ++ "weekday": "Friday", ++ "weekday_number": "5", ++ "weeknumber": "32", ++ "year": "2020" ++ }, ++ "ansible_default_ipv4": { ++ "address": "192.168.122.29", ++ "alias": "eth0", ++ "broadcast": "192.168.122.255", ++ "gateway": "192.168.122.1", ++ "interface": "eth0", ++ "macaddress": "52:54:00:3e:04:ce", ++ "mtu": 1500, ++ "netmask": "255.255.255.0", ++ "network": "192.168.122.0", ++ "type": "ether" ++ }, ++ "ansible_default_ipv6": {}, ++ "ansible_device_links": { ++ "ids": {}, ++ "labels": {}, ++ "masters": {}, ++ "uuids": { ++ "vda1": [ ++ "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ ], ++ "vda2": [ ++ "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ ], ++ "vda3": [ ++ "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ ], ++ "vda5": [ ++ "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ ] ++ } ++ }, ++ "ansible_devices": { ++ "vda": { ++ "holders": [], ++ "host": "", ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [] ++ }, ++ "model": null, ++ "partitions": { ++ "vda1": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ ] ++ }, ++ "sectors": "2097152", ++ "sectorsize": 512, ++ "size": "1.00 GB", ++ "start": "2048", ++ "uuid": "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ }, ++ "vda2": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ ] ++ }, ++ "sectors": "4196352", ++ "sectorsize": 512, ++ "size": "2.00 GB", ++ "start": "2099200", ++ "uuid": "5ec08dbf-55e4-4fb1-a866-7b0fedcb4a24" ++ }, ++ "vda3": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ ] ++ }, ++ "sectors": "104857600", ++ "sectorsize": 512, ++ "size": "50.00 GB", ++ "start": "6295552", ++ "uuid": "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "vda4": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [] ++ }, ++ "sectors": "2", ++ "sectorsize": 512, ++ "size": "1.00 KB", ++ "start": "111153152", ++ "uuid": null ++ }, ++ "vda5": { ++ "holders": [], ++ "links": { ++ "ids": [], ++ "labels": [], ++ "masters": [], ++ "uuids": [ ++ "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ ] ++ }, ++ "sectors": "308275200", ++ "sectorsize": 512, ++ "size": "147.00 GB", ++ "start": "111155200", ++ "uuid": "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ } ++ }, ++ "removable": "0", ++ "rotational": "1", ++ "sas_address": null, ++ "sas_device_handle": null, ++ "scheduler_mode": "mq-deadline", ++ "sectors": "419430400", ++ "sectorsize": "512", ++ "size": "200.00 GB", ++ "support_discard": "0", ++ "vendor": "0x1af4", ++ "virtual": 1 ++ } ++ }, ++ "ansible_distribution": "CentOS", ++ "ansible_distribution_file_parsed": true, ++ "ansible_distribution_file_path": "/etc/redhat-release", ++ "ansible_distribution_file_variety": "RedHat", ++ "ansible_distribution_major_version": "7", ++ "ansible_distribution_release": "Core", ++ "ansible_distribution_version": "7.5", ++ "ansible_dns": { ++ "nameservers": [ ++ "192.168.122.1" ++ ] ++ }, ++ "ansible_domain": "tf.local", ++ "ansible_effective_group_id": 0, ++ "ansible_effective_user_id": 0, ++ "ansible_env": { ++ "HOME": "/root", ++ "LANG": "es_ES.utf8", ++ "LC_ADDRESS": "C", ++ "LC_COLLATE": "C", ++ "LC_CTYPE": "C", ++ "LC_IDENTIFICATION": "C", ++ "LC_MEASUREMENT": "C", ++ "LC_MESSAGES": "C", ++ "LC_MONETARY": "C", ++ "LC_NAME": "C", ++ "LC_NUMERIC": "C", ++ "LC_PAPER": "C", ++ "LC_TELEPHONE": "C", ++ "LC_TIME": "C", ++ "LESSOPEN": "||/usr/bin/lesspipe.sh %s", ++ "LOGNAME": "root", ++ "LS_COLORS": "rs=0:di=38;5;27:ln=38;5;51:mh=44;38;5;15:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=05;48;5;232;38;5;15:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;34:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.Z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.axv=38;5;13:*.anx=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.axa=38;5;45:*.oga=38;5;45:*.spx=38;5;45:*.xspf=38;5;45:", ++ "MAIL": "/var/mail/root", ++ "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", ++ "PWD": "/root", ++ "SHELL": "/bin/bash", ++ "SHLVL": "2", ++ "SSH_CLIENT": "192.168.122.179 55766 22", ++ "SSH_CONNECTION": "192.168.122.179 55766 192.168.122.29 22", ++ "SSH_TTY": "/dev/pts/0", ++ "TERM": "xterm-256color", ++ "USER": "root", ++ "XDG_RUNTIME_DIR": "/run/user/0", ++ "XDG_SESSION_ID": "110", ++ "_": "/usr/bin/python" ++ }, ++ "ansible_eth0": { ++ "active": true, ++ "device": "eth0", ++ "features": { ++ "busy_poll": "off [fixed]", ++ "fcoe_mtu": "off [fixed]", ++ "generic_receive_offload": "on", ++ "generic_segmentation_offload": "on", ++ "highdma": "on [fixed]", ++ "hw_tc_offload": "off [fixed]", ++ "l2_fwd_offload": "off [fixed]", ++ "large_receive_offload": "off [fixed]", ++ "loopback": "off [fixed]", ++ "netns_local": "off [fixed]", ++ "ntuple_filters": "off [fixed]", ++ "receive_hashing": "off [fixed]", ++ "rx_all": "off [fixed]", ++ "rx_checksumming": "on [fixed]", ++ "rx_fcs": "off [fixed]", ++ "rx_udp_tunnel_port_offload": "off [fixed]", ++ "rx_vlan_filter": "on [fixed]", ++ "rx_vlan_offload": "off [fixed]", ++ "rx_vlan_stag_filter": "off [fixed]", ++ "rx_vlan_stag_hw_parse": "off [fixed]", ++ "scatter_gather": "on", ++ "tcp_segmentation_offload": "on", ++ "tx_checksum_fcoe_crc": "off [fixed]", ++ "tx_checksum_ip_generic": "on", ++ "tx_checksum_ipv4": "off [fixed]", ++ "tx_checksum_ipv6": "off [fixed]", ++ "tx_checksum_sctp": "off [fixed]", ++ "tx_checksumming": "on", ++ "tx_fcoe_segmentation": "off [fixed]", ++ "tx_gre_csum_segmentation": "off [fixed]", ++ "tx_gre_segmentation": "off [fixed]", ++ "tx_gso_partial": "off [fixed]", ++ "tx_gso_robust": "off [fixed]", ++ "tx_ipip_segmentation": "off [fixed]", ++ "tx_lockless": "off [fixed]", ++ "tx_nocache_copy": "off", ++ "tx_scatter_gather": "on", ++ "tx_scatter_gather_fraglist": "off [fixed]", ++ "tx_sctp_segmentation": "off [fixed]", ++ "tx_sit_segmentation": "off [fixed]", ++ "tx_tcp6_segmentation": "on", ++ "tx_tcp_ecn_segmentation": "on", ++ "tx_tcp_mangleid_segmentation": "off", ++ "tx_tcp_segmentation": "on", ++ "tx_udp_tnl_csum_segmentation": "off [fixed]", ++ "tx_udp_tnl_segmentation": "off [fixed]", ++ "tx_vlan_offload": "off [fixed]", ++ "tx_vlan_stag_hw_insert": "off [fixed]", ++ "udp_fragmentation_offload": "on", ++ "vlan_challenged": "off [fixed]" ++ }, ++ "hw_timestamp_filters": [], ++ "ipv4": { ++ "address": "192.168.122.29", ++ "broadcast": "192.168.122.255", ++ "netmask": "255.255.255.0", ++ "network": "192.168.122.0" ++ }, ++ "ipv6": [ ++ { ++ "address": "fe80::5054:ff:fe3e:4ce", ++ "prefix": "64", ++ "scope": "link" ++ } ++ ], ++ "macaddress": "52:54:00:3e:04:ce", ++ "module": "virtio_net", ++ "mtu": 1500, ++ "pciid": "virtio0", ++ "promisc": false, ++ "timestamping": [ ++ "rx_software", ++ "software" ++ ], ++ "type": "ether" ++ }, ++ "ansible_fibre_channel_wwn": [], ++ "ansible_fips": false, ++ "ansible_form_factor": "Other", ++ "ansible_fqdn": "centos7-host1.tf.local", ++ "ansible_hostname": "centos7-host1", ++ "ansible_hostnqn": "", ++ "ansible_interfaces": [ ++ "lo", ++ "eth0" ++ ], ++ "ansible_is_chroot": false, ++ "ansible_iscsi_iqn": "", ++ "ansible_kernel": "3.10.0-862.el7.x86_64", ++ "ansible_kernel_version": "#1 SMP Fri Apr 20 16:44:24 UTC 2018", ++ "ansible_lo": { ++ "active": true, ++ "device": "lo", ++ "features": { ++ "busy_poll": "off [fixed]", ++ "fcoe_mtu": "off [fixed]", ++ "generic_receive_offload": "on", ++ "generic_segmentation_offload": "on", ++ "highdma": "on [fixed]", ++ "hw_tc_offload": "off [fixed]", ++ "l2_fwd_offload": "off [fixed]", ++ "large_receive_offload": "off [fixed]", ++ "loopback": "on [fixed]", ++ "netns_local": "on [fixed]", ++ "ntuple_filters": "off [fixed]", ++ "receive_hashing": "off [fixed]", ++ "rx_all": "off [fixed]", ++ "rx_checksumming": "on [fixed]", ++ "rx_fcs": "off [fixed]", ++ "rx_udp_tunnel_port_offload": "off [fixed]", ++ "rx_vlan_filter": "off [fixed]", ++ "rx_vlan_offload": "off [fixed]", ++ "rx_vlan_stag_filter": "off [fixed]", ++ "rx_vlan_stag_hw_parse": "off [fixed]", ++ "scatter_gather": "on", ++ "tcp_segmentation_offload": "on", ++ "tx_checksum_fcoe_crc": "off [fixed]", ++ "tx_checksum_ip_generic": "on [fixed]", ++ "tx_checksum_ipv4": "off [fixed]", ++ "tx_checksum_ipv6": "off [fixed]", ++ "tx_checksum_sctp": "on [fixed]", ++ "tx_checksumming": "on", ++ "tx_fcoe_segmentation": "off [fixed]", ++ "tx_gre_csum_segmentation": "off [fixed]", ++ "tx_gre_segmentation": "off [fixed]", ++ "tx_gso_partial": "off [fixed]", ++ "tx_gso_robust": "off [fixed]", ++ "tx_ipip_segmentation": "off [fixed]", ++ "tx_lockless": "on [fixed]", ++ "tx_nocache_copy": "off [fixed]", ++ "tx_scatter_gather": "on [fixed]", ++ "tx_scatter_gather_fraglist": "on [fixed]", ++ "tx_sctp_segmentation": "on", ++ "tx_sit_segmentation": "off [fixed]", ++ "tx_tcp6_segmentation": "on", ++ "tx_tcp_ecn_segmentation": "on", ++ "tx_tcp_mangleid_segmentation": "on", ++ "tx_tcp_segmentation": "on", ++ "tx_udp_tnl_csum_segmentation": "off [fixed]", ++ "tx_udp_tnl_segmentation": "off [fixed]", ++ "tx_vlan_offload": "off [fixed]", ++ "tx_vlan_stag_hw_insert": "off [fixed]", ++ "udp_fragmentation_offload": "on", ++ "vlan_challenged": "on [fixed]" ++ }, ++ "hw_timestamp_filters": [], ++ "ipv4": { ++ "address": "127.0.0.1", ++ "broadcast": "host", ++ "netmask": "255.0.0.0", ++ "network": "127.0.0.0" ++ }, ++ "ipv6": [ ++ { ++ "address": "::1", ++ "prefix": "128", ++ "scope": "host" ++ } ++ ], ++ "mtu": 65536, ++ "promisc": false, ++ "timestamping": [ ++ "rx_software", ++ "software" ++ ], ++ "type": "loopback" ++ }, ++ "ansible_local": {}, ++ "ansible_lsb": {}, ++ "ansible_machine": "x86_64", ++ "ansible_machine_id": "d5f025e24919a00e864180785ebaa8c9", ++ "ansible_memfree_mb": 717, ++ "ansible_memory_mb": { ++ "nocache": { ++ "free": 893, ++ "used": 98 ++ }, ++ "real": { ++ "free": 717, ++ "total": 991, ++ "used": 274 ++ }, ++ "swap": { ++ "cached": 0, ++ "free": 2048, ++ "total": 2048, ++ "used": 0 ++ } ++ }, ++ "ansible_memtotal_mb": 991, ++ "ansible_mounts": [ ++ { ++ "block_available": 243103, ++ "block_size": 4096, ++ "block_total": 259584, ++ "block_used": 16481, ++ "device": "/dev/vda1", ++ "fstype": "xfs", ++ "inode_available": 523998, ++ "inode_total": 524288, ++ "inode_used": 290, ++ "mount": "/boot", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 995749888, ++ "size_total": 1063256064, ++ "uuid": "81b5a934-1fbb-4d6f-a972-bc7c9eb48345" ++ }, ++ { ++ "block_available": 12902661, ++ "block_size": 4096, ++ "block_total": 13100800, ++ "block_used": 198139, ++ "device": "/dev/vda3", ++ "fstype": "xfs", ++ "inode_available": 26189994, ++ "inode_total": 26214400, ++ "inode_used": 24406, ++ "mount": "/", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 52849299456, ++ "size_total": 53660876800, ++ "uuid": "2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ { ++ "block_available": 38507349, ++ "block_size": 4096, ++ "block_total": 38515585, ++ "block_used": 8236, ++ "device": "/dev/vda5", ++ "fstype": "xfs", ++ "inode_available": 77068797, ++ "inode_total": 77068800, ++ "inode_used": 3, ++ "mount": "/home", ++ "options": "rw,relatime,attr2,inode64,noquota", ++ "size_available": 157726101504, ++ "size_total": 157759836160, ++ "uuid": "7f7965bf-54e8-43d4-a2f6-cb7f56a9a249" ++ } ++ ], ++ "ansible_nodename": "centos7-host1", ++ "ansible_os_family": "RedHat", ++ "ansible_pkg_mgr": "yum", ++ "ansible_proc_cmdline": { ++ "BOOT_IMAGE": "/vmlinuz-3.10.0-862.el7.x86_64", ++ "LANG": "en_US.UTF-8", ++ "console": "ttyS0,115200", ++ "crashkernel": "auto", ++ "quiet": true, ++ "rhgb": true, ++ "ro": true, ++ "root": "UUID=2b13ca03-1e1d-4f51-8929-4e7fef390e0c" ++ }, ++ "ansible_processor": [ ++ "0", ++ "GenuineIntel", ++ "QEMU Virtual CPU version 2.5+" ++ ], ++ "ansible_processor_cores": 1, ++ "ansible_processor_count": 1, ++ "ansible_processor_threads_per_core": 1, ++ "ansible_processor_vcpus": 1, ++ "ansible_product_name": "Standard PC (i440FX + PIIX, 1996)", ++ "ansible_product_serial": "NA", ++ "ansible_product_uuid": "18FEBA4D-2060-45E8-87AF-AD6574F522CC", ++ "ansible_product_version": "pc-i440fx-4.2", ++ "ansible_python": { ++ "executable": "/usr/bin/python", ++ "has_sslcontext": true, ++ "type": "CPython", ++ "version": { ++ "major": 2, ++ "micro": 5, ++ "minor": 7, ++ "releaselevel": "final", ++ "serial": 0 ++ }, ++ "version_info": [ ++ 2, ++ 7, ++ 5, ++ "final", ++ 0 ++ ] ++ }, ++ "ansible_python_version": "2.7.5", ++ "ansible_real_group_id": 0, ++ "ansible_real_user_id": 0, ++ "ansible_selinux": { ++ "status": "disabled" ++ }, ++ "ansible_selinux_python_present": true, ++ "ansible_service_mgr": "systemd", ++ "ansible_ssh_host_key_ecdsa_public": "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE3bXHUHyjmlbxE6LCP2ohRTr0pTX7sq89g0yKvovFK1qhP1rsBvy2jW8wjo2P8mlBWhL7obRGl8B+i3cMxZdrc=", ++ "ansible_ssh_host_key_ed25519_public": "AAAAC3NzaC1lZDI1NTE5AAAAIHv4wovK7u1Est8e1rMvQifupxLPpxtNEJIvKHq/iIVF", ++ "ansible_ssh_host_key_rsa_public": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDPW4spvldGYXFraJCWJAqkuyQQRogSL+aECRU0hAG+IwESq3ceVkUZrvMVnhxmVImcRGWLCP24wmiMC2G/sDMHfBIhQIc4ySvLLyVd20VIsQHWiODQsSZTKCWkIwNmWuUD/8FcIpHm4YKlzZdHRVPwx9oIkdzoxgGyGZ3em7QwhryPZ+GiK8P9dEE2xy2lfAMXCFEL6Eyw/WF1AS0KLZiKl5ct9aYedUZN1rWkWW1Kb9S+OsZ+qzjdZbU2EfQI8SnP8kkvKt1E/B1UnsfZ5R0nlsyIX6Bh8oCluqJrxXrsTBf/s4Pe76/Q7JH/QHp2Yw+sQb+l7wXhlNmDRTpqXDdR", ++ "ansible_swapfree_mb": 2048, ++ "ansible_swaptotal_mb": 2048, ++ "ansible_system": "Linux", ++ "ansible_system_capabilities": [ ++ "cap_chown", ++ "cap_dac_override", ++ "cap_dac_read_search", ++ "cap_fowner", ++ "cap_fsetid", ++ "cap_kill", ++ "cap_setgid", ++ "cap_setuid", ++ "cap_setpcap", ++ "cap_linux_immutable", ++ "cap_net_bind_service", ++ "cap_net_broadcast", ++ "cap_net_admin", ++ "cap_net_raw", ++ "cap_ipc_lock", ++ "cap_ipc_owner", ++ "cap_sys_module", ++ "cap_sys_rawio", ++ "cap_sys_chroot", ++ "cap_sys_ptrace", ++ "cap_sys_pacct", ++ "cap_sys_admin", ++ "cap_sys_boot", ++ "cap_sys_nice", ++ "cap_sys_resource", ++ "cap_sys_time", ++ "cap_sys_tty_config", ++ "cap_mknod", ++ "cap_lease", ++ "cap_audit_write", ++ "cap_audit_control", ++ "cap_setfcap", ++ "cap_mac_override", ++ "cap_mac_admin", ++ "cap_syslog", ++ "35", ++ "36+ep" ++ ], ++ "ansible_system_capabilities_enforced": "True", ++ "ansible_system_vendor": "QEMU", ++ "ansible_uptime_seconds": 178578, ++ "ansible_user_dir": "/root", ++ "ansible_user_gecos": "root", ++ "ansible_user_gid": 0, ++ "ansible_user_id": "root", ++ "ansible_user_shell": "/bin/bash", ++ "ansible_user_uid": 0, ++ "ansible_userspace_architecture": "x86_64", ++ "ansible_userspace_bits": "64", ++ "ansible_virtualization_role": "guest", ++ "ansible_virtualization_type": "kvm", ++ "discovered_interpreter_python": "/usr/bin/python", ++ "gather_subset": [ ++ "all" ++ ], ++ "module_setup": true ++ }, ++ "changed": false, ++ "deprecations": [], ++ "warnings": [] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:55.578128Z", ++ "start": "2020-08-14T11:55:54.313122Z" ++ }, ++ "id": "5254001e-9fce-f8b5-c66a-00000000000f", ++ "name": "Gathering Facts" ++ } ++ }, ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "action": "yum", ++ "changed": false, ++ "invocation": { ++ "module_args": { ++ "allow_downgrade": false, ++ "autoremove": false, ++ "bugfix": false, ++ "conf_file": null, ++ "disable_excludes": null, ++ "disable_gpg_check": false, ++ "disable_plugin": [], ++ "disablerepo": [], ++ "download_dir": null, ++ "download_only": false, ++ "enable_plugin": [], ++ "enablerepo": [], ++ "exclude": [], ++ "install_repoquery": true, ++ "install_weak_deps": true, ++ "installroot": "/", ++ "list": null, ++ "lock_timeout": 30, ++ "name": [ ++ "httpd" ++ ], ++ "releasever": null, ++ "security": false, ++ "skip_broken": false, ++ "state": "present", ++ "update_cache": false, ++ "update_only": false, ++ "use_backend": "auto", ++ "validate_certs": true ++ } ++ }, ++ "msg": "", ++ "rc": 0, ++ "results": [ ++ "httpd-2.4.6-93.el7.centos.x86_64 providing httpd is already installed" ++ ] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:56.737921Z", ++ "start": "2020-08-14T11:55:55.596293Z" ++ }, ++ "id": "5254001e-9fce-f8b5-c66a-000000000009", ++ "name": "yum" ++ } ++ }, ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "action": "yum", ++ "changed": false, ++ "invocation": { ++ "module_args": { ++ "allow_downgrade": false, ++ "autoremove": false, ++ "bugfix": false, ++ "conf_file": null, ++ "disable_excludes": null, ++ "disable_gpg_check": false, ++ "disable_plugin": [], ++ "disablerepo": [], ++ "download_dir": null, ++ "download_only": false, ++ "enable_plugin": [], ++ "enablerepo": [], ++ "exclude": [], ++ "install_repoquery": true, ++ "install_weak_deps": true, ++ "installroot": "/", ++ "list": null, ++ "lock_timeout": 30, ++ "name": [ ++ "rsync" ++ ], ++ "releasever": null, ++ "security": false, ++ "skip_broken": false, ++ "state": "present", ++ "update_cache": false, ++ "update_only": false, ++ "use_backend": "auto", ++ "validate_certs": true ++ } ++ }, ++ "msg": "", ++ "rc": 0, ++ "results": [ ++ "rsync-3.1.2-10.el7.x86_64 providing rsync is already installed" ++ ] ++ } ++ }, ++ "task": { ++ "duration": { ++ "end": "2020-08-14T11:55:57.609670Z", ++ "start": "2020-08-14T11:55:56.755620Z" ++ }, ++ "id": "5254001e-9fce-f8b5-c66a-00000000000a", ++ "name": "yum" ++ } ++ }, ++ { ++ "hosts": { ++ "centos7-host1.tf.local": { ++ "_ansible_no_log": false, ++ "action": "synchronize", ++ "changed": true, ++ "cmd": "/usr/bin/rsync --delay-updates -F --compress --delete-after --archive --rsh=/usr/bin/ssh -S none -i /etc/ansible/keys/mykey.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null --out-format=<>%i %n%L /root/myfiles/ centos7-host1.tf.local:/var/www/html/", ++ "invocation": { ++ "module_args": { ++ "_local_rsync_password": null, ++ "_local_rsync_path": "rsync", ++ "_substitute_controller": false, ++ "archive": true, ++ "checksum": false, ++ "compress": true, ++ "copy_links": false, ++ "delete": true, ++ "dest": "centos7-host1.tf.local:/var/www/html/", ++ "dest_port": null, ++ "dirs": false, ++ "existing_only": false, ++ "group": null, ++ "link_dest": null, ++ "links": null, ++ "mode": "push", ++ "owner": null, ++ "partial": false, ++ "perms": null, ++ "private_key": "/etc/ansible/keys/mykey.pem", ++ "recursive": null, ++ "rsync_opts": [], ++ "rsync_path": null, ++ "rsync_timeout": 0, ++ "set_remote_user": true, ++ "src": "/root/myfiles/", ++ "ssh_args": null, ++ "times": null, ++ "verify_host": false ++ } ++ }, ++ "msg": " +Date: Mon, 7 Sep 2020 15:00:40 +0200 +Subject: [PATCH] Backport virt patches from 3001+ (#256) + +* Fix various spelling mistakes in master branch (#55954) + +* Fix typo of additional + +Signed-off-by: Benjamin Drung + +* Fix typo of against + +Signed-off-by: Benjamin Drung + +* Fix typo of amount + +Signed-off-by: Benjamin Drung + +* Fix typo of argument + +Signed-off-by: Benjamin Drung + +* Fix typo of attempt + +Signed-off-by: Benjamin Drung + +* Fix typo of bandwidth + +Signed-off-by: Benjamin Drung + +* Fix typo of caught + +Signed-off-by: Benjamin Drung + +* Fix typo of compatibility + +Signed-off-by: Benjamin Drung + +* Fix typo of consistency + +Signed-off-by: Benjamin Drung + +* Fix typo of conversions + +Signed-off-by: Benjamin Drung + +* Fix typo of corresponding + +Signed-off-by: Benjamin Drung + +* Fix typo of dependent + +Signed-off-by: Benjamin Drung + +* Fix typo of dictionary + +Signed-off-by: Benjamin Drung + +* Fix typo of disabled + +Signed-off-by: Benjamin Drung + +* Fix typo of adapters + +Signed-off-by: Benjamin Drung + +* Fix typo of disassociates + +Signed-off-by: Benjamin Drung + +* Fix typo of changes + +Signed-off-by: Benjamin Drung + +* Fix typo of command + +Signed-off-by: Benjamin Drung + +* Fix typo of communicate + +Signed-off-by: Benjamin Drung + +* Fix typo of community + +Signed-off-by: Benjamin Drung + +* Fix typo of configuration + +Signed-off-by: Benjamin Drung + +* Fix typo of default + +Signed-off-by: Benjamin Drung + +* Fix typo of absence + +Signed-off-by: Benjamin Drung + +* Fix typo of attribute + +Signed-off-by: Benjamin Drung + +* Fix typo of container + +Signed-off-by: Benjamin Drung + +* Fix typo of described + +Signed-off-by: Benjamin Drung + +* Fix typo of existence + +Signed-off-by: Benjamin Drung + +* Fix typo of explicit + +Signed-off-by: Benjamin Drung + +* Fix typo of formatted + +Signed-off-by: Benjamin Drung + +* Fix typo of guarantees + +Signed-off-by: Benjamin Drung + +* Fix typo of hexadecimal + +Signed-off-by: Benjamin Drung + +* Fix typo of hierarchy + +Signed-off-by: Benjamin Drung + +* Fix typo of initialize + +Signed-off-by: Benjamin Drung + +* Fix typo of label + +Signed-off-by: Benjamin Drung + +* Fix typo of management + +Signed-off-by: Benjamin Drung + +* Fix typo of mismatch + +Signed-off-by: Benjamin Drung + +* Fix typo of don't + +Signed-off-by: Benjamin Drung + +* Fix typo of manually + +Signed-off-by: Benjamin Drung + +* Fix typo of getting + +Signed-off-by: Benjamin Drung + +* Fix typo of information + +Signed-off-by: Benjamin Drung + +* Fix typo of meant + +Signed-off-by: Benjamin Drung + +* Fix typo of nonexistent + +Signed-off-by: Benjamin Drung + +* Fix typo of occur + +Signed-off-by: Benjamin Drung + +* Fix typo of omitted + +Signed-off-by: Benjamin Drung + +* Fix typo of normally + +Signed-off-by: Benjamin Drung + +* Fix typo of overridden + +Signed-off-by: Benjamin Drung + +* Fix typo of repository + +Signed-off-by: Benjamin Drung + +* Fix typo of separate + +Signed-off-by: Benjamin Drung + +* Fix typo of separator + +Signed-off-by: Benjamin Drung + +* Fix typo of specific + +Signed-off-by: Benjamin Drung + +* Fix typo of successful + +Signed-off-by: Benjamin Drung + +* Fix typo of succeeded + +Signed-off-by: Benjamin Drung + +* Fix typo of support + +Signed-off-by: Benjamin Drung + +* Fix typo of version + +Signed-off-by: Benjamin Drung + +* Fix typo of that's + +Signed-off-by: Benjamin Drung + +* Fix typo of "will be removed" + +Signed-off-by: Benjamin Drung + +* Fix typo of release + +Signed-off-by: Benjamin Drung + +* Fix typo of synchronize + +Signed-off-by: Benjamin Drung + +* Fix typo of python + +Signed-off-by: Benjamin Drung + +* Fix typo of usually + +Signed-off-by: Benjamin Drung + +* Fix typo of override + +Signed-off-by: Benjamin Drung + +* Fix typo of running + +Signed-off-by: Benjamin Drung + +* Fix typo of whether + +Signed-off-by: Benjamin Drung + +* Fix typo of package + +Signed-off-by: Benjamin Drung + +* Fix typo of persist + +Signed-off-by: Benjamin Drung + +* Fix typo of preferred + +Signed-off-by: Benjamin Drung + +* Fix typo of present + +Signed-off-by: Benjamin Drung + +* Fix typo of run + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of "allows someone to..." + +"Allows to" is not correct English. It must either be "allows someone +to" or "allows doing". + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of "number of times" + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of msgpack + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of daemonized + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of daemons + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of extemporaneous + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of instead + +Signed-off-by: Benjamin Drung + +* Fix spelling mistake of returning + +Signed-off-by: Benjamin Drung + +* Fix literal comparissons + +* virt: Convert cpu_baseline ElementTree to string + +In commit 0f5184c (Remove minidom use in virt module) the value +of `cpu` become `xml.etree.ElementTree.Element` and no longer +has a method `toxml()`. This results in the following error: + +$ salt '*' virt.cpu_baseline +host2: + The minion function caused an exception: Traceback (most recent call last): + File "/usr/lib/python3.7/site-packages/salt/minion.py", line 1675, in _thread_return + return_data = minion_instance.executors[fname](opts, data, func, args, kwargs) + File "/usr/lib/python3.7/site-packages/salt/executors/direct_call.py", line 12, in execute + return func(*args, **kwargs) + File "/usr/lib/python3.7/site-packages/salt/modules/virt.py", line 4410, in cpu_baseline + return cpu.toxml() + AttributeError: 'xml.etree.ElementTree.Element' object has no attribute 'toxml' + +Signed-off-by: Radostin Stoyanov + +* PR#57374 backport + +virt: pool secret should be undefined in pool_undefine not pool_delete + +virt: handle build differently depending on the pool type + +virt: don't fail if the pool secret has been removed + +* PR #57396 backport + +add firmware auto select feature + +* virt: Update dependencies + +Closes: #57641 + +Signed-off-by: Radostin Stoyanov + +* use null in sls file to map None object + +add sls file example + +reword doc + +* Update virt module and states and their tests to python3 + +* PR #57545 backport + +Move virt.init boot_dev parameter away from the kwargs + +virt: handle boot device in virt.update() + +virt: add boot_dev parameter to virt.running state + +* PR #57431 backport + +virt: Handle no available hypervisors + +virt: Remove unused imports + +* Blacken salt + +* Add method to remove circular references in data objects and add test (#54930) + +* Add method to remove circular references in data objects and add test + +* remove trailing whitespace + +* Blacken changed files + +Co-authored-by: xeacott +Co-authored-by: Frode Gundersen +Co-authored-by: Daniel A. Wozniak + +* PR #58332 backport + +virt: add debug log with VM XML definition + +Add xmlutil.get_xml_node() helper function + +Add salt.utils.data.get_value function + +Add change_xml() function to xmlutil + +virt.update: refactor the XML diffing code + +virt.test_update: move some code to make test more readable + +Co-authored-by: Benjamin Drung +Co-authored-by: Pedro Algarvio +Co-authored-by: Radostin Stoyanov +Co-authored-by: Firefly +Co-authored-by: Blacken Salt +Co-authored-by: Joe Eacott <31625359+xeacott@users.noreply.github.com> +Co-authored-by: xeacott +Co-authored-by: Frode Gundersen +Co-authored-by: Daniel A. Wozniak +--- + changelog/56454.fixed | 1 + + changelog/57544.added | 1 + + changelog/58331.fixed | 1 + + salt/modules/virt.py | 442 ++++--- + salt/states/virt.py | 171 ++- + salt/templates/virt/libvirt_domain.jinja | 2 +- + salt/utils/data.py | 977 +++++++++------ + salt/utils/xmlutil.py | 251 +++- + tests/pytests/unit/utils/test_data.py | 57 + + tests/pytests/unit/utils/test_xmlutil.py | 169 +++ + tests/unit/modules/test_virt.py | 218 ++-- + tests/unit/states/test_virt.py | 98 +- + tests/unit/utils/test_data.py | 1399 ++++++++++++---------- + tests/unit/utils/test_xmlutil.py | 164 +-- + 14 files changed, 2588 insertions(+), 1363 deletions(-) + create mode 100644 changelog/56454.fixed + create mode 100644 changelog/57544.added + create mode 100644 changelog/58331.fixed + create mode 100644 tests/pytests/unit/utils/test_data.py + create mode 100644 tests/pytests/unit/utils/test_xmlutil.py + +diff --git a/changelog/56454.fixed b/changelog/56454.fixed +new file mode 100644 +index 0000000000..978b4b6e03 +--- /dev/null ++++ b/changelog/56454.fixed +@@ -0,0 +1 @@ ++Better handle virt.pool_rebuild in virt.pool_running and virt.pool_defined states +diff --git a/changelog/57544.added b/changelog/57544.added +new file mode 100644 +index 0000000000..52071cf2c7 +--- /dev/null ++++ b/changelog/57544.added +@@ -0,0 +1 @@ ++Allow setting VM boot devices order in virt.running and virt.defined states +diff --git a/changelog/58331.fixed b/changelog/58331.fixed +new file mode 100644 +index 0000000000..4b8f78dd53 +--- /dev/null ++++ b/changelog/58331.fixed +@@ -0,0 +1 @@ ++Leave boot parameters untouched if boot parameter is set to None in virt.update +diff --git a/salt/modules/virt.py b/salt/modules/virt.py +index a78c21e323..cd80fbe608 100644 +--- a/salt/modules/virt.py ++++ b/salt/modules/virt.py +@@ -1,8 +1,11 @@ +-# -*- coding: utf-8 -*- + """ + Work with virtual machines managed by libvirt + +-:depends: libvirt Python module ++:depends: ++ * libvirt Python module ++ * libvirt client ++ * qemu-img ++ * grep + + Connection + ========== +@@ -73,7 +76,7 @@ The calls not using the libvirt connection setup are: + # of his in the virt func module have been used + + # Import python libs +-from __future__ import absolute_import, print_function, unicode_literals ++ + import base64 + import copy + import datetime +@@ -89,23 +92,19 @@ from xml.etree import ElementTree + from xml.sax import saxutils + + # Import third party libs +-import jinja2 + import jinja2.exceptions + + # Import salt libs ++import salt.utils.data + import salt.utils.files + import salt.utils.json +-import salt.utils.network + import salt.utils.path + import salt.utils.stringutils + import salt.utils.templates +-import salt.utils.validate.net +-import salt.utils.versions + import salt.utils.xmlutil as xmlutil + import salt.utils.yaml + from salt._compat import ipaddress + from salt.exceptions import CommandExecutionError, SaltInvocationError +-from salt.ext import six + from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin + from salt.ext.six.moves.urllib.parse import urlparse, urlunparse + from salt.utils.virt import check_remote, download_remote +@@ -227,8 +226,8 @@ def __get_conn(**kwargs): + ) + except Exception: # pylint: disable=broad-except + raise CommandExecutionError( +- "Sorry, {0} failed to open a connection to the hypervisor " +- "software at {1}".format(__grains__["fqdn"], conn_str) ++ "Sorry, {} failed to open a connection to the hypervisor " ++ "software at {}".format(__grains__["fqdn"], conn_str) + ) + return conn + +@@ -405,7 +404,7 @@ def _get_nics(dom): + # driver, source, and match can all have optional attributes + if re.match("(driver|source|address)", v_node.tag): + temp = {} +- for key, value in six.iteritems(v_node.attrib): ++ for key, value in v_node.attrib.items(): + temp[key] = value + nic[v_node.tag] = temp + # virtualport needs to be handled separately, to pick up the +@@ -413,7 +412,7 @@ def _get_nics(dom): + if v_node.tag == "virtualport": + temp = {} + temp["type"] = v_node.get("type") +- for key, value in six.iteritems(v_node.attrib): ++ for key, value in v_node.attrib.items(): + temp[key] = value + nic["virtualport"] = temp + if "mac" not in nic: +@@ -435,7 +434,7 @@ def _get_graphics(dom): + } + doc = ElementTree.fromstring(dom.XMLDesc(0)) + for g_node in doc.findall("devices/graphics"): +- for key, value in six.iteritems(g_node.attrib): ++ for key, value in g_node.attrib.items(): + out[key] = value + return out + +@@ -448,7 +447,7 @@ def _get_loader(dom): + doc = ElementTree.fromstring(dom.XMLDesc(0)) + for g_node in doc.findall("os/loader"): + out["path"] = g_node.text +- for key, value in six.iteritems(g_node.attrib): ++ for key, value in g_node.attrib.items(): + out[key] = value + return out + +@@ -503,7 +502,7 @@ def _get_disks(conn, dom): + qemu_target = source.get("protocol") + source_name = source.get("name") + if source_name: +- qemu_target = "{0}:{1}".format(qemu_target, source_name) ++ qemu_target = "{}:{}".format(qemu_target, source_name) + + # Reverse the magic for the rbd and gluster pools + if source.get("protocol") in ["rbd", "gluster"]: +@@ -633,7 +632,7 @@ def _get_target(target, ssh): + proto = "qemu" + if ssh: + proto += "+ssh" +- return " {0}://{1}/{2}".format(proto, target, "system") ++ return " {}://{}/{}".format(proto, target, "system") + + + def _gen_xml( +@@ -648,6 +647,7 @@ def _gen_xml( + arch, + graphics=None, + boot=None, ++ boot_dev=None, + **kwargs + ): + """ +@@ -657,8 +657,8 @@ def _gen_xml( + context = { + "hypervisor": hypervisor, + "name": name, +- "cpu": six.text_type(cpu), +- "mem": six.text_type(mem), ++ "cpu": str(cpu), ++ "mem": str(mem), + } + if hypervisor in ["qemu", "kvm"]: + context["controller_model"] = False +@@ -681,15 +681,17 @@ def _gen_xml( + graphics = None + context["graphics"] = graphics + +- if "boot_dev" in kwargs: +- context["boot_dev"] = [] +- for dev in kwargs["boot_dev"].split(): +- context["boot_dev"].append(dev) +- else: +- context["boot_dev"] = ["hd"] ++ context["boot_dev"] = boot_dev.split() if boot_dev is not None else ["hd"] + + context["boot"] = boot if boot else {} + ++ # if efi parameter is specified, prepare os_attrib ++ efi_value = context["boot"].get("efi", None) if boot else None ++ if efi_value is True: ++ context["boot"]["os_attrib"] = "firmware='efi'" ++ elif efi_value is not None and type(efi_value) != bool: ++ raise SaltInvocationError("Invalid efi value") ++ + if os_type == "xen": + # Compute the Xen PV boot method + if __grains__["os_family"] == "Suse": +@@ -720,7 +722,7 @@ def _gen_xml( + "target_dev": _get_disk_target(targets, len(diskp), prefix), + "disk_bus": disk["model"], + "format": disk.get("format", "raw"), +- "index": six.text_type(i), ++ "index": str(i), + } + targets.append(disk_context["target_dev"]) + if disk.get("source_file"): +@@ -825,8 +827,8 @@ def _gen_vol_xml( + "name": name, + "target": {"permissions": permissions, "nocow": nocow}, + "format": format, +- "size": six.text_type(size), +- "allocation": six.text_type(int(allocation) * 1024), ++ "size": str(size), ++ "allocation": str(int(allocation) * 1024), + "backingStore": backing_store, + } + fn_ = "libvirt_volume.jinja" +@@ -978,31 +980,29 @@ def _zfs_image_create( + """ + if not disk_image_name and not disk_size: + raise CommandExecutionError( +- "Unable to create new disk {0}, please specify" ++ "Unable to create new disk {}, please specify" + " the disk image name or disk size argument".format(disk_name) + ) + + if not pool: + raise CommandExecutionError( +- "Unable to create new disk {0}, please specify" ++ "Unable to create new disk {}, please specify" + " the disk pool name".format(disk_name) + ) + +- destination_fs = os.path.join(pool, "{0}.{1}".format(vm_name, disk_name)) ++ destination_fs = os.path.join(pool, "{}.{}".format(vm_name, disk_name)) + log.debug("Image destination will be %s", destination_fs) + + existing_disk = __salt__["zfs.list"](name=pool) + if "error" in existing_disk: + raise CommandExecutionError( +- "Unable to create new disk {0}. {1}".format( ++ "Unable to create new disk {}. {}".format( + destination_fs, existing_disk["error"] + ) + ) + elif destination_fs in existing_disk: + log.info( +- "ZFS filesystem {0} already exists. Skipping creation".format( +- destination_fs +- ) ++ "ZFS filesystem {} already exists. Skipping creation".format(destination_fs) + ) + blockdevice_path = os.path.join("/dev/zvol", pool, vm_name) + return blockdevice_path +@@ -1025,7 +1025,7 @@ def _zfs_image_create( + ) + + blockdevice_path = os.path.join( +- "/dev/zvol", pool, "{0}.{1}".format(vm_name, disk_name) ++ "/dev/zvol", pool, "{}.{}".format(vm_name, disk_name) + ) + log.debug("Image path will be %s", blockdevice_path) + return blockdevice_path +@@ -1042,7 +1042,7 @@ def _qemu_image_create(disk, create_overlay=False, saltenv="base"): + + if not disk_size and not disk_image: + raise CommandExecutionError( +- "Unable to create new disk {0}, please specify" ++ "Unable to create new disk {}, please specify" + " disk size and/or disk image argument".format(disk["filename"]) + ) + +@@ -1066,7 +1066,7 @@ def _qemu_image_create(disk, create_overlay=False, saltenv="base"): + if create_overlay and qcow2: + log.info("Cloning qcow2 image %s using copy on write", sfn) + __salt__["cmd.run"]( +- 'qemu-img create -f qcow2 -o backing_file="{0}" "{1}"'.format( ++ 'qemu-img create -f qcow2 -o backing_file="{}" "{}"'.format( + sfn, img_dest + ).split() + ) +@@ -1079,16 +1079,16 @@ def _qemu_image_create(disk, create_overlay=False, saltenv="base"): + if disk_size and qcow2: + log.debug("Resize qcow2 image to %sM", disk_size) + __salt__["cmd.run"]( +- 'qemu-img resize "{0}" {1}M'.format(img_dest, disk_size) ++ 'qemu-img resize "{}" {}M'.format(img_dest, disk_size) + ) + + log.debug("Apply umask and remove exec bit") + mode = (0o0777 ^ mask) & 0o0666 + os.chmod(img_dest, mode) + +- except (IOError, OSError) as err: ++ except OSError as err: + raise CommandExecutionError( +- "Problem while copying image. {0} - {1}".format(disk_image, err) ++ "Problem while copying image. {} - {}".format(disk_image, err) + ) + + else: +@@ -1099,13 +1099,13 @@ def _qemu_image_create(disk, create_overlay=False, saltenv="base"): + if disk_size: + log.debug("Create empty image with size %sM", disk_size) + __salt__["cmd.run"]( +- 'qemu-img create -f {0} "{1}" {2}M'.format( ++ 'qemu-img create -f {} "{}" {}M'.format( + disk.get("format", "qcow2"), img_dest, disk_size + ) + ) + else: + raise CommandExecutionError( +- "Unable to create new disk {0}," ++ "Unable to create new disk {}," + " please specify argument".format(img_dest) + ) + +@@ -1113,9 +1113,9 @@ def _qemu_image_create(disk, create_overlay=False, saltenv="base"): + mode = (0o0777 ^ mask) & 0o0666 + os.chmod(img_dest, mode) + +- except (IOError, OSError) as err: ++ except OSError as err: + raise CommandExecutionError( +- "Problem while creating volume {0} - {1}".format(img_dest, err) ++ "Problem while creating volume {} - {}".format(img_dest, err) + ) + + return img_dest +@@ -1252,7 +1252,7 @@ def _disk_profile(conn, profile, hypervisor, disks, vm_name): + __salt__["config.get"]("virt:disk", {}).get(profile, default) + ) + +- # Transform the list to remove one level of dictionnary and add the name as a property ++ # Transform the list to remove one level of dictionary and add the name as a property + disklist = [dict(d, name=name) for disk in disklist for name, d in disk.items()] + + # Merge with the user-provided disks definitions +@@ -1274,7 +1274,7 @@ def _disk_profile(conn, profile, hypervisor, disks, vm_name): + disk["model"] = "ide" + + # Add the missing properties that have defaults +- for key, val in six.iteritems(overlay): ++ for key, val in overlay.items(): + if key not in disk: + disk[key] = val + +@@ -1296,7 +1296,7 @@ def _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps): + Compute the disk file name and update it in the disk value. + """ + # Compute the filename without extension since it may not make sense for some pool types +- disk["filename"] = "{0}_{1}".format(vm_name, disk["name"]) ++ disk["filename"] = "{}_{}".format(vm_name, disk["name"]) + + # Compute the source file path + base_dir = disk.get("pool", None) +@@ -1311,7 +1311,7 @@ def _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps): + # For path-based disks, keep the qcow2 default format + if not disk.get("format"): + disk["format"] = "qcow2" +- disk["filename"] = "{0}.{1}".format(disk["filename"], disk["format"]) ++ disk["filename"] = "{}.{}".format(disk["filename"], disk["format"]) + disk["source_file"] = os.path.join(base_dir, disk["filename"]) + else: + if "pool" not in disk: +@@ -1365,7 +1365,7 @@ def _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps): + disk["format"] = volume_options.get("default_format", None) + + elif hypervisor == "bhyve" and vm_name: +- disk["filename"] = "{0}.{1}".format(vm_name, disk["name"]) ++ disk["filename"] = "{}.{}".format(vm_name, disk["name"]) + disk["source_file"] = os.path.join( + "/dev/zvol", base_dir or "", disk["filename"] + ) +@@ -1373,8 +1373,8 @@ def _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps): + elif hypervisor in ["esxi", "vmware"]: + if not base_dir: + base_dir = __salt__["config.get"]("virt:storagepool", "[0] ") +- disk["filename"] = "{0}.{1}".format(disk["filename"], disk["format"]) +- disk["source_file"] = "{0}{1}".format(base_dir, disk["filename"]) ++ disk["filename"] = "{}.{}".format(disk["filename"], disk["format"]) ++ disk["source_file"] = "{}{}".format(base_dir, disk["filename"]) + + + def _complete_nics(interfaces, hypervisor): +@@ -1422,7 +1422,7 @@ def _complete_nics(interfaces, hypervisor): + """ + Apply the default overlay to attributes + """ +- for key, value in six.iteritems(overlays[hypervisor]): ++ for key, value in overlays[hypervisor].items(): + if key not in attributes or not attributes[key]: + attributes[key] = value + +@@ -1449,7 +1449,7 @@ def _nic_profile(profile_name, hypervisor): + """ + Append dictionary profile data to interfaces list + """ +- for interface_name, attributes in six.iteritems(profile_dict): ++ for interface_name, attributes in profile_dict.items(): + attributes["name"] = interface_name + interfaces.append(attributes) + +@@ -1522,17 +1522,24 @@ def _handle_remote_boot_params(orig_boot): + new_boot = orig_boot.copy() + keys = orig_boot.keys() + cases = [ ++ {"efi"}, ++ {"kernel", "initrd", "efi"}, ++ {"kernel", "initrd", "cmdline", "efi"}, + {"loader", "nvram"}, + {"kernel", "initrd"}, + {"kernel", "initrd", "cmdline"}, +- {"loader", "nvram", "kernel", "initrd"}, +- {"loader", "nvram", "kernel", "initrd", "cmdline"}, ++ {"kernel", "initrd", "loader", "nvram"}, ++ {"kernel", "initrd", "cmdline", "loader", "nvram"}, + ] + + try: + if keys in cases: + for key in keys: +- if orig_boot.get(key) is not None and check_remote(orig_boot.get(key)): ++ if key == "efi" and type(orig_boot.get(key)) == bool: ++ new_boot[key] = orig_boot.get(key) ++ elif orig_boot.get(key) is not None and check_remote( ++ orig_boot.get(key) ++ ): + if saltinst_dir is None: + os.makedirs(CACHE_DIR) + saltinst_dir = CACHE_DIR +@@ -1540,12 +1547,41 @@ def _handle_remote_boot_params(orig_boot): + return new_boot + else: + raise SaltInvocationError( +- "Invalid boot parameters, (kernel, initrd) or/and (loader, nvram) must be both present" ++ "Invalid boot parameters,It has to follow this combination: [(kernel, initrd) or/and cmdline] or/and [(loader, nvram) or efi]" + ) + except Exception as err: # pylint: disable=broad-except + raise err + + ++def _handle_efi_param(boot, desc): ++ """ ++ Checks if boot parameter contains efi boolean value, if so, handles the firmware attribute. ++ :param boot: The boot parameters passed to the init or update functions. ++ :param desc: The XML description of that domain. ++ :return: A boolean value. ++ """ ++ efi_value = boot.get("efi", None) if boot else None ++ parent_tag = desc.find("os") ++ os_attrib = parent_tag.attrib ++ ++ # newly defined vm without running, loader tag might not be filled yet ++ if efi_value is False and os_attrib != {}: ++ parent_tag.attrib.pop("firmware", None) ++ return True ++ ++ # check the case that loader tag might be present. This happens after the vm ran ++ elif type(efi_value) == bool and os_attrib == {}: ++ if efi_value is True and parent_tag.find("loader") is None: ++ parent_tag.set("firmware", "efi") ++ if efi_value is False and parent_tag.find("loader") is not None: ++ parent_tag.remove(parent_tag.find("loader")) ++ parent_tag.remove(parent_tag.find("nvram")) ++ return True ++ elif type(efi_value) != bool: ++ raise SaltInvocationError("Invalid efi value") ++ return False ++ ++ + def init( + name, + cpu, +@@ -1566,6 +1602,7 @@ def init( + os_type=None, + arch=None, + boot=None, ++ boot_dev=None, + **kwargs + ): + """ +@@ -1635,7 +1672,8 @@ def init( + This is an optional parameter, all of the keys are optional within the dictionary. The structure of + the dictionary is documented in :ref:`init-boot-def`. If a remote path is provided to kernel or initrd, + salt will handle the downloading of the specified remote file and modify the XML accordingly. +- To boot VM with UEFI, specify loader and nvram path. ++ To boot VM with UEFI, specify loader and nvram path or specify 'efi': ``True`` if your libvirtd version ++ is >= 5.2.0 and QEMU >= 3.0.0. + + .. versionadded:: 3000 + +@@ -1649,6 +1687,12 @@ def init( + 'nvram': '/usr/share/OVMF/OVMF_VARS.ms.fd' + } + ++ :param boot_dev: ++ Space separated list of devices to boot from sorted by decreasing priority. ++ Values can be ``hd``, ``fd``, ``cdrom`` or ``network``. ++ ++ By default, the value will ``"hd"``. ++ + .. _init-boot-def: + + .. rubric:: Boot parameters definition +@@ -1674,6 +1718,11 @@ def init( + + .. versionadded:: sodium + ++ efi ++ A boolean value. ++ ++ .. versionadded:: sodium ++ + .. _init-nic-def: + + .. rubric:: Network Interfaces Definitions +@@ -1797,7 +1846,7 @@ def init( + + .. rubric:: Graphics Definition + +- The graphics dictionnary can have the following properties: ++ The graphics dictionary can have the following properties: + + type + Graphics type. The possible values are ``none``, ``'spice'``, ``'vnc'`` and other values +@@ -1858,6 +1907,8 @@ def init( + for x in y + } + ) ++ if len(hypervisors) == 0: ++ raise SaltInvocationError("No supported hypervisors were found") + virt_hypervisor = "kvm" if "kvm" in hypervisors else hypervisors[0] + + # esxi used to be a possible value for the hypervisor: map it to vmware since it's the same +@@ -1890,8 +1941,8 @@ def init( + else: + # assume libvirt manages disks for us + log.debug("Generating libvirt XML for %s", _disk) +- volume_name = "{0}/{1}".format(name, _disk["name"]) +- filename = "{0}.{1}".format(volume_name, _disk["format"]) ++ volume_name = "{}/{}".format(name, _disk["name"]) ++ filename = "{}.{}".format(volume_name, _disk["format"]) + vol_xml = _gen_vol_xml( + filename, _disk["size"], format=_disk["format"] + ) +@@ -1939,7 +1990,7 @@ def init( + else: + # Unknown hypervisor + raise SaltInvocationError( +- "Unsupported hypervisor when handling disk image: {0}".format( ++ "Unsupported hypervisor when handling disk image: {}".format( + virt_hypervisor + ) + ) +@@ -1965,8 +2016,10 @@ def init( + arch, + graphics, + boot, ++ boot_dev, + **kwargs + ) ++ log.debug("New virtual machine definition: %s", vm_xml) + conn.defineXML(vm_xml) + except libvirt.libvirtError as err: + conn.close() +@@ -2192,6 +2245,7 @@ def update( + live=True, + boot=None, + test=False, ++ boot_dev=None, + **kwargs + ): + """ +@@ -2234,11 +2288,28 @@ def update( + + Refer to :ref:`init-boot-def` for the complete boot parameter description. + +- To update any boot parameters, specify the new path for each. To remove any boot parameters, +- pass a None object, for instance: 'kernel': ``None``. ++ To update any boot parameters, specify the new path for each. To remove any boot parameters, pass ``None`` object, ++ for instance: 'kernel': ``None``. To switch back to BIOS boot, specify ('loader': ``None`` and 'nvram': ``None``) ++ or 'efi': ``False``. Please note that ``None`` is mapped to ``null`` in sls file, pass ``null`` in sls file instead. ++ ++ SLS file Example: ++ ++ .. code-block:: yaml ++ ++ - boot: ++ loader: null ++ nvram: null + + .. versionadded:: 3000 + ++ :param boot_dev: ++ Space separated list of devices to boot from sorted by decreasing priority. ++ Values can be ``hd``, ``fd``, ``cdrom`` or ``network``. ++ ++ By default, the value will ``"hd"``. ++ ++ .. versionadded:: Magnesium ++ + :param test: run in dry-run mode if set to True + + .. versionadded:: sodium +@@ -2286,6 +2357,8 @@ def update( + + if boot is not None: + boot = _handle_remote_boot_params(boot) ++ if boot.get("efi", None) is not None: ++ need_update = _handle_efi_param(boot, desc) + + new_desc = ElementTree.fromstring( + _gen_xml( +@@ -2307,76 +2380,58 @@ def update( + # Update the cpu + cpu_node = desc.find("vcpu") + if cpu and int(cpu_node.text) != cpu: +- cpu_node.text = six.text_type(cpu) +- cpu_node.set("current", six.text_type(cpu)) ++ cpu_node.text = str(cpu) ++ cpu_node.set("current", str(cpu)) + need_update = True + +- # Update the kernel boot parameters +- boot_tags = ["kernel", "initrd", "cmdline", "loader", "nvram"] +- parent_tag = desc.find("os") +- +- # We need to search for each possible subelement, and update it. +- for tag in boot_tags: +- # The Existing Tag... +- found_tag = parent_tag.find(tag) +- +- # The new value +- boot_tag_value = boot.get(tag, None) if boot else None +- +- # Existing tag is found and values don't match +- if found_tag is not None and found_tag.text != boot_tag_value: +- +- # If the existing tag is found, but the new value is None +- # remove it. If the existing tag is found, and the new value +- # doesn't match update it. In either case, mark for update. +- if boot_tag_value is None and boot is not None and parent_tag is not None: +- parent_tag.remove(found_tag) +- else: +- found_tag.text = boot_tag_value ++ def _set_loader(node, value): ++ salt.utils.xmlutil.set_node_text(node, value) ++ if value is not None: ++ node.set("readonly", "yes") ++ node.set("type", "pflash") + +- # If the existing tag is loader or nvram, we need to update the corresponding attribute +- if found_tag.tag == "loader" and boot_tag_value is not None: +- found_tag.set("readonly", "yes") +- found_tag.set("type", "pflash") ++ def _set_nvram(node, value): ++ node.set("template", value) + +- if found_tag.tag == "nvram" and boot_tag_value is not None: +- found_tag.set("template", found_tag.text) +- found_tag.text = None ++ def _set_with_mib_unit(node, value): ++ node.text = str(value) ++ node.set("unit", "MiB") + +- need_update = True +- +- # Existing tag is not found, but value is not None +- elif found_tag is None and boot_tag_value is not None: +- +- # Need to check for parent tag, and add it if it does not exist. +- # Add a subelement and set the value to the new value, and then +- # mark for update. +- if parent_tag is not None: +- child_tag = ElementTree.SubElement(parent_tag, tag) +- else: +- new_parent_tag = ElementTree.Element("os") +- child_tag = ElementTree.SubElement(new_parent_tag, tag) +- +- child_tag.text = boot_tag_value +- +- # If the newly created tag is loader or nvram, we need to update the corresponding attribute +- if child_tag.tag == "loader": +- child_tag.set("readonly", "yes") +- child_tag.set("type", "pflash") +- +- if child_tag.tag == "nvram": +- child_tag.set("template", child_tag.text) +- child_tag.text = None +- +- need_update = True ++ # Update the kernel boot parameters ++ params_mapping = [ ++ {"path": "boot:kernel", "xpath": "os/kernel"}, ++ {"path": "boot:initrd", "xpath": "os/initrd"}, ++ {"path": "boot:cmdline", "xpath": "os/cmdline"}, ++ {"path": "boot:loader", "xpath": "os/loader", "set": _set_loader}, ++ {"path": "boot:nvram", "xpath": "os/nvram", "set": _set_nvram}, ++ # Update the memory, note that libvirt outputs all memory sizes in KiB ++ { ++ "path": "mem", ++ "xpath": "memory", ++ "get": lambda n: int(n.text) / 1024, ++ "set": _set_with_mib_unit, ++ }, ++ { ++ "path": "mem", ++ "xpath": "currentMemory", ++ "get": lambda n: int(n.text) / 1024, ++ "set": _set_with_mib_unit, ++ }, ++ { ++ "path": "boot_dev:{dev}", ++ "xpath": "os/boot[$dev]", ++ "get": lambda n: n.get("dev"), ++ "set": lambda n, v: n.set("dev", v), ++ "del": salt.utils.xmlutil.del_attribute("dev"), ++ }, ++ ] + +- # Update the memory, note that libvirt outputs all memory sizes in KiB +- for mem_node_name in ["memory", "currentMemory"]: +- mem_node = desc.find(mem_node_name) +- if mem and int(mem_node.text) != mem * 1024: +- mem_node.text = six.text_type(mem) +- mem_node.set("unit", "MiB") +- need_update = True ++ data = {k: v for k, v in locals().items() if bool(v)} ++ if boot_dev: ++ data["boot_dev"] = {i + 1: dev for i, dev in enumerate(boot_dev.split())} ++ need_update = need_update or salt.utils.xmlutil.change_xml( ++ desc, data, params_mapping ++ ) + + # Update the XML definition with the new disks and diff changes + devices_node = desc.find("devices") +@@ -2395,8 +2450,8 @@ def update( + if func_locals.get(param, None) is not None + ]: + old = devices_node.findall(dev_type) +- new = new_desc.findall("devices/{0}".format(dev_type)) +- changes[dev_type] = globals()["_diff_{0}_lists".format(dev_type)](old, new) ++ new = new_desc.findall("devices/{}".format(dev_type)) ++ changes[dev_type] = globals()["_diff_{}_lists".format(dev_type)](old, new) + if changes[dev_type]["deleted"] or changes[dev_type]["new"]: + for item in old: + devices_node.remove(item) +@@ -2423,9 +2478,9 @@ def update( + _disk_volume_create(conn, all_disks[idx]) + + if not test: +- conn.defineXML( +- salt.utils.stringutils.to_str(ElementTree.tostring(desc)) +- ) ++ xml_desc = ElementTree.tostring(desc) ++ log.debug("Update virtual machine definition: %s", xml_desc) ++ conn.defineXML(salt.utils.stringutils.to_str(xml_desc)) + status["definition"] = True + except libvirt.libvirtError as err: + conn.close() +@@ -2554,7 +2609,7 @@ def update( + except libvirt.libvirtError as err: + if "errors" not in status: + status["errors"] = [] +- status["errors"].append(six.text_type(err)) ++ status["errors"].append(str(err)) + + conn.close() + return status +@@ -2768,7 +2823,7 @@ def _node_info(conn): + info = { + "cpucores": raw[6], + "cpumhz": raw[3], +- "cpumodel": six.text_type(raw[0]), ++ "cpumodel": str(raw[0]), + "cpus": raw[2], + "cputhreads": raw[7], + "numanodes": raw[4], +@@ -3207,24 +3262,21 @@ def get_profiles(hypervisor=None, **kwargs): + for x in y + } + ) +- default_hypervisor = "kvm" if "kvm" in hypervisors else hypervisors[0] ++ if len(hypervisors) == 0: ++ raise SaltInvocationError("No supported hypervisors were found") + + if not hypervisor: +- hypervisor = default_hypervisor ++ hypervisor = "kvm" if "kvm" in hypervisors else hypervisors[0] + virtconf = __salt__["config.get"]("virt", {}) + for typ in ["disk", "nic"]: +- _func = getattr(sys.modules[__name__], "_{0}_profile".format(typ)) ++ _func = getattr(sys.modules[__name__], "_{}_profile".format(typ)) + ret[typ] = { +- "default": _func( +- "default", hypervisor if hypervisor else default_hypervisor +- ) ++ "default": _func("default", hypervisor) + } + if typ in virtconf: + ret.setdefault(typ, {}) + for prf in virtconf[typ]: +- ret[typ][prf] = _func( +- prf, hypervisor if hypervisor else default_hypervisor +- ) ++ ret[typ][prf] = _func(prf, hypervisor) + return ret + + +@@ -3506,7 +3558,7 @@ def create_xml_path(path, **kwargs): + return create_xml_str( + salt.utils.stringutils.to_unicode(fp_.read()), **kwargs + ) +- except (OSError, IOError): ++ except OSError: + return False + + +@@ -3564,7 +3616,7 @@ def define_xml_path(path, **kwargs): + return define_xml_str( + salt.utils.stringutils.to_unicode(fp_.read()), **kwargs + ) +- except (OSError, IOError): ++ except OSError: + return False + + +@@ -3576,7 +3628,7 @@ def _define_vol_xml_str(conn, xml, pool=None): # pylint: disable=redefined-oute + poolname = ( + pool if pool else __salt__["config.get"]("virt:storagepool", default_pool) + ) +- pool = conn.storagePoolLookupByName(six.text_type(poolname)) ++ pool = conn.storagePoolLookupByName(str(poolname)) + ret = pool.createXML(xml, 0) is not None + return ret + +@@ -3660,7 +3712,7 @@ def define_vol_xml_path(path, pool=None, **kwargs): + return define_vol_xml_str( + salt.utils.stringutils.to_unicode(fp_.read()), pool=pool, **kwargs + ) +- except (OSError, IOError): ++ except OSError: + return False + + +@@ -3777,7 +3829,7 @@ def seed_non_shared_migrate(disks, force=False): + + salt '*' virt.seed_non_shared_migrate + """ +- for _, data in six.iteritems(disks): ++ for _, data in disks.items(): + fn_ = data["file"] + form = data["file format"] + size = data["virtual size"].split()[1][1:] +@@ -3921,14 +3973,14 @@ def purge(vm_, dirs=False, removables=False, **kwargs): + # TODO create solution for 'dataset is busy' + time.sleep(3) + fs_name = disks[disk]["file"][len("/dev/zvol/") :] +- log.info("Destroying VM ZFS volume {0}".format(fs_name)) ++ log.info("Destroying VM ZFS volume {}".format(fs_name)) + __salt__["zfs.destroy"](name=fs_name, force=True) + elif os.path.exists(disks[disk]["file"]): + os.remove(disks[disk]["file"]) + directories.add(os.path.dirname(disks[disk]["file"])) + else: + # We may have a volume to delete here +- matcher = re.match("^(?P[^/]+)/(?P.*)$", disks[disk]["file"],) ++ matcher = re.match("^(?P[^/]+)/(?P.*)$", disks[disk]["file"]) + if matcher: + pool_name = matcher.group("pool") + pool = None +@@ -3975,7 +4027,7 @@ def _is_kvm_hyper(): + with salt.utils.files.fopen("/proc/modules") as fp_: + if "kvm_" not in salt.utils.stringutils.to_unicode(fp_.read()): + return False +- except IOError: ++ except OSError: + # No /proc/modules? Are we on Windows? Or Solaris? + return False + return "libvirtd" in __salt__["cmd.run"](__grains__["ps"]) +@@ -3995,7 +4047,7 @@ def _is_xen_hyper(): + with salt.utils.files.fopen("/proc/modules") as fp_: + if "xen_" not in salt.utils.stringutils.to_unicode(fp_.read()): + return False +- except (OSError, IOError): ++ except OSError: + # No /proc/modules? Are we on Windows? Or Solaris? + return False + return "libvirtd" in __salt__["cmd.run"](__grains__["ps"]) +@@ -4110,7 +4162,7 @@ def vm_cputime(vm_=None, **kwargs): + cputime_percent = (1.0e-7 * cputime / host_cpus) / vcpus + return { + "cputime": int(raw[4]), +- "cputime_percent": int("{0:.0f}".format(cputime_percent)), ++ "cputime_percent": int("{:.0f}".format(cputime_percent)), + } + + info = {} +@@ -4180,7 +4232,7 @@ def vm_netstats(vm_=None, **kwargs): + "tx_errs": 0, + "tx_drop": 0, + } +- for attrs in six.itervalues(nics): ++ for attrs in nics.values(): + if "target" in attrs: + dev = attrs["target"] + stats = dom.interfaceStats(dev) +@@ -4508,7 +4560,7 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + conn.close() + raise CommandExecutionError( + snapshot +- and 'Snapshot "{0}" not found'.format(vm_snapshot) ++ and 'Snapshot "{}" not found'.format(vm_snapshot) + or "No more previous snapshots available" + ) + elif snap.isCurrent(): +@@ -5102,10 +5154,10 @@ def cpu_baseline(full=False, migratable=False, out="libvirt", **kwargs): + ] + + if not cpu_specs: +- raise ValueError("Model {0} not found in CPU map".format(cpu_model)) ++ raise ValueError("Model {} not found in CPU map".format(cpu_model)) + elif len(cpu_specs) > 1: + raise ValueError( +- "Multiple models {0} found in CPU map".format(cpu_model) ++ "Multiple models {} found in CPU map".format(cpu_model) + ) + + cpu_specs = cpu_specs[0] +@@ -5126,7 +5178,7 @@ def cpu_baseline(full=False, migratable=False, out="libvirt", **kwargs): + "vendor": cpu.find("vendor").text, + "features": [feature.get("name") for feature in cpu.findall("feature")], + } +- return cpu.toxml() ++ return ElementTree.tostring(cpu) + + + def network_define(name, bridge, forward, ipv4_config=None, ipv6_config=None, **kwargs): +@@ -5250,7 +5302,7 @@ def list_networks(**kwargs): + + def network_info(name=None, **kwargs): + """ +- Return informations on a virtual network provided its name. ++ Return information on a virtual network provided its name. + + :param name: virtual network name + :param connection: libvirt connection URI, overriding defaults +@@ -5446,20 +5498,20 @@ def _parse_pools_caps(doc): + for option_kind in ["pool", "vol"]: + options = {} + default_format_node = pool.find( +- "{0}Options/defaultFormat".format(option_kind) ++ "{}Options/defaultFormat".format(option_kind) + ) + if default_format_node is not None: + options["default_format"] = default_format_node.get("type") + options_enums = { + enum.get("name"): [value.text for value in enum.findall("value")] +- for enum in pool.findall("{0}Options/enum".format(option_kind)) ++ for enum in pool.findall("{}Options/enum".format(option_kind)) + } + if options_enums: + options.update(options_enums) + if options: + if "options" not in pool_caps: + pool_caps["options"] = {} +- kind = option_kind if option_kind is not "vol" else "volume" ++ kind = option_kind if option_kind != "vol" else "volume" + pool_caps["options"][kind] = options + return pool_caps + +@@ -5695,7 +5747,7 @@ def pool_define( + keys. The path is the qualified name for iSCSI devices. + + Report to `this libvirt page `_ +- for more informations on the use of ``part_separator`` ++ for more information on the use of ``part_separator`` + :param source_dir: + Path to the source directory for pools of type ``dir``, ``netfs`` or ``gluster``. + (Default: ``None``) +@@ -5847,15 +5899,19 @@ def _pool_set_secret( + if secret_type: + # Get the previously defined secret if any + secret = None +- if usage: +- usage_type = ( +- libvirt.VIR_SECRET_USAGE_TYPE_CEPH +- if secret_type == "ceph" +- else libvirt.VIR_SECRET_USAGE_TYPE_ISCSI +- ) +- secret = conn.secretLookupByUsage(usage_type, usage) +- elif uuid: +- secret = conn.secretLookupByUUIDString(uuid) ++ try: ++ if usage: ++ usage_type = ( ++ libvirt.VIR_SECRET_USAGE_TYPE_CEPH ++ if secret_type == "ceph" ++ else libvirt.VIR_SECRET_USAGE_TYPE_ISCSI ++ ) ++ secret = conn.secretLookupByUsage(usage_type, usage) ++ elif uuid: ++ secret = conn.secretLookupByUUIDString(uuid) ++ except libvirt.libvirtError as err: ++ # For some reason the secret has been removed. Don't fail since we'll recreate it ++ log.info("Secret not found: %s", err.get_error_message()) + + # Create secret if needed + if not secret: +@@ -5918,7 +5974,7 @@ def pool_update( + keys. The path is the qualified name for iSCSI devices. + + Report to `this libvirt page `_ +- for more informations on the use of ``part_separator`` ++ for more information on the use of ``part_separator`` + :param source_dir: + Path to the source directory for pools of type ``dir``, ``netfs`` or ``gluster``. + (Default: ``None``) +@@ -6107,7 +6163,7 @@ def list_pools(**kwargs): + + def pool_info(name=None, **kwargs): + """ +- Return informations on a storage pool provided its name. ++ Return information on a storage pool provided its name. + + :param name: libvirt storage pool name + :param connection: libvirt connection URI, overriding defaults +@@ -6283,6 +6339,22 @@ def pool_undefine(name, **kwargs): + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) ++ desc = ElementTree.fromstring(pool.XMLDesc()) ++ ++ # Is there a secret that we generated and would need to be removed? ++ # Don't remove the other secrets ++ auth_node = desc.find("source/auth") ++ if auth_node is not None: ++ auth_types = { ++ "ceph": libvirt.VIR_SECRET_USAGE_TYPE_CEPH, ++ "iscsi": libvirt.VIR_SECRET_USAGE_TYPE_ISCSI, ++ } ++ secret_type = auth_types[auth_node.get("type")] ++ secret_usage = auth_node.find("secret").get("usage") ++ if secret_type and "pool_{}".format(name) == secret_usage: ++ secret = conn.secretLookupByUsage(secret_type, secret_usage) ++ secret.undefine() ++ + return not bool(pool.undefine()) + finally: + conn.close() +@@ -6308,22 +6380,6 @@ def pool_delete(name, **kwargs): + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +- desc = ElementTree.fromstring(pool.XMLDesc()) +- +- # Is there a secret that we generated and would need to be removed? +- # Don't remove the other secrets +- auth_node = desc.find("source/auth") +- if auth_node is not None: +- auth_types = { +- "ceph": libvirt.VIR_SECRET_USAGE_TYPE_CEPH, +- "iscsi": libvirt.VIR_SECRET_USAGE_TYPE_ISCSI, +- } +- secret_type = auth_types[auth_node.get("type")] +- secret_usage = auth_node.find("secret").get("usage") +- if secret_type and "pool_{}".format(name) == secret_usage: +- secret = conn.secretLookupByUsage(secret_type, secret_usage) +- secret.undefine() +- + return not bool(pool.delete(libvirt.VIR_STORAGE_POOL_DELETE_NORMAL)) + finally: + conn.close() +@@ -6768,7 +6824,7 @@ def _volume_upload(conn, pool, volume, file, offset=0, length=0, sparse=False): + stream.abort() + if ret: + raise CommandExecutionError( +- "Failed to close file: {0}".format(err.strerror) ++ "Failed to close file: {}".format(err.strerror) + ) + if stream: + try: +@@ -6776,7 +6832,7 @@ def _volume_upload(conn, pool, volume, file, offset=0, length=0, sparse=False): + except libvirt.libvirtError as err: + if ret: + raise CommandExecutionError( +- "Failed to finish stream: {0}".format(err.get_error_message()) ++ "Failed to finish stream: {}".format(err.get_error_message()) + ) + return ret + +diff --git a/salt/states/virt.py b/salt/states/virt.py +index fdef002293..3d99fd53c8 100644 +--- a/salt/states/virt.py ++++ b/salt/states/virt.py +@@ -1,4 +1,3 @@ +-# -*- coding: utf-8 -*- + """ + Manage virt + =========== +@@ -13,9 +12,9 @@ for the generation and signing of certificates for systems running libvirt: + """ + + # Import Python libs +-from __future__ import absolute_import, print_function, unicode_literals + + import fnmatch ++import logging + import os + + # Import Salt libs +@@ -25,9 +24,6 @@ import salt.utils.stringutils + import salt.utils.versions + from salt.exceptions import CommandExecutionError, SaltInvocationError + +-# Import 3rd-party libs +-from salt.ext import six +- + try: + import libvirt # pylint: disable=import-error + +@@ -38,6 +34,8 @@ except ImportError: + + __virtualname__ = "virt" + ++log = logging.getLogger(__name__) ++ + + def __virtual__(): + """ +@@ -99,8 +97,8 @@ def keys(name, basepath="/etc/pki", **kwargs): + # rename them to something hopefully unique to avoid + # overriding anything existing + pillar_kwargs = {} +- for key, value in six.iteritems(kwargs): +- pillar_kwargs["ext_pillar_virt.{0}".format(key)] = value ++ for key, value in kwargs.items(): ++ pillar_kwargs["ext_pillar_virt.{}".format(key)] = value + + pillar = __salt__["pillar.ext"]({"libvirt": "_"}, pillar_kwargs) + paths = { +@@ -112,7 +110,7 @@ def keys(name, basepath="/etc/pki", **kwargs): + } + + for key in paths: +- p_key = "libvirt.{0}.pem".format(key) ++ p_key = "libvirt.{}.pem".format(key) + if p_key not in pillar: + continue + if not os.path.exists(os.path.dirname(paths[key])): +@@ -134,7 +132,7 @@ def keys(name, basepath="/etc/pki", **kwargs): + for key in ret["changes"]: + with salt.utils.files.fopen(paths[key], "w+") as fp_: + fp_.write( +- salt.utils.stringutils.to_str(pillar["libvirt.{0}.pem".format(key)]) ++ salt.utils.stringutils.to_str(pillar["libvirt.{}.pem".format(key)]) + ) + + ret["comment"] = "Updated libvirt certs and keys" +@@ -176,7 +174,7 @@ def _virt_call( + domain_state = __salt__["virt.vm_state"](targeted_domain) + action_needed = domain_state.get(targeted_domain) != state + if action_needed: +- response = __salt__["virt.{0}".format(function)]( ++ response = __salt__["virt.{}".format(function)]( + targeted_domain, + connection=connection, + username=username, +@@ -189,9 +187,7 @@ def _virt_call( + else: + noaction_domains.append(targeted_domain) + except libvirt.libvirtError as err: +- ignored_domains.append( +- {"domain": targeted_domain, "issue": six.text_type(err)} +- ) ++ ignored_domains.append({"domain": targeted_domain, "issue": str(err)}) + if not changed_domains: + ret["result"] = not ignored_domains and bool(targeted_domains) + ret["comment"] = "No changes had happened" +@@ -292,6 +288,7 @@ def defined( + arch=None, + boot=None, + update=True, ++ boot_dev=None, + ): + """ + Starts an existing guest, or defines and starts a new VM with specified arguments. +@@ -352,6 +349,14 @@ def defined( + + .. deprecated:: sodium + ++ :param boot_dev: ++ Space separated list of devices to boot from sorted by decreasing priority. ++ Values can be ``hd``, ``fd``, ``cdrom`` or ``network``. ++ ++ By default, the value will ``"hd"``. ++ ++ .. versionadded:: Magnesium ++ + .. rubric:: Example States + + Make sure a virtual machine called ``domain_name`` is defined: +@@ -362,6 +367,7 @@ def defined( + virt.defined: + - cpu: 2 + - mem: 2048 ++ - boot_dev: network hd + - disk_profile: prod + - disks: + - name: system +@@ -414,17 +420,18 @@ def defined( + password=password, + boot=boot, + test=__opts__["test"], ++ boot_dev=boot_dev, + ) + ret["changes"][name] = status + if not status.get("definition"): +- ret["comment"] = "Domain {0} unchanged".format(name) ++ ret["comment"] = "Domain {} unchanged".format(name) + ret["result"] = True + elif status.get("errors"): + ret[ + "comment" +- ] = "Domain {0} updated with live update(s) failures".format(name) ++ ] = "Domain {} updated with live update(s) failures".format(name) + else: +- ret["comment"] = "Domain {0} updated".format(name) ++ ret["comment"] = "Domain {} updated".format(name) + else: + if not __opts__["test"]: + __salt__["virt.init"]( +@@ -448,12 +455,13 @@ def defined( + password=password, + boot=boot, + start=False, ++ boot_dev=boot_dev, + ) + ret["changes"][name] = {"definition": True} +- ret["comment"] = "Domain {0} defined".format(name) ++ ret["comment"] = "Domain {} defined".format(name) + except libvirt.libvirtError as err: + # Something bad happened when defining / updating the VM, report it +- ret["comment"] = six.text_type(err) ++ ret["comment"] = str(err) + ret["result"] = False + + return ret +@@ -480,6 +488,7 @@ def running( + os_type=None, + arch=None, + boot=None, ++ boot_dev=None, + ): + """ + Starts an existing guest, or defines and starts a new VM with specified arguments. +@@ -591,6 +600,14 @@ def running( + + .. versionadded:: 3000 + ++ :param boot_dev: ++ Space separated list of devices to boot from sorted by decreasing priority. ++ Values can be ``hd``, ``fd``, ``cdrom`` or ``network``. ++ ++ By default, the value will ``"hd"``. ++ ++ .. versionadded:: Magnesium ++ + .. rubric:: Example States + + Make sure an already-defined virtual machine called ``domain_name`` is running: +@@ -609,6 +626,7 @@ def running( + - cpu: 2 + - mem: 2048 + - disk_profile: prod ++ - boot_dev: network hd + - disks: + - name: system + size: 8192 +@@ -657,6 +675,7 @@ def running( + arch=arch, + boot=boot, + update=update, ++ boot_dev=boot_dev, + connection=connection, + username=username, + password=password, +@@ -681,11 +700,11 @@ def running( + ret["comment"] = comment + ret["changes"][name]["started"] = True + elif not changed: +- ret["comment"] = "Domain {0} exists and is running".format(name) ++ ret["comment"] = "Domain {} exists and is running".format(name) + + except libvirt.libvirtError as err: + # Something bad happened when starting / updating the VM, report it +- ret["comment"] = six.text_type(err) ++ ret["comment"] = str(err) + ret["result"] = False + + return ret +@@ -830,7 +849,7 @@ def reverted( + try: + domains = fnmatch.filter(__salt__["virt.list_domains"](), name) + if not domains: +- ret["comment"] = 'No domains found for criteria "{0}"'.format(name) ++ ret["comment"] = 'No domains found for criteria "{}"'.format(name) + else: + ignored_domains = list() + if len(domains) > 1: +@@ -848,9 +867,7 @@ def reverted( + } + except CommandExecutionError as err: + if len(domains) > 1: +- ignored_domains.append( +- {"domain": domain, "issue": six.text_type(err)} +- ) ++ ignored_domains.append({"domain": domain, "issue": str(err)}) + if len(domains) > 1: + if result: + ret["changes"]["reverted"].append(result) +@@ -860,7 +877,7 @@ def reverted( + + ret["result"] = len(domains) != len(ignored_domains) + if ret["result"]: +- ret["comment"] = "Domain{0} has been reverted".format( ++ ret["comment"] = "Domain{} has been reverted".format( + len(domains) > 1 and "s" or "" + ) + if ignored_domains: +@@ -868,9 +885,9 @@ def reverted( + if not ret["changes"]["reverted"]: + ret["changes"].pop("reverted") + except libvirt.libvirtError as err: +- ret["comment"] = six.text_type(err) ++ ret["comment"] = str(err) + except CommandExecutionError as err: +- ret["comment"] = six.text_type(err) ++ ret["comment"] = str(err) + + return ret + +@@ -955,7 +972,7 @@ def network_defined( + name, connection=connection, username=username, password=password + ) + if info and info[name]: +- ret["comment"] = "Network {0} exists".format(name) ++ ret["comment"] = "Network {} exists".format(name) + ret["result"] = True + else: + if not __opts__["test"]: +@@ -974,7 +991,7 @@ def network_defined( + password=password, + ) + ret["changes"][name] = "Network defined" +- ret["comment"] = "Network {0} defined".format(name) ++ ret["comment"] = "Network {} defined".format(name) + except libvirt.libvirtError as err: + ret["result"] = False + ret["comment"] = err.get_error_message() +@@ -1108,6 +1125,10 @@ def network_running( + return ret + + ++# Some of the libvirt storage drivers do not support the build action ++BUILDABLE_POOL_TYPES = {"disk", "fs", "netfs", "dir", "logical", "vstorage", "zfs"} ++ ++ + def pool_defined( + name, + ptype=None, +@@ -1222,25 +1243,35 @@ def pool_defined( + + action = "" + if info[name]["state"] != "running": +- if not __opts__["test"]: +- __salt__["virt.pool_build"]( +- name, +- connection=connection, +- username=username, +- password=password, +- ) +- action = ", built" ++ if ptype in BUILDABLE_POOL_TYPES: ++ if not __opts__["test"]: ++ # Storage pools build like disk or logical will fail if the disk or LV group ++ # was already existing. Since we can't easily figure that out, just log the ++ # possible libvirt error. ++ try: ++ __salt__["virt.pool_build"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ except libvirt.libvirtError as err: ++ log.warning( ++ "Failed to build libvirt storage pool: %s", ++ err.get_error_message(), ++ ) ++ action = ", built" + + action = ( + "{}, autostart flag changed".format(action) + if needs_autostart + else action + ) +- ret["changes"][name] = "Pool updated{0}".format(action) +- ret["comment"] = "Pool {0} updated{1}".format(name, action) ++ ret["changes"][name] = "Pool updated{}".format(action) ++ ret["comment"] = "Pool {} updated{}".format(name, action) + + else: +- ret["comment"] = "Pool {0} unchanged".format(name) ++ ret["comment"] = "Pool {} unchanged".format(name) + ret["result"] = True + else: + needs_autostart = autostart +@@ -1265,15 +1296,28 @@ def pool_defined( + password=password, + ) + +- __salt__["virt.pool_build"]( +- name, connection=connection, username=username, password=password +- ) ++ if ptype in BUILDABLE_POOL_TYPES: ++ # Storage pools build like disk or logical will fail if the disk or LV group ++ # was already existing. Since we can't easily figure that out, just log the ++ # possible libvirt error. ++ try: ++ __salt__["virt.pool_build"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ except libvirt.libvirtError as err: ++ log.warning( ++ "Failed to build libvirt storage pool: %s", ++ err.get_error_message(), ++ ) + if needs_autostart: + ret["changes"][name] = "Pool defined, marked for autostart" +- ret["comment"] = "Pool {0} defined, marked for autostart".format(name) ++ ret["comment"] = "Pool {} defined, marked for autostart".format(name) + else: + ret["changes"][name] = "Pool defined" +- ret["comment"] = "Pool {0} defined".format(name) ++ ret["comment"] = "Pool {} defined".format(name) + + if needs_autostart: + if not __opts__["test"]: +@@ -1374,7 +1418,7 @@ def pool_running( + is_running = info.get(name, {}).get("state", "stopped") == "running" + if is_running: + if updated: +- action = "built, restarted" ++ action = "restarted" + if not __opts__["test"]: + __salt__["virt.pool_stop"]( + name, +@@ -1382,13 +1426,16 @@ def pool_running( + username=username, + password=password, + ) +- if not __opts__["test"]: +- __salt__["virt.pool_build"]( +- name, +- connection=connection, +- username=username, +- password=password, +- ) ++ # if the disk or LV group is already existing build will fail (issue #56454) ++ if ptype in BUILDABLE_POOL_TYPES - {"disk", "logical"}: ++ if not __opts__["test"]: ++ __salt__["virt.pool_build"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ action = "built, {}".format(action) + else: + action = "already running" + result = True +@@ -1402,16 +1449,16 @@ def pool_running( + password=password, + ) + +- comment = "Pool {0}".format(name) ++ comment = "Pool {}".format(name) + change = "Pool" + if name in ret["changes"]: +- comment = "{0},".format(ret["comment"]) +- change = "{0},".format(ret["changes"][name]) ++ comment = "{},".format(ret["comment"]) ++ change = "{},".format(ret["changes"][name]) + + if action != "already running": +- ret["changes"][name] = "{0} {1}".format(change, action) ++ ret["changes"][name] = "{} {}".format(change, action) + +- ret["comment"] = "{0} {1}".format(comment, action) ++ ret["comment"] = "{} {}".format(comment, action) + ret["result"] = result + + except libvirt.libvirtError as err: +@@ -1539,15 +1586,13 @@ def pool_deleted(name, purge=False, connection=None, username=None, password=Non + ret["result"] = None + + if unsupported: +- ret[ +- "comment" +- ] = 'Unsupported actions for pool of type "{0}": {1}'.format( ++ ret["comment"] = 'Unsupported actions for pool of type "{}": {}'.format( + info[name]["type"], ", ".join(unsupported) + ) + else: +- ret["comment"] = "Storage pool could not be found: {0}".format(name) ++ ret["comment"] = "Storage pool could not be found: {}".format(name) + except libvirt.libvirtError as err: +- ret["comment"] = "Failed deleting pool: {0}".format(err.get_error_message()) ++ ret["comment"] = "Failed deleting pool: {}".format(err.get_error_message()) + ret["result"] = False + + return ret +diff --git a/salt/templates/virt/libvirt_domain.jinja b/salt/templates/virt/libvirt_domain.jinja +index aac6283eb0..04a61ffa78 100644 +--- a/salt/templates/virt/libvirt_domain.jinja ++++ b/salt/templates/virt/libvirt_domain.jinja +@@ -3,7 +3,7 @@ + {{ cpu }} + {{ mem }} + {{ mem }} +- ++ + {{ os_type }} + {% if boot %} + {% if 'kernel' in boot %} +diff --git a/salt/utils/data.py b/salt/utils/data.py +index 8f84c2ea42..5a7acc9e7c 100644 +--- a/salt/utils/data.py ++++ b/salt/utils/data.py +@@ -1,22 +1,15 @@ +-# -*- coding: utf-8 -*- +-''' ++""" + Functions for manipulating, inspecting, or otherwise working with data types + and data structures. +-''' ++""" + +-from __future__ import absolute_import, print_function, unicode_literals + + # Import Python libs + import copy + import fnmatch ++import functools + import logging + import re +-import functools +- +-try: +- from collections.abc import Mapping, MutableMapping, Sequence +-except ImportError: +- from collections import Mapping, MutableMapping, Sequence + + # Import Salt libs + import salt.utils.dictupdate +@@ -24,13 +17,22 @@ import salt.utils.stringutils + import salt.utils.yaml + from salt.defaults import DEFAULT_TARGET_DELIM + from salt.exceptions import SaltException +-from salt.utils.decorators.jinja import jinja_filter +-from salt.utils.odict import OrderedDict ++from salt.ext import six + + # Import 3rd-party libs +-from salt.ext.six.moves import zip # pylint: disable=redefined-builtin +-from salt.ext import six + from salt.ext.six.moves import range # pylint: disable=redefined-builtin ++from salt.ext.six.moves import zip # pylint: disable=redefined-builtin ++from salt.utils.decorators.jinja import jinja_filter ++from salt.utils.odict import OrderedDict ++ ++try: ++ from collections.abc import Mapping, MutableMapping, Sequence ++except ImportError: ++ # pylint: disable=no-name-in-module ++ from collections import Mapping, MutableMapping, Sequence ++ ++ # pylint: enable=no-name-in-module ++ + + try: + import jmespath +@@ -41,15 +43,16 @@ log = logging.getLogger(__name__) + + + class CaseInsensitiveDict(MutableMapping): +- ''' ++ """ + Inspired by requests' case-insensitive dict implementation, but works with + non-string keys as well. +- ''' ++ """ ++ + def __init__(self, init=None, **kwargs): +- ''' ++ """ + Force internal dict to be ordered to ensure a consistent iteration + order, irrespective of case. +- ''' ++ """ + self._data = OrderedDict() + self.update(init or {}, **kwargs) + +@@ -67,7 +70,7 @@ class CaseInsensitiveDict(MutableMapping): + return self._data[to_lowercase(key)][1] + + def __iter__(self): +- return (item[0] for item in six.itervalues(self._data)) ++ return (item[0] for item in self._data.values()) + + def __eq__(self, rval): + if not isinstance(rval, Mapping): +@@ -76,28 +79,28 @@ class CaseInsensitiveDict(MutableMapping): + return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower()) + + def __repr__(self): +- return repr(dict(six.iteritems(self))) ++ return repr(dict(self.items())) + + def items_lower(self): +- ''' ++ """ + Returns a generator iterating over keys and values, with the keys all + being lowercase. +- ''' +- return ((key, val[1]) for key, val in six.iteritems(self._data)) ++ """ ++ return ((key, val[1]) for key, val in self._data.items()) + + def copy(self): +- ''' ++ """ + Returns a copy of the object +- ''' +- return CaseInsensitiveDict(six.iteritems(self._data)) ++ """ ++ return CaseInsensitiveDict(self._data.items()) + + + def __change_case(data, attr, preserve_dict_class=False): +- ''' ++ """ + Calls data.attr() if data has an attribute/method called attr. + Processes data recursively if data is a Mapping or Sequence. + For Mapping, processes both keys and values. +- ''' ++ """ + try: + return getattr(data, attr)() + except AttributeError: +@@ -107,73 +110,120 @@ def __change_case(data, attr, preserve_dict_class=False): + + if isinstance(data, Mapping): + return (data_type if preserve_dict_class else dict)( +- (__change_case(key, attr, preserve_dict_class), +- __change_case(val, attr, preserve_dict_class)) +- for key, val in six.iteritems(data) ++ ( ++ __change_case(key, attr, preserve_dict_class), ++ __change_case(val, attr, preserve_dict_class), ++ ) ++ for key, val in data.items() + ) + if isinstance(data, Sequence): + return data_type( +- __change_case(item, attr, preserve_dict_class) for item in data) ++ __change_case(item, attr, preserve_dict_class) for item in data ++ ) + return data + + + def to_lowercase(data, preserve_dict_class=False): +- ''' ++ """ + Recursively changes everything in data to lowercase. +- ''' +- return __change_case(data, 'lower', preserve_dict_class) ++ """ ++ return __change_case(data, "lower", preserve_dict_class) + + + def to_uppercase(data, preserve_dict_class=False): +- ''' ++ """ + Recursively changes everything in data to uppercase. +- ''' +- return __change_case(data, 'upper', preserve_dict_class) ++ """ ++ return __change_case(data, "upper", preserve_dict_class) + + +-@jinja_filter('compare_dicts') ++@jinja_filter("compare_dicts") + def compare_dicts(old=None, new=None): +- ''' ++ """ + Compare before and after results from various salt functions, returning a + dict describing the changes that were made. +- ''' ++ """ + ret = {} +- for key in set((new or {})).union((old or {})): ++ for key in set(new or {}).union(old or {}): + if key not in old: + # New key +- ret[key] = {'old': '', +- 'new': new[key]} ++ ret[key] = {"old": "", "new": new[key]} + elif key not in new: + # Key removed +- ret[key] = {'new': '', +- 'old': old[key]} ++ ret[key] = {"new": "", "old": old[key]} + elif new[key] != old[key]: + # Key modified +- ret[key] = {'old': old[key], +- 'new': new[key]} ++ ret[key] = {"old": old[key], "new": new[key]} + return ret + + +-@jinja_filter('compare_lists') ++@jinja_filter("compare_lists") + def compare_lists(old=None, new=None): +- ''' ++ """ + Compare before and after results from various salt functions, returning a + dict describing the changes that were made +- ''' ++ """ + ret = {} + for item in new: + if item not in old: +- ret.setdefault('new', []).append(item) ++ ret.setdefault("new", []).append(item) + for item in old: + if item not in new: +- ret.setdefault('old', []).append(item) ++ ret.setdefault("old", []).append(item) + return ret + + +-def decode(data, encoding=None, errors='strict', keep=False, +- normalize=False, preserve_dict_class=False, preserve_tuples=False, +- to_str=False): +- ''' ++def _remove_circular_refs(ob, _seen=None): ++ """ ++ Generic method to remove circular references from objects. ++ This has been taken from author Martijn Pieters ++ https://stackoverflow.com/questions/44777369/ ++ remove-circular-references-in-dicts-lists-tuples/44777477#44777477 ++ :param ob: dict, list, typle, set, and frozenset ++ Standard python object ++ :param object _seen: ++ Object that has circular reference ++ :returns: ++ Cleaned Python object ++ :rtype: ++ type(ob) ++ """ ++ if _seen is None: ++ _seen = set() ++ if id(ob) in _seen: ++ # Here we caught a circular reference. ++ # Alert user and cleanup to continue. ++ log.exception( ++ "Caught a circular reference in data structure below." ++ "Cleaning and continuing execution.\n%r\n", ++ ob, ++ ) ++ return None ++ _seen.add(id(ob)) ++ res = ob ++ if isinstance(ob, dict): ++ res = { ++ _remove_circular_refs(k, _seen): _remove_circular_refs(v, _seen) ++ for k, v in ob.items() ++ } ++ elif isinstance(ob, (list, tuple, set, frozenset)): ++ res = type(ob)(_remove_circular_refs(v, _seen) for v in ob) ++ # remove id again; only *nested* references count ++ _seen.remove(id(ob)) ++ return res ++ ++ ++def decode( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ normalize=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++ to_str=False, ++): ++ """ + Generic function which will decode whichever type is passed, if necessary. + Optionally use to_str=True to ensure strings are str types and not unicode + on Python 2. +@@ -199,22 +249,55 @@ def decode(data, encoding=None, errors='strict', keep=False, + two strings above, in which "й" is represented as two code points (i.e. one + for the base character, and one for the breve mark). Normalizing allows for + a more reliable test case. +- ''' +- _decode_func = salt.utils.stringutils.to_unicode \ +- if not to_str \ ++ ++ """ ++ # Clean data object before decoding to avoid circular references ++ data = _remove_circular_refs(data) ++ ++ _decode_func = ( ++ salt.utils.stringutils.to_unicode ++ if not to_str + else salt.utils.stringutils.to_str ++ ) + if isinstance(data, Mapping): +- return decode_dict(data, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ return decode_dict( ++ data, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + if isinstance(data, list): +- return decode_list(data, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ return decode_list( ++ data, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + if isinstance(data, tuple): +- return decode_tuple(data, encoding, errors, keep, normalize, +- preserve_dict_class, to_str) \ +- if preserve_tuples \ +- else decode_list(data, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ return ( ++ decode_tuple( ++ data, encoding, errors, keep, normalize, preserve_dict_class, to_str ++ ) ++ if preserve_tuples ++ else decode_list( ++ data, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) ++ ) + try: + data = _decode_func(data, encoding, errors, normalize) + except TypeError: +@@ -228,25 +311,48 @@ def decode(data, encoding=None, errors='strict', keep=False, + return data + + +-def decode_dict(data, encoding=None, errors='strict', keep=False, +- normalize=False, preserve_dict_class=False, +- preserve_tuples=False, to_str=False): +- ''' ++def decode_dict( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ normalize=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++ to_str=False, ++): ++ """ + Decode all string values to Unicode. Optionally use to_str=True to ensure + strings are str types and not unicode on Python 2. +- ''' +- _decode_func = salt.utils.stringutils.to_unicode \ +- if not to_str \ ++ """ ++ # Clean data object before decoding to avoid circular references ++ data = _remove_circular_refs(data) ++ ++ _decode_func = ( ++ salt.utils.stringutils.to_unicode ++ if not to_str + else salt.utils.stringutils.to_str ++ ) + # Make sure we preserve OrderedDicts + ret = data.__class__() if preserve_dict_class else {} +- for key, value in six.iteritems(data): ++ for key, value in data.items(): + if isinstance(key, tuple): +- key = decode_tuple(key, encoding, errors, keep, normalize, +- preserve_dict_class, to_str) \ +- if preserve_tuples \ +- else decode_list(key, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ key = ( ++ decode_tuple( ++ key, encoding, errors, keep, normalize, preserve_dict_class, to_str ++ ) ++ if preserve_tuples ++ else decode_list( ++ key, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) ++ ) + else: + try: + key = _decode_func(key, encoding, errors, normalize) +@@ -260,17 +366,50 @@ def decode_dict(data, encoding=None, errors='strict', keep=False, + raise + + if isinstance(value, list): +- value = decode_list(value, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ value = decode_list( ++ value, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + elif isinstance(value, tuple): +- value = decode_tuple(value, encoding, errors, keep, normalize, +- preserve_dict_class, to_str) \ +- if preserve_tuples \ +- else decode_list(value, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ value = ( ++ decode_tuple( ++ value, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ to_str, ++ ) ++ if preserve_tuples ++ else decode_list( ++ value, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) ++ ) + elif isinstance(value, Mapping): +- value = decode_dict(value, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ value = decode_dict( ++ value, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + else: + try: + value = _decode_func(value, encoding, errors, normalize) +@@ -287,30 +426,69 @@ def decode_dict(data, encoding=None, errors='strict', keep=False, + return ret + + +-def decode_list(data, encoding=None, errors='strict', keep=False, +- normalize=False, preserve_dict_class=False, +- preserve_tuples=False, to_str=False): +- ''' ++def decode_list( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ normalize=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++ to_str=False, ++): ++ """ + Decode all string values to Unicode. Optionally use to_str=True to ensure + strings are str types and not unicode on Python 2. +- ''' +- _decode_func = salt.utils.stringutils.to_unicode \ +- if not to_str \ ++ """ ++ # Clean data object before decoding to avoid circular references ++ data = _remove_circular_refs(data) ++ ++ _decode_func = ( ++ salt.utils.stringutils.to_unicode ++ if not to_str + else salt.utils.stringutils.to_str ++ ) + ret = [] + for item in data: + if isinstance(item, list): +- item = decode_list(item, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ item = decode_list( ++ item, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + elif isinstance(item, tuple): +- item = decode_tuple(item, encoding, errors, keep, normalize, +- preserve_dict_class, to_str) \ +- if preserve_tuples \ +- else decode_list(item, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ item = ( ++ decode_tuple( ++ item, encoding, errors, keep, normalize, preserve_dict_class, to_str ++ ) ++ if preserve_tuples ++ else decode_list( ++ item, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) ++ ) + elif isinstance(item, Mapping): +- item = decode_dict(item, encoding, errors, keep, normalize, +- preserve_dict_class, preserve_tuples, to_str) ++ item = decode_dict( ++ item, ++ encoding, ++ errors, ++ keep, ++ normalize, ++ preserve_dict_class, ++ preserve_tuples, ++ to_str, ++ ) + else: + try: + item = _decode_func(item, encoding, errors, normalize) +@@ -327,21 +505,35 @@ def decode_list(data, encoding=None, errors='strict', keep=False, + return ret + + +-def decode_tuple(data, encoding=None, errors='strict', keep=False, +- normalize=False, preserve_dict_class=False, to_str=False): +- ''' ++def decode_tuple( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ normalize=False, ++ preserve_dict_class=False, ++ to_str=False, ++): ++ """ + Decode all string values to Unicode. Optionally use to_str=True to ensure + strings are str types and not unicode on Python 2. +- ''' ++ """ + return tuple( +- decode_list(data, encoding, errors, keep, normalize, +- preserve_dict_class, True, to_str) ++ decode_list( ++ data, encoding, errors, keep, normalize, preserve_dict_class, True, to_str ++ ) + ) + + +-def encode(data, encoding=None, errors='strict', keep=False, +- preserve_dict_class=False, preserve_tuples=False): +- ''' ++def encode( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++): ++ """ + Generic function which will encode whichever type is passed, if necessary + + If `strict` is True, and `keep` is False, and we fail to encode, a +@@ -349,18 +541,27 @@ def encode(data, encoding=None, errors='strict', keep=False, + original value to silently be returned in cases where encoding fails. This + can be useful for cases where the data passed to this function is likely to + contain binary blobs. +- ''' ++ ++ """ ++ # Clean data object before encoding to avoid circular references ++ data = _remove_circular_refs(data) ++ + if isinstance(data, Mapping): +- return encode_dict(data, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ return encode_dict( ++ data, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + if isinstance(data, list): +- return encode_list(data, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ return encode_list( ++ data, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + if isinstance(data, tuple): +- return encode_tuple(data, encoding, errors, keep, preserve_dict_class) \ +- if preserve_tuples \ +- else encode_list(data, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ return ( ++ encode_tuple(data, encoding, errors, keep, preserve_dict_class) ++ if preserve_tuples ++ else encode_list( ++ data, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) ++ ) + try: + return salt.utils.stringutils.to_bytes(data, encoding, errors) + except TypeError: +@@ -374,20 +575,31 @@ def encode(data, encoding=None, errors='strict', keep=False, + return data + + +-@jinja_filter('json_decode_dict') # Remove this for Aluminium +-@jinja_filter('json_encode_dict') +-def encode_dict(data, encoding=None, errors='strict', keep=False, +- preserve_dict_class=False, preserve_tuples=False): +- ''' ++@jinja_filter("json_decode_dict") # Remove this for Aluminium ++@jinja_filter("json_encode_dict") ++def encode_dict( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++): ++ """ + Encode all string values to bytes +- ''' ++ """ ++ # Clean data object before encoding to avoid circular references ++ data = _remove_circular_refs(data) + ret = data.__class__() if preserve_dict_class else {} +- for key, value in six.iteritems(data): ++ for key, value in data.items(): + if isinstance(key, tuple): +- key = encode_tuple(key, encoding, errors, keep, preserve_dict_class) \ +- if preserve_tuples \ +- else encode_list(key, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ key = ( ++ encode_tuple(key, encoding, errors, keep, preserve_dict_class) ++ if preserve_tuples ++ else encode_list( ++ key, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) ++ ) + else: + try: + key = salt.utils.stringutils.to_bytes(key, encoding, errors) +@@ -401,16 +613,21 @@ def encode_dict(data, encoding=None, errors='strict', keep=False, + raise + + if isinstance(value, list): +- value = encode_list(value, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ value = encode_list( ++ value, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + elif isinstance(value, tuple): +- value = encode_tuple(value, encoding, errors, keep, preserve_dict_class) \ +- if preserve_tuples \ +- else encode_list(value, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ value = ( ++ encode_tuple(value, encoding, errors, keep, preserve_dict_class) ++ if preserve_tuples ++ else encode_list( ++ value, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) ++ ) + elif isinstance(value, Mapping): +- value = encode_dict(value, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ value = encode_dict( ++ value, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + else: + try: + value = salt.utils.stringutils.to_bytes(value, encoding, errors) +@@ -427,26 +644,40 @@ def encode_dict(data, encoding=None, errors='strict', keep=False, + return ret + + +-@jinja_filter('json_decode_list') # Remove this for Aluminium +-@jinja_filter('json_encode_list') +-def encode_list(data, encoding=None, errors='strict', keep=False, +- preserve_dict_class=False, preserve_tuples=False): +- ''' ++@jinja_filter("json_decode_list") # Remove this for Aluminium ++@jinja_filter("json_encode_list") ++def encode_list( ++ data, ++ encoding=None, ++ errors="strict", ++ keep=False, ++ preserve_dict_class=False, ++ preserve_tuples=False, ++): ++ """ + Encode all string values to bytes +- ''' ++ """ ++ # Clean data object before encoding to avoid circular references ++ data = _remove_circular_refs(data) ++ + ret = [] + for item in data: + if isinstance(item, list): +- item = encode_list(item, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ item = encode_list( ++ item, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + elif isinstance(item, tuple): +- item = encode_tuple(item, encoding, errors, keep, preserve_dict_class) \ +- if preserve_tuples \ +- else encode_list(item, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ item = ( ++ encode_tuple(item, encoding, errors, keep, preserve_dict_class) ++ if preserve_tuples ++ else encode_list( ++ item, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) ++ ) + elif isinstance(item, Mapping): +- item = encode_dict(item, encoding, errors, keep, +- preserve_dict_class, preserve_tuples) ++ item = encode_dict( ++ item, encoding, errors, keep, preserve_dict_class, preserve_tuples ++ ) + else: + try: + item = salt.utils.stringutils.to_bytes(item, encoding, errors) +@@ -463,42 +694,37 @@ def encode_list(data, encoding=None, errors='strict', keep=False, + return ret + + +-def encode_tuple(data, encoding=None, errors='strict', keep=False, +- preserve_dict_class=False): +- ''' ++def encode_tuple( ++ data, encoding=None, errors="strict", keep=False, preserve_dict_class=False ++): ++ """ + Encode all string values to Unicode +- ''' +- return tuple( +- encode_list(data, encoding, errors, keep, preserve_dict_class, True)) ++ """ ++ return tuple(encode_list(data, encoding, errors, keep, preserve_dict_class, True)) + + +-@jinja_filter('exactly_n_true') ++@jinja_filter("exactly_n_true") + def exactly_n(iterable, amount=1): +- ''' ++ """ + Tests that exactly N items in an iterable are "truthy" (neither None, + False, nor 0). +- ''' ++ """ + i = iter(iterable) + return all(any(i) for j in range(amount)) and not any(i) + + +-@jinja_filter('exactly_one_true') ++@jinja_filter("exactly_one_true") + def exactly_one(iterable): +- ''' ++ """ + Check if only one item is not None, False, or 0 in an iterable. +- ''' ++ """ + return exactly_n(iterable) + + +-def filter_by(lookup_dict, +- lookup, +- traverse, +- merge=None, +- default='default', +- base=None): +- ''' ++def filter_by(lookup_dict, lookup, traverse, merge=None, default="default", base=None): ++ """ + Common code to filter data structures like grains and pillar +- ''' ++ """ + ret = None + # Default value would be an empty list if lookup not found + val = traverse_dict_and_list(traverse, lookup, []) +@@ -507,10 +733,8 @@ def filter_by(lookup_dict, + # lookup_dict keys + for each in val if isinstance(val, list) else [val]: + for key in lookup_dict: +- test_key = key if isinstance(key, six.string_types) \ +- else six.text_type(key) +- test_each = each if isinstance(each, six.string_types) \ +- else six.text_type(each) ++ test_key = key if isinstance(key, str) else str(key) ++ test_each = each if isinstance(each, str) else str(each) + if fnmatch.fnmatchcase(test_each, test_key): + ret = lookup_dict[key] + break +@@ -528,14 +752,13 @@ def filter_by(lookup_dict, + elif isinstance(base_values, Mapping): + if not isinstance(ret, Mapping): + raise SaltException( +- 'filter_by default and look-up values must both be ' +- 'dictionaries.') ++ "filter_by default and look-up values must both be " "dictionaries." ++ ) + ret = salt.utils.dictupdate.update(copy.deepcopy(base_values), ret) + + if merge: + if not isinstance(merge, Mapping): +- raise SaltException( +- 'filter_by merge argument must be a dictionary.') ++ raise SaltException("filter_by merge argument must be a dictionary.") + + if ret is None: + ret = merge +@@ -546,12 +769,12 @@ def filter_by(lookup_dict, + + + def traverse_dict(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): +- ''' ++ """ + Traverse a dict using a colon-delimited (or otherwise delimited, using the + 'delimiter' param) target string. The target 'foo:bar:baz' will return + data['foo']['bar']['baz'] if this value exists, and will otherwise return + the dict in the default argument. +- ''' ++ """ + ptr = data + try: + for each in key.split(delimiter): +@@ -562,9 +785,9 @@ def traverse_dict(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): + return ptr + + +-@jinja_filter('traverse') ++@jinja_filter("traverse") + def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): +- ''' ++ """ + Traverse a dict or list using a colon-delimited (or otherwise delimited, + using the 'delimiter' param) target string. The target 'foo:bar:0' will + return data['foo']['bar'][0] if this value exists, and will otherwise +@@ -573,7 +796,7 @@ def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DEL + The target 'foo:bar:0' will return data['foo']['bar'][0] if data like + {'foo':{'bar':['baz']}} , if data like {'foo':{'bar':{'0':'baz'}}} + then return data['foo']['bar']['0'] +- ''' ++ """ + ptr = data + for each in key.split(delimiter): + if isinstance(ptr, list): +@@ -605,18 +828,17 @@ def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DEL + return ptr + + +-def subdict_match(data, +- expr, +- delimiter=DEFAULT_TARGET_DELIM, +- regex_match=False, +- exact_match=False): +- ''' ++def subdict_match( ++ data, expr, delimiter=DEFAULT_TARGET_DELIM, regex_match=False, exact_match=False ++): ++ """ + Check for a match in a dictionary using a delimiter character to denote + levels of subdicts, and also allowing the delimiter character to be + matched. Thus, 'foo:bar:baz' will match data['foo'] == 'bar:baz' and + data['foo']['bar'] == 'baz'. The latter would take priority over the + former, as more deeply-nested matches are tried first. +- ''' ++ """ ++ + def _match(target, pattern, regex_match=False, exact_match=False): + # The reason for using six.text_type first and _then_ using + # to_unicode as a fallback is because we want to eventually have +@@ -628,11 +850,11 @@ def subdict_match(data, + # begin with is that (by design) to_unicode will raise a TypeError if a + # non-string/bytestring/bytearray value is passed. + try: +- target = six.text_type(target).lower() ++ target = str(target).lower() + except UnicodeDecodeError: + target = salt.utils.stringutils.to_unicode(target).lower() + try: +- pattern = six.text_type(pattern).lower() ++ pattern = str(pattern).lower() + except UnicodeDecodeError: + pattern = salt.utils.stringutils.to_unicode(pattern).lower() + +@@ -640,48 +862,54 @@ def subdict_match(data, + try: + return re.match(pattern, target) + except Exception: # pylint: disable=broad-except +- log.error('Invalid regex \'%s\' in match', pattern) ++ log.error("Invalid regex '%s' in match", pattern) + return False + else: +- return target == pattern if exact_match \ +- else fnmatch.fnmatch(target, pattern) ++ return ( ++ target == pattern if exact_match else fnmatch.fnmatch(target, pattern) ++ ) + + def _dict_match(target, pattern, regex_match=False, exact_match=False): + ret = False +- wildcard = pattern.startswith('*:') ++ wildcard = pattern.startswith("*:") + if wildcard: + pattern = pattern[2:] + +- if pattern == '*': ++ if pattern == "*": + # We are just checking that the key exists + ret = True + if not ret and pattern in target: + # We might want to search for a key + ret = True +- if not ret and subdict_match(target, +- pattern, +- regex_match=regex_match, +- exact_match=exact_match): ++ if not ret and subdict_match( ++ target, pattern, regex_match=regex_match, exact_match=exact_match ++ ): + ret = True + if not ret and wildcard: + for key in target: + if isinstance(target[key], dict): +- if _dict_match(target[key], +- pattern, +- regex_match=regex_match, +- exact_match=exact_match): ++ if _dict_match( ++ target[key], ++ pattern, ++ regex_match=regex_match, ++ exact_match=exact_match, ++ ): + return True + elif isinstance(target[key], list): + for item in target[key]: +- if _match(item, +- pattern, +- regex_match=regex_match, +- exact_match=exact_match): +- return True +- elif _match(target[key], ++ if _match( ++ item, + pattern, + regex_match=regex_match, +- exact_match=exact_match): ++ exact_match=exact_match, ++ ): ++ return True ++ elif _match( ++ target[key], ++ pattern, ++ regex_match=regex_match, ++ exact_match=exact_match, ++ ): + return True + return ret + +@@ -695,7 +923,7 @@ def subdict_match(data, + # want to use are 3, 2, and 1, in that order. + for idx in range(num_splits - 1, 0, -1): + key = delimiter.join(splits[:idx]) +- if key == '*': ++ if key == "*": + # We are matching on everything under the top level, so we need to + # treat the match as the entire data being passed in + matchstr = expr +@@ -703,54 +931,55 @@ def subdict_match(data, + else: + matchstr = delimiter.join(splits[idx:]) + match = traverse_dict_and_list(data, key, {}, delimiter=delimiter) +- log.debug("Attempting to match '%s' in '%s' using delimiter '%s'", +- matchstr, key, delimiter) ++ log.debug( ++ "Attempting to match '%s' in '%s' using delimiter '%s'", ++ matchstr, ++ key, ++ delimiter, ++ ) + if match == {}: + continue + if isinstance(match, dict): +- if _dict_match(match, +- matchstr, +- regex_match=regex_match, +- exact_match=exact_match): ++ if _dict_match( ++ match, matchstr, regex_match=regex_match, exact_match=exact_match ++ ): + return True + continue + if isinstance(match, (list, tuple)): + # We are matching a single component to a single list member + for member in match: + if isinstance(member, dict): +- if _dict_match(member, +- matchstr, +- regex_match=regex_match, +- exact_match=exact_match): ++ if _dict_match( ++ member, ++ matchstr, ++ regex_match=regex_match, ++ exact_match=exact_match, ++ ): + return True +- if _match(member, +- matchstr, +- regex_match=regex_match, +- exact_match=exact_match): ++ if _match( ++ member, matchstr, regex_match=regex_match, exact_match=exact_match ++ ): + return True + continue +- if _match(match, +- matchstr, +- regex_match=regex_match, +- exact_match=exact_match): ++ if _match(match, matchstr, regex_match=regex_match, exact_match=exact_match): + return True + return False + + +-@jinja_filter('substring_in_list') ++@jinja_filter("substring_in_list") + def substr_in_list(string_to_search_for, list_to_search): +- ''' ++ """ + Return a boolean value that indicates whether or not a given + string is present in any of the strings which comprise a list +- ''' ++ """ + return any(string_to_search_for in s for s in list_to_search) + + + def is_dictlist(data): +- ''' ++ """ + Returns True if data is a list of one-element dicts (as found in many SLS + schemas), otherwise returns False +- ''' ++ """ + if isinstance(data, list): + for element in data: + if isinstance(element, dict): +@@ -762,16 +991,12 @@ def is_dictlist(data): + return False + + +-def repack_dictlist(data, +- strict=False, +- recurse=False, +- key_cb=None, +- val_cb=None): +- ''' ++def repack_dictlist(data, strict=False, recurse=False, key_cb=None, val_cb=None): ++ """ + Takes a list of one-element dicts (as found in many SLS schemas) and + repacks into a single dictionary. +- ''' +- if isinstance(data, six.string_types): ++ """ ++ if isinstance(data, str): + try: + data = salt.utils.yaml.safe_load(data) + except salt.utils.yaml.parser.ParserError as err: +@@ -783,7 +1008,7 @@ def repack_dictlist(data, + if val_cb is None: + val_cb = lambda x, y: y + +- valid_non_dict = (six.string_types, six.integer_types, float) ++ valid_non_dict = ((str,), (int,), float) + if isinstance(data, list): + for element in data: + if isinstance(element, valid_non_dict): +@@ -791,21 +1016,21 @@ def repack_dictlist(data, + if isinstance(element, dict): + if len(element) != 1: + log.error( +- 'Invalid input for repack_dictlist: key/value pairs ' +- 'must contain only one element (data passed: %s).', +- element ++ "Invalid input for repack_dictlist: key/value pairs " ++ "must contain only one element (data passed: %s).", ++ element, + ) + return {} + else: + log.error( +- 'Invalid input for repack_dictlist: element %s is ' +- 'not a string/dict/numeric value', element ++ "Invalid input for repack_dictlist: element %s is " ++ "not a string/dict/numeric value", ++ element, + ) + return {} + else: + log.error( +- 'Invalid input for repack_dictlist, data passed is not a list ' +- '(%s)', data ++ "Invalid input for repack_dictlist, data passed is not a list " "(%s)", data + ) + return {} + +@@ -821,8 +1046,8 @@ def repack_dictlist(data, + ret[key_cb(key)] = repack_dictlist(val, recurse=recurse) + elif strict: + log.error( +- 'Invalid input for repack_dictlist: nested dictlist ' +- 'found, but recurse is set to False' ++ "Invalid input for repack_dictlist: nested dictlist " ++ "found, but recurse is set to False" + ) + return {} + else: +@@ -832,17 +1057,17 @@ def repack_dictlist(data, + return ret + + +-@jinja_filter('is_list') ++@jinja_filter("is_list") + def is_list(value): +- ''' ++ """ + Check if a variable is a list. +- ''' ++ """ + return isinstance(value, list) + + +-@jinja_filter('is_iter') +-def is_iter(thing, ignore=six.string_types): +- ''' ++@jinja_filter("is_iter") ++def is_iter(thing, ignore=(str,)): ++ """ + Test if an object is iterable, but not a string type. + + Test if an object is an iterator or is iterable itself. By default this +@@ -853,7 +1078,7 @@ def is_iter(thing, ignore=six.string_types): + dictionaries or named tuples. + + Based on https://bitbucket.org/petershinners/yter +- ''' ++ """ + if ignore and isinstance(thing, ignore): + return False + try: +@@ -863,9 +1088,9 @@ def is_iter(thing, ignore=six.string_types): + return False + + +-@jinja_filter('sorted_ignorecase') ++@jinja_filter("sorted_ignorecase") + def sorted_ignorecase(to_sort): +- ''' ++ """ + Sort a list of strings ignoring case. + + >>> L = ['foo', 'Foo', 'bar', 'Bar'] +@@ -874,19 +1099,19 @@ def sorted_ignorecase(to_sort): + >>> sorted(L, key=lambda x: x.lower()) + ['bar', 'Bar', 'foo', 'Foo'] + >>> +- ''' ++ """ + return sorted(to_sort, key=lambda x: x.lower()) + + + def is_true(value=None): +- ''' ++ """ + Returns a boolean value representing the "truth" of the value passed. The + rules for what is a "True" value are: + + 1. Integer/float values greater than 0 + 2. The string values "True" and "true" + 3. Any object for which bool(obj) returns True +- ''' ++ """ + # First, try int/float conversion + try: + value = int(value) +@@ -898,26 +1123,26 @@ def is_true(value=None): + pass + + # Now check for truthiness +- if isinstance(value, (six.integer_types, float)): ++ if isinstance(value, ((int,), float)): + return value > 0 +- if isinstance(value, six.string_types): +- return six.text_type(value).lower() == 'true' ++ if isinstance(value, str): ++ return str(value).lower() == "true" + return bool(value) + + +-@jinja_filter('mysql_to_dict') ++@jinja_filter("mysql_to_dict") + def mysql_to_dict(data, key): +- ''' ++ """ + Convert MySQL-style output to a python dictionary +- ''' ++ """ + ret = {} +- headers = [''] ++ headers = [""] + for line in data: + if not line: + continue +- if line.startswith('+'): ++ if line.startswith("+"): + continue +- comps = line.split('|') ++ comps = line.split("|") + for comp in range(len(comps)): + comps[comp] = comps[comp].strip() + if len(headers) > 1: +@@ -934,14 +1159,14 @@ def mysql_to_dict(data, key): + + + def simple_types_filter(data): +- ''' ++ """ + Convert the data list, dictionary into simple types, i.e., int, float, string, + bool, etc. +- ''' ++ """ + if data is None: + return data + +- simpletypes_keys = (six.string_types, six.text_type, six.integer_types, float, bool) ++ simpletypes_keys = ((str,), str, (int,), float, bool) + simpletypes_values = tuple(list(simpletypes_keys) + [list, tuple]) + + if isinstance(data, (list, tuple)): +@@ -957,7 +1182,7 @@ def simple_types_filter(data): + + if isinstance(data, dict): + simpledict = {} +- for key, value in six.iteritems(data): ++ for key, value in data.items(): + if key is not None and not isinstance(key, simpletypes_keys): + key = repr(key) + if value is not None and isinstance(value, (dict, list, tuple)): +@@ -971,23 +1196,23 @@ def simple_types_filter(data): + + + def stringify(data): +- ''' ++ """ + Given an iterable, returns its items as a list, with any non-string items + converted to unicode strings. +- ''' ++ """ + ret = [] + for item in data: + if six.PY2 and isinstance(item, str): + item = salt.utils.stringutils.to_unicode(item) +- elif not isinstance(item, six.string_types): +- item = six.text_type(item) ++ elif not isinstance(item, str): ++ item = str(item) + ret.append(item) + return ret + + +-@jinja_filter('json_query') ++@jinja_filter("json_query") + def json_query(data, expr): +- ''' ++ """ + Query data using JMESPath language (http://jmespath.org). + + Requires the https://github.com/jmespath/jmespath.py library. +@@ -1009,16 +1234,16 @@ def json_query(data, expr): + .. code-block:: text + + [80, 25, 22] +- ''' ++ """ + if jmespath is None: +- err = 'json_query requires jmespath module installed' ++ err = "json_query requires jmespath module installed" + log.error(err) + raise RuntimeError(err) + return jmespath.search(expr, data) + + + def _is_not_considered_falsey(value, ignore_types=()): +- ''' ++ """ + Helper function for filter_falsey to determine if something is not to be + considered falsey. + +@@ -1026,12 +1251,12 @@ def _is_not_considered_falsey(value, ignore_types=()): + :param list ignore_types: The types to ignore when considering the value. + + :return bool +- ''' ++ """ + return isinstance(value, bool) or type(value) in ignore_types or value + + + def filter_falsey(data, recurse_depth=None, ignore_types=()): +- ''' ++ """ + Helper function to remove items from an iterable with falsey value. + Removes ``None``, ``{}`` and ``[]``, 0, '' (but does not remove ``False``). + Recurses into sub-iterables if ``recurse`` is set to ``True``. +@@ -1045,37 +1270,42 @@ def filter_falsey(data, recurse_depth=None, ignore_types=()): + :return type(data) + + .. versionadded:: 3000 +- ''' ++ """ + filter_element = ( +- functools.partial(filter_falsey, +- recurse_depth=recurse_depth-1, +- ignore_types=ignore_types) +- if recurse_depth else lambda x: x ++ functools.partial( ++ filter_falsey, recurse_depth=recurse_depth - 1, ignore_types=ignore_types ++ ) ++ if recurse_depth ++ else lambda x: x + ) + + if isinstance(data, dict): +- processed_elements = [(key, filter_element(value)) for key, value in six.iteritems(data)] +- return type(data)([ +- (key, value) +- for key, value in processed_elements +- if _is_not_considered_falsey(value, ignore_types=ignore_types) +- ]) ++ processed_elements = [ ++ (key, filter_element(value)) for key, value in data.items() ++ ] ++ return type(data)( ++ [ ++ (key, value) ++ for key, value in processed_elements ++ if _is_not_considered_falsey(value, ignore_types=ignore_types) ++ ] ++ ) + if is_iter(data): + processed_elements = (filter_element(value) for value in data) +- return type(data)([ +- value for value in processed_elements +- if _is_not_considered_falsey(value, ignore_types=ignore_types) +- ]) ++ return type(data)( ++ [ ++ value ++ for value in processed_elements ++ if _is_not_considered_falsey(value, ignore_types=ignore_types) ++ ] ++ ) + return data + + + def recursive_diff( +- old, +- new, +- ignore_keys=None, +- ignore_order=False, +- ignore_missing_keys=False): +- ''' ++ old, new, ignore_keys=None, ignore_order=False, ignore_missing_keys=False ++): ++ """ + Performs a recursive diff on mappings and/or iterables and returns the result + in a {'old': values, 'new': values}-style. + Compares dicts and sets unordered (obviously), OrderedDicts and Lists ordered +@@ -1090,12 +1320,16 @@ def recursive_diff( + but missing in ``new``. Only works for regular dicts. + + :return dict: Returns dict with keys 'old' and 'new' containing the differences. +- ''' ++ """ + ignore_keys = ignore_keys or [] + res = {} + ret_old = copy.deepcopy(old) + ret_new = copy.deepcopy(new) +- if isinstance(old, OrderedDict) and isinstance(new, OrderedDict) and not ignore_order: ++ if ( ++ isinstance(old, OrderedDict) ++ and isinstance(new, OrderedDict) ++ and not ignore_order ++ ): + append_old, append_new = [], [] + if len(old) != len(new): + min_length = min(len(old), len(new)) +@@ -1114,13 +1348,14 @@ def recursive_diff( + new[key_new], + ignore_keys=ignore_keys, + ignore_order=ignore_order, +- ignore_missing_keys=ignore_missing_keys) ++ ignore_missing_keys=ignore_missing_keys, ++ ) + if not res: # Equal + del ret_old[key_old] + del ret_new[key_new] + else: +- ret_old[key_old] = res['old'] +- ret_new[key_new] = res['new'] ++ ret_old[key_old] = res["old"] ++ ret_new[key_new] = res["new"] + else: + if key_old in ignore_keys: + del ret_old[key_old] +@@ -1131,7 +1366,7 @@ def recursive_diff( + ret_old[item] = old[item] + for item in append_new: + ret_new[item] = new[item] +- ret = {'old': ret_old, 'new': ret_new} if ret_old or ret_new else {} ++ ret = {"old": ret_old, "new": ret_new} if ret_old or ret_new else {} + elif isinstance(old, Mapping) and isinstance(new, Mapping): + # Compare unordered + for key in set(list(old) + list(new)): +@@ -1146,16 +1381,17 @@ def recursive_diff( + new[key], + ignore_keys=ignore_keys, + ignore_order=ignore_order, +- ignore_missing_keys=ignore_missing_keys) ++ ignore_missing_keys=ignore_missing_keys, ++ ) + if not res: # Equal + del ret_old[key] + del ret_new[key] + else: +- ret_old[key] = res['old'] +- ret_new[key] = res['new'] +- ret = {'old': ret_old, 'new': ret_new} if ret_old or ret_new else {} ++ ret_old[key] = res["old"] ++ ret_new[key] = res["new"] ++ ret = {"old": ret_old, "new": ret_new} if ret_old or ret_new else {} + elif isinstance(old, set) and isinstance(new, set): +- ret = {'old': old - new, 'new': new - old} if old - new or new - old else {} ++ ret = {"old": old - new, "new": new - old} if old - new or new - old else {} + elif is_iter(old) and is_iter(new): + # Create a list so we can edit on an index-basis. + list_old = list(ret_old) +@@ -1168,7 +1404,8 @@ def recursive_diff( + item_new, + ignore_keys=ignore_keys, + ignore_order=ignore_order, +- ignore_missing_keys=ignore_missing_keys) ++ ignore_missing_keys=ignore_missing_keys, ++ ) + if not res: + list_old.remove(item_old) + list_new.remove(item_new) +@@ -1181,19 +1418,87 @@ def recursive_diff( + iter_new, + ignore_keys=ignore_keys, + ignore_order=ignore_order, +- ignore_missing_keys=ignore_missing_keys) ++ ignore_missing_keys=ignore_missing_keys, ++ ) + if not res: # Equal + remove_indices.append(index) + else: +- list_old[index] = res['old'] +- list_new[index] = res['new'] ++ list_old[index] = res["old"] ++ list_new[index] = res["new"] + for index in reversed(remove_indices): + list_old.pop(index) + list_new.pop(index) + # Instantiate a new whatever-it-was using the list as iterable source. + # This may not be the most optimized in way of speed and memory usage, + # but it will work for all iterable types. +- ret = {'old': type(old)(list_old), 'new': type(new)(list_new)} if list_old or list_new else {} ++ ret = ( ++ {"old": type(old)(list_old), "new": type(new)(list_new)} ++ if list_old or list_new ++ else {} ++ ) + else: +- ret = {} if old == new else {'old': ret_old, 'new': ret_new} ++ ret = {} if old == new else {"old": ret_old, "new": ret_new} + return ret ++ ++ ++def get_value(obj, path, default=None): ++ """ ++ Get the values for a given path. ++ ++ :param path: ++ keys of the properties in the tree separated by colons. ++ One segment in the path can be replaced by an id surrounded by curly braces. ++ This will match all items in a list of dictionary. ++ ++ :param default: ++ default value to return when no value is found ++ ++ :return: ++ a list of dictionaries, with at least the "value" key providing the actual value. ++ If a placeholder was used, the placeholder id will be a key providing the replacement for it. ++ Note that a value that wasn't found in the tree will be an empty list. ++ This ensures we can make the difference with a None value set by the user. ++ """ ++ res = [{"value": obj}] ++ if path: ++ key = path[: path.find(":")] if ":" in path else path ++ next_path = path[path.find(":") + 1 :] if ":" in path else None ++ ++ if key.startswith("{") and key.endswith("}"): ++ placeholder_name = key[1:-1] ++ # There will be multiple values to get here ++ items = [] ++ if obj is None: ++ return res ++ if isinstance(obj, dict): ++ items = obj.items() ++ elif isinstance(obj, list): ++ items = enumerate(obj) ++ ++ def _append_placeholder(value_dict, key): ++ value_dict[placeholder_name] = key ++ return value_dict ++ ++ values = [ ++ [ ++ _append_placeholder(item, key) ++ for item in get_value(val, next_path, default) ++ ] ++ for key, val in items ++ ] ++ ++ # flatten the list ++ values = [y for x in values for y in x] ++ return values ++ elif isinstance(obj, dict): ++ if key not in obj.keys(): ++ return [{"value": default}] ++ ++ value = obj.get(key) ++ if res is not None: ++ res = get_value(value, next_path, default) ++ else: ++ res = [{"value": value}] ++ else: ++ return [{"value": default if obj is not None else obj}] ++ return res +diff --git a/salt/utils/xmlutil.py b/salt/utils/xmlutil.py +index 6d8d74fd3f..2b9c7bf43f 100644 +--- a/salt/utils/xmlutil.py ++++ b/salt/utils/xmlutil.py +@@ -1,30 +1,34 @@ +-# -*- coding: utf-8 -*- +-''' ++""" + Various XML utilities +-''' ++""" + + # Import Python libs +-from __future__ import absolute_import, print_function, unicode_literals ++import re ++import string # pylint: disable=deprecated-module ++from xml.etree import ElementTree ++ ++# Import salt libs ++import salt.utils.data + + + def _conv_name(x): +- ''' ++ """ + If this XML tree has an xmlns attribute, then etree will add it + to the beginning of the tag, like: "{http://path}tag". +- ''' +- if '}' in x: +- comps = x.split('}') ++ """ ++ if "}" in x: ++ comps = x.split("}") + name = comps[1] + return name + return x + + + def _to_dict(xmltree): +- ''' ++ """ + Converts an XML ElementTree to a dictionary that only contains items. + This is the default behavior in version 2017.7. This will default to prevent + unexpected parsing issues on modules dependant on this. +- ''' ++ """ + # If this object has no children, the for..loop below will return nothing + # for it, so just return a single dict representing it. + if len(xmltree.getchildren()) < 1: +@@ -51,9 +55,9 @@ def _to_dict(xmltree): + + + def _to_full_dict(xmltree): +- ''' ++ """ + Returns the full XML dictionary including attributes. +- ''' ++ """ + xmldict = {} + + for attrName, attrValue in xmltree.attrib.items(): +@@ -87,15 +91,234 @@ def _to_full_dict(xmltree): + + + def to_dict(xmltree, attr=False): +- ''' ++ """ + Convert an XML tree into a dict. The tree that is passed in must be an + ElementTree object. + Args: + xmltree: An ElementTree object. + attr: If true, attributes will be parsed. If false, they will be ignored. + +- ''' ++ """ + if attr: + return _to_full_dict(xmltree) + else: + return _to_dict(xmltree) ++ ++ ++def get_xml_node(node, xpath): ++ """ ++ Get an XML node using a path (super simple xpath showing complete node ancestry). ++ This also creates the missing nodes. ++ ++ The supported XPath can contain elements filtering using [@attr='value']. ++ ++ Args: ++ node: an Element object ++ xpath: simple XPath to look for. ++ """ ++ if not xpath.startswith("./"): ++ xpath = "./{}".format(xpath) ++ res = node.find(xpath) ++ if res is None: ++ parent_xpath = xpath[: xpath.rfind("/")] ++ parent = node.find(parent_xpath) ++ if parent is None: ++ parent = get_xml_node(node, parent_xpath) ++ segment = xpath[xpath.rfind("/") + 1 :] ++ # We may have [] filter in the segment ++ matcher = re.match( ++ r"""(?P[^[]+)(?:\[@(?P\w+)=["'](?P[^"']+)["']])?""", ++ segment, ++ ) ++ attrib = ( ++ {matcher.group("attr"): matcher.group("value")} ++ if matcher.group("attr") and matcher.group("value") ++ else {} ++ ) ++ res = ElementTree.SubElement(parent, matcher.group("tag"), attrib) ++ return res ++ ++ ++def set_node_text(node, value): ++ """ ++ Function to use in the ``set`` value in the :py:func:`change_xml` mapping items to set the text. ++ This is the default. ++ ++ :param node: the node to set the text to ++ :param value: the value to set ++ """ ++ node.text = str(value) ++ ++ ++def clean_node(parent_map, node, ignored=None): ++ """ ++ Remove the node from its parent if it has no attribute but the ignored ones, no text and no child. ++ Recursively called up to the document root to ensure no empty node is left. ++ ++ :param parent_map: dictionary mapping each node to its parent ++ :param node: the node to clean ++ :param ignored: a list of ignored attributes. ++ """ ++ has_text = node.text is not None and node.text.strip() ++ parent = parent_map.get(node) ++ if ( ++ len(node.attrib.keys() - (ignored or [])) == 0 ++ and not list(node) ++ and not has_text ++ ): ++ parent.remove(node) ++ # Clean parent nodes if needed ++ if parent is not None: ++ clean_node(parent_map, parent, ignored) ++ ++ ++def del_text(parent_map, node): ++ """ ++ Function to use as ``del`` value in the :py:func:`change_xml` mapping items to remove the text. ++ This is the default function. ++ Calls :py:func:`clean_node` before returning. ++ """ ++ parent = parent_map[node] ++ parent.remove(node) ++ clean_node(parent, node) ++ ++ ++def del_attribute(attribute, ignored=None): ++ """ ++ Helper returning a function to use as ``del`` value in the :py:func:`change_xml` mapping items to ++ remove an attribute. ++ ++ The generated function calls :py:func:`clean_node` before returning. ++ ++ :param attribute: the name of the attribute to remove ++ :param ignored: the list of attributes to ignore during the cleanup ++ ++ :return: the function called by :py:func:`change_xml`. ++ """ ++ ++ def _do_delete(parent_map, node): ++ if attribute not in node.keys(): ++ return ++ node.attrib.pop(attribute) ++ clean_node(parent_map, node, ignored) ++ ++ return _do_delete ++ ++ ++def change_xml(doc, data, mapping): ++ """ ++ Change an XML ElementTree document according. ++ ++ :param doc: the ElementTree parsed XML document to modify ++ :param data: the dictionary of values used to modify the XML. ++ :param mapping: a list of items describing how to modify the XML document. ++ Each item is a dictionary containing the following keys: ++ ++ .. glossary:: ++ path ++ the path to the value to set or remove in the ``data`` parameter. ++ See :py:func:`salt.utils.data.get_value ` for the format ++ of the value. ++ ++ xpath ++ Simplified XPath expression used to locate the change in the XML tree. ++ See :py:func:`get_xml_node` documentation for details on the supported XPath syntax ++ ++ get ++ function gettin the value from the XML. ++ Takes a single parameter for the XML node found by the XPath expression. ++ Default returns the node text value. ++ This may be used to return an attribute or to perform value transformation. ++ ++ set ++ function setting the value in the XML. ++ Takes two parameters for the XML node and the value to set. ++ Default is to set the text value. ++ ++ del ++ function deleting the value in the XML. ++ Takes two parameters for the parent node and the node matched by the XPath. ++ Default is to remove the text value. ++ More cleanup may be performed, see the :py:func:`clean_node` function for details. ++ ++ convert ++ function modifying the user-provided value right before comparing it with the one from the XML. ++ Takes the value as single parameter. ++ Default is to apply no conversion. ++ ++ :return: ``True`` if the XML has been modified, ``False`` otherwise. ++ """ ++ need_update = False ++ for param in mapping: ++ # Get the value from the function parameter using the path-like description ++ # Using an empty list as a default value will cause values not provided by the user ++ # to be left untouched, as opposed to explicit None unsetting the value ++ values = salt.utils.data.get_value(data, param["path"], []) ++ xpath = param["xpath"] ++ # Prepend the xpath with ./ to handle the root more easily ++ if not xpath.startswith("./"): ++ xpath = "./{}".format(xpath) ++ ++ placeholders = [ ++ s[1:-1] ++ for s in param["path"].split(":") ++ if s.startswith("{") and s.endswith("}") ++ ] ++ ++ ctx = {placeholder: "$$$" for placeholder in placeholders} ++ all_nodes_xpath = string.Template(xpath).substitute(ctx) ++ all_nodes_xpath = re.sub( ++ r"""(?:=['"]\$\$\$["'])|(?:\[\$\$\$\])""", "", all_nodes_xpath ++ ) ++ ++ # Store the nodes that are not removed for later cleanup ++ kept_nodes = set() ++ ++ for value_item in values: ++ new_value = value_item["value"] ++ ++ # Only handle simple type values. Use multiple entries or a custom get for dict or lists ++ if isinstance(new_value, list) or isinstance(new_value, dict): ++ continue ++ ++ if new_value is not None: ++ ctx = { ++ placeholder: value_item.get(placeholder, "") ++ for placeholder in placeholders ++ } ++ node_xpath = string.Template(xpath).substitute(ctx) ++ node = get_xml_node(doc, node_xpath) ++ ++ kept_nodes.add(node) ++ ++ get_fn = param.get("get", lambda n: n.text) ++ set_fn = param.get("set", set_node_text) ++ current_value = get_fn(node) ++ ++ # Do we need to apply some conversion to the user-provided value? ++ convert_fn = param.get("convert") ++ if convert_fn: ++ new_value = convert_fn(new_value) ++ ++ if current_value != new_value: ++ set_fn(node, new_value) ++ need_update = True ++ else: ++ nodes = doc.findall(all_nodes_xpath) ++ del_fn = param.get("del", del_text) ++ parent_map = {c: p for p in doc.iter() for c in p} ++ for node in nodes: ++ del_fn(parent_map, node) ++ need_update = True ++ ++ # Clean the left over XML elements if there were placeholders ++ if placeholders and values[0].get("value") != []: ++ all_nodes = set(doc.findall(all_nodes_xpath)) ++ to_remove = all_nodes - kept_nodes ++ del_fn = param.get("del", del_text) ++ parent_map = {c: p for p in doc.iter() for c in p} ++ for node in to_remove: ++ del_fn(parent_map, node) ++ need_update = True ++ ++ return need_update +diff --git a/tests/pytests/unit/utils/test_data.py b/tests/pytests/unit/utils/test_data.py +new file mode 100644 +index 0000000000..b3f0ba04ae +--- /dev/null ++++ b/tests/pytests/unit/utils/test_data.py +@@ -0,0 +1,57 @@ ++import salt.utils.data ++ ++ ++def test_get_value_simple_path(): ++ data = {"a": {"b": {"c": "foo"}}} ++ assert [{"value": "foo"}] == salt.utils.data.get_value(data, "a:b:c") ++ ++ ++def test_get_value_placeholder_dict(): ++ data = {"a": {"b": {"name": "foo"}, "c": {"name": "bar"}}} ++ assert [ ++ {"value": "foo", "id": "b"}, ++ {"value": "bar", "id": "c"}, ++ ] == salt.utils.data.get_value(data, "a:{id}:name") ++ ++ ++def test_get_value_placeholder_list(): ++ data = {"a": [{"name": "foo"}, {"name": "bar"}]} ++ assert [ ++ {"value": "foo", "id": 0}, ++ {"value": "bar", "id": 1}, ++ ] == salt.utils.data.get_value(data, "a:{id}:name") ++ ++ ++def test_get_value_nested_placeholder(): ++ data = { ++ "a": { ++ "b": {"b1": {"name": "foo1"}, "b2": {"name": "foo2"}}, ++ "c": {"c1": {"name": "bar"}}, ++ } ++ } ++ assert [ ++ {"value": "foo1", "id": "b", "sub": "b1"}, ++ {"value": "foo2", "id": "b", "sub": "b2"}, ++ {"value": "bar", "id": "c", "sub": "c1"}, ++ ] == salt.utils.data.get_value(data, "a:{id}:{sub}:name") ++ ++ ++def test_get_value_nested_notfound(): ++ data = {"a": {"b": {"c": "foo"}}} ++ assert [{"value": []}] == salt.utils.data.get_value(data, "a:b:d", []) ++ ++ ++def test_get_value_not_found(): ++ assert [{"value": []}] == salt.utils.data.get_value({}, "a", []) ++ ++ ++def test_get_value_none(): ++ assert [{"value": None}] == salt.utils.data.get_value({"a": None}, "a") ++ ++ ++def test_get_value_simple_type_path(): ++ assert [{"value": []}] == salt.utils.data.get_value({"a": 1024}, "a:b", []) ++ ++ ++def test_get_value_None_path(): ++ assert [{"value": None}] == salt.utils.data.get_value({"a": None}, "a:b", []) +diff --git a/tests/pytests/unit/utils/test_xmlutil.py b/tests/pytests/unit/utils/test_xmlutil.py +new file mode 100644 +index 0000000000..081cc64193 +--- /dev/null ++++ b/tests/pytests/unit/utils/test_xmlutil.py +@@ -0,0 +1,169 @@ ++import pytest ++import salt.utils.xmlutil as xml ++from salt._compat import ElementTree as ET ++ ++ ++@pytest.fixture ++def xml_doc(): ++ return ET.fromstring( ++ """ ++ ++ test01 ++ 1024 ++ ++ ++ ++ ++ ++ ++ ++ """ ++ ) ++ ++ ++def test_change_xml_text(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, {"name": "test02"}, [{"path": "name", "xpath": "name"}] ++ ) ++ assert ret ++ assert "test02" == xml_doc.find("name").text ++ ++ ++def test_change_xml_text_nochange(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, {"name": "test01"}, [{"path": "name", "xpath": "name"}] ++ ) ++ assert not ret ++ ++ ++def test_change_xml_text_notdefined(xml_doc): ++ ret = xml.change_xml(xml_doc, {}, [{"path": "name", "xpath": "name"}]) ++ assert not ret ++ ++ ++def test_change_xml_text_removed(xml_doc): ++ ret = xml.change_xml(xml_doc, {"name": None}, [{"path": "name", "xpath": "name"}]) ++ assert ret ++ assert xml_doc.find("name") is None ++ ++ ++def test_change_xml_text_add(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"vendor": "ACME"}}, ++ [{"path": "cpu:vendor", "xpath": "cpu/vendor"}], ++ ) ++ assert ret ++ assert "ACME" == xml_doc.find("cpu/vendor").text ++ ++ ++def test_change_xml_convert(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"mem": 2}, ++ [{"path": "mem", "xpath": "memory", "convert": lambda v: v * 1024}], ++ ) ++ assert ret ++ assert "2048" == xml_doc.find("memory").text ++ ++ ++def test_change_xml_attr(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"topology": {"cores": 4}}}, ++ [ ++ { ++ "path": "cpu:topology:cores", ++ "xpath": "cpu/topology", ++ "get": lambda n: int(n.get("cores")) if n.get("cores") else None, ++ "set": lambda n, v: n.set("cores", str(v)), ++ "del": xml.del_attribute("cores"), ++ } ++ ], ++ ) ++ assert ret ++ assert "4" == xml_doc.find("cpu/topology").get("cores") ++ ++ ++def test_change_xml_attr_unchanged(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"topology": {"sockets": 1}}}, ++ [ ++ { ++ "path": "cpu:topology:sockets", ++ "xpath": "cpu/topology", ++ "get": lambda n: int(n.get("sockets")) if n.get("sockets") else None, ++ "set": lambda n, v: n.set("sockets", str(v)), ++ "del": xml.del_attribute("sockets"), ++ } ++ ], ++ ) ++ assert not ret ++ ++ ++def test_change_xml_attr_remove(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"topology": {"sockets": None}}}, ++ [ ++ { ++ "path": "cpu:topology:sockets", ++ "xpath": "./cpu/topology", ++ "get": lambda n: int(n.get("sockets")) if n.get("sockets") else None, ++ "set": lambda n, v: n.set("sockets", str(v)), ++ "del": xml.del_attribute("sockets"), ++ } ++ ], ++ ) ++ assert ret ++ assert xml_doc.find("cpu") is None ++ ++ ++def test_change_xml_not_simple_value(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"topology": {"sockets": None}}}, ++ [{"path": "cpu", "xpath": "vcpu", "get": lambda n: int(n.text)}], ++ ) ++ assert not ret ++ ++ ++def test_change_xml_template(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"vcpus": {2: {"enabled": True}, 4: {"enabled": False}}}}, ++ [ ++ { ++ "path": "cpu:vcpus:{id}:enabled", ++ "xpath": "vcpus/vcpu[@id='$id']", ++ "convert": lambda v: "yes" if v else "no", ++ "get": lambda n: n.get("enabled"), ++ "set": lambda n, v: n.set("enabled", v), ++ "del": xml.del_attribute("enabled", ["id"]), ++ }, ++ ], ++ ) ++ assert ret ++ assert xml_doc.find("vcpus/vcpu[@id='1']") is None ++ assert "yes" == xml_doc.find("vcpus/vcpu[@id='2']").get("enabled") ++ assert "no" == xml_doc.find("vcpus/vcpu[@id='4']").get("enabled") ++ ++ ++def test_change_xml_template_remove(xml_doc): ++ ret = xml.change_xml( ++ xml_doc, ++ {"cpu": {"vcpus": None}}, ++ [ ++ { ++ "path": "cpu:vcpus:{id}:enabled", ++ "xpath": "vcpus/vcpu[@id='$id']", ++ "convert": lambda v: "yes" if v else "no", ++ "get": lambda n: n.get("enabled"), ++ "set": lambda n, v: n.set("enabled", v), ++ "del": xml.del_attribute("enabled", ["id"]), ++ }, ++ ], ++ ) ++ assert ret ++ assert xml_doc.find("vcpus") is None +diff --git a/tests/unit/modules/test_virt.py b/tests/unit/modules/test_virt.py +index d3988464f6..5ec8de77e7 100644 +--- a/tests/unit/modules/test_virt.py ++++ b/tests/unit/modules/test_virt.py +@@ -1,4 +1,3 @@ +-# -*- coding: utf-8 -*- + """ + virt execution module unit tests + """ +@@ -6,7 +5,6 @@ virt execution module unit tests + # pylint: disable=3rd-party-module-not-gated + + # Import python libs +-from __future__ import absolute_import, print_function, unicode_literals + + import datetime + import os +@@ -23,9 +21,6 @@ import salt.utils.yaml + from salt._compat import ElementTree as ET + from salt.exceptions import CommandExecutionError, SaltInvocationError + +-# Import third party libs +-from salt.ext import six +- + # pylint: disable=import-error + from salt.ext.six.moves import range # pylint: disable=redefined-builtin + +@@ -136,7 +131,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "model": "virtio", + "filename": "myvm_system.qcow2", + "image": "/path/to/image", +- "source_file": "{0}{1}myvm_system.qcow2".format(root_dir, os.sep), ++ "source_file": "{}{}myvm_system.qcow2".format(root_dir, os.sep), + }, + { + "name": "data", +@@ -145,7 +140,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "format": "raw", + "model": "virtio", + "filename": "myvm_data.raw", +- "source_file": "{0}{1}myvm_data.raw".format(root_dir, os.sep), ++ "source_file": "{}{}myvm_data.raw".format(root_dir, os.sep), + }, + ], + disks, +@@ -582,8 +577,8 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.assertIsNone(root.get("type")) + self.assertEqual(root.find("name").text, "vmname/system.vmdk") + self.assertEqual(root.find("capacity").attrib["unit"], "KiB") +- self.assertEqual(root.find("capacity").text, six.text_type(8192 * 1024)) +- self.assertEqual(root.find("allocation").text, six.text_type(0)) ++ self.assertEqual(root.find("capacity").text, str(8192 * 1024)) ++ self.assertEqual(root.find("allocation").text, str(0)) + self.assertEqual(root.find("target/format").get("type"), "vmdk") + self.assertIsNone(root.find("target/permissions")) + self.assertIsNone(root.find("target/nocow")) +@@ -615,9 +610,9 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.assertIsNone(root.find("target/path")) + self.assertEqual(root.find("target/format").get("type"), "qcow2") + self.assertEqual(root.find("capacity").attrib["unit"], "KiB") +- self.assertEqual(root.find("capacity").text, six.text_type(8192 * 1024)) ++ self.assertEqual(root.find("capacity").text, str(8192 * 1024)) + self.assertEqual(root.find("capacity").attrib["unit"], "KiB") +- self.assertEqual(root.find("allocation").text, six.text_type(4096 * 1024)) ++ self.assertEqual(root.find("allocation").text, str(4096 * 1024)) + self.assertEqual(root.find("target/permissions/mode").text, "0775") + self.assertEqual(root.find("target/permissions/owner").text, "123") + self.assertEqual(root.find("target/permissions/group").text, "456") +@@ -638,7 +633,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + root = ET.fromstring(xml_data) + self.assertEqual(root.attrib["type"], "kvm") + self.assertEqual(root.find("vcpu").text, "1") +- self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").text, str(512 * 1024)) + self.assertEqual(root.find("memory").attrib["unit"], "KiB") + + disks = root.findall(".//disk") +@@ -671,7 +666,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + root = ET.fromstring(xml_data) + self.assertEqual(root.attrib["type"], "vmware") + self.assertEqual(root.find("vcpu").text, "1") +- self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").text, str(512 * 1024)) + self.assertEqual(root.find("memory").attrib["unit"], "KiB") + + disks = root.findall(".//disk") +@@ -714,7 +709,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + root = ET.fromstring(xml_data) + self.assertEqual(root.attrib["type"], "xen") + self.assertEqual(root.find("vcpu").text, "1") +- self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").text, str(512 * 1024)) + self.assertEqual(root.find("memory").attrib["unit"], "KiB") + self.assertEqual( + root.find(".//kernel").text, "/usr/lib/grub2/x86_64-xen/grub.xen" +@@ -768,7 +763,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + root = ET.fromstring(xml_data) + self.assertEqual(root.attrib["type"], "vmware") + self.assertEqual(root.find("vcpu").text, "1") +- self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").text, str(512 * 1024)) + self.assertEqual(root.find("memory").attrib["unit"], "KiB") + self.assertTrue(len(root.findall(".//disk")) == 2) + self.assertTrue(len(root.findall(".//interface")) == 2) +@@ -801,7 +796,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + root = ET.fromstring(xml_data) + self.assertEqual(root.attrib["type"], "kvm") + self.assertEqual(root.find("vcpu").text, "1") +- self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").text, str(512 * 1024)) + self.assertEqual(root.find("memory").attrib["unit"], "KiB") + disks = root.findall(".//disk") + self.assertTrue(len(disks) == 2) +@@ -1635,7 +1630,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.assertIsNone(definition.find("./devices/disk[2]/source")) + self.assertEqual( + mock_run.call_args[0][0], +- 'qemu-img create -f qcow2 "{0}" 10240M'.format(expected_disk_path), ++ 'qemu-img create -f qcow2 "{}" 10240M'.format(expected_disk_path), + ) + self.assertEqual(mock_chmod.call_args[0][0], expected_disk_path) + +@@ -1729,11 +1724,12 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + 1 + + hvm ++ + + + + +- ++ + + + +@@ -1850,17 +1846,36 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "cmdline": "console=ttyS0 ks=http://example.com/f8-i386/os/", + } + +- boot_uefi = { +- "loader": "/usr/share/OVMF/OVMF_CODE.fd", +- "nvram": "/usr/share/OVMF/OVMF_VARS.ms.fd", +- } ++ # Update boot devices case ++ define_mock.reset_mock() ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", boot_dev="cdrom network hd"), ++ ) ++ setxml = ET.fromstring(define_mock.call_args[0][0]) ++ self.assertEqual( ++ ["cdrom", "network", "hd"], ++ [node.get("dev") for node in setxml.findall("os/boot")], ++ ) + +- invalid_boot = { +- "loader": "/usr/share/OVMF/OVMF_CODE.fd", +- "initrd": "/root/f8-i386-initrd", +- } ++ # Update unchanged boot devices case ++ define_mock.reset_mock() ++ self.assertEqual( ++ { ++ "definition": False, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", boot_dev="hd"), ++ ) ++ define_mock.assert_not_called() + + # Update with boot parameter case ++ define_mock.reset_mock() + self.assertEqual( + { + "definition": True, +@@ -1884,6 +1899,11 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "console=ttyS0 ks=http://example.com/f8-i386/os/", + ) + ++ boot_uefi = { ++ "loader": "/usr/share/OVMF/OVMF_CODE.fd", ++ "nvram": "/usr/share/OVMF/OVMF_VARS.ms.fd", ++ } ++ + self.assertEqual( + { + "definition": True, +@@ -1903,9 +1923,28 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "/usr/share/OVMF/OVMF_VARS.ms.fd", + ) + ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", boot={"efi": True}), ++ ) ++ setxml = ET.fromstring(define_mock.call_args[0][0]) ++ self.assertEqual(setxml.find("os").attrib.get("firmware"), "efi") ++ ++ invalid_boot = { ++ "loader": "/usr/share/OVMF/OVMF_CODE.fd", ++ "initrd": "/root/f8-i386-initrd", ++ } ++ + with self.assertRaises(SaltInvocationError): + virt.update("my_vm", boot=invalid_boot) + ++ with self.assertRaises(SaltInvocationError): ++ virt.update("my_vm", boot={"efi": "Not a boolean value"}) ++ + # Update memory case + setmem_mock = MagicMock(return_value=0) + domain_mock.setMemoryFlags = setmem_mock +@@ -1955,7 +1994,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + ) # pylint: disable=no-member + self.assertEqual( + mock_run.call_args[0][0], +- 'qemu-img create -f qcow2 "{0}" 2048M'.format(added_disk_path), ++ 'qemu-img create -f qcow2 "{}" 2048M'.format(added_disk_path), + ) + self.assertEqual(mock_chmod.call_args[0][0], added_disk_path) + self.assertListEqual( +@@ -2397,6 +2436,43 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + ], + ) + ++ def test_update_xen_boot_params(self): ++ """ ++ Test virt.update() a Xen definition no boot parameter. ++ """ ++ root_dir = os.path.join(salt.syspaths.ROOT_DIR, "srv", "salt-images") ++ xml_boot = """ ++ ++ vm ++ 1048576 ++ 1048576 ++ 1 ++ ++ hvm ++ /usr/lib/xen/boot/hvmloader ++ ++ ++ """ ++ domain_mock_boot = self.set_mock_vm("vm", xml_boot) ++ domain_mock_boot.OSType = MagicMock(return_value="hvm") ++ define_mock_boot = MagicMock(return_value=True) ++ define_mock_boot.setVcpusFlags = MagicMock(return_value=0) ++ self.mock_conn.defineXML = define_mock_boot ++ self.assertEqual( ++ { ++ "cpu": False, ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("vm", cpu=2), ++ ) ++ setxml = ET.fromstring(define_mock_boot.call_args[0][0]) ++ self.assertEqual(setxml.find("os").find("loader").attrib.get("type"), "rom") ++ self.assertEqual( ++ setxml.find("os").find("loader").text, "/usr/lib/xen/boot/hvmloader" ++ ) ++ + def test_update_existing_boot_params(self): + """ + Test virt.update() with existing boot parameters. +@@ -2537,6 +2613,18 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.assertEqual(setxml.find("os").find("initrd"), None) + self.assertEqual(setxml.find("os").find("cmdline"), None) + ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("vm_with_boot_param", boot={"efi": False}), ++ ) ++ setxml = ET.fromstring(define_mock_boot.call_args[0][0]) ++ self.assertEqual(setxml.find("os").find("nvram"), None) ++ self.assertEqual(setxml.find("os").find("loader"), None) ++ + self.assertEqual( + { + "definition": True, +@@ -2582,7 +2670,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + salt.modules.config.__opts__, mock_config # pylint: disable=no-member + ): + +- for name in six.iterkeys(mock_config["virt"]["nic"]): ++ for name in mock_config["virt"]["nic"].keys(): + profile = salt.modules.virt._nic_profile(name, "kvm") + self.assertEqual(len(profile), 2) + +@@ -3592,8 +3680,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + "44454c4c-3400-105a-8033-b3c04f4b344a", caps["host"]["host"]["uuid"] + ) + self.assertEqual( +- set(["qemu", "kvm"]), +- set([domainCaps["domain"] for domainCaps in caps["domains"]]), ++ {"qemu", "kvm"}, {domainCaps["domain"] for domainCaps in caps["domains"]}, + ) + + def test_network_tag(self): +@@ -3694,9 +3781,9 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + for i in range(2): + net_mock = MagicMock() + +- net_mock.name.return_value = "net{0}".format(i) ++ net_mock.name.return_value = "net{}".format(i) + net_mock.UUIDString.return_value = "some-uuid" +- net_mock.bridgeName.return_value = "br{0}".format(i) ++ net_mock.bridgeName.return_value = "br{}".format(i) + net_mock.autostart.return_value = True + net_mock.isActive.return_value = False + net_mock.isPersistent.return_value = True +@@ -4156,8 +4243,8 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + pool_mocks = [] + for i in range(2): + pool_mock = MagicMock() +- pool_mock.name.return_value = "pool{0}".format(i) +- pool_mock.UUIDString.return_value = "some-uuid-{0}".format(i) ++ pool_mock.name.return_value = "pool{}".format(i) ++ pool_mock.UUIDString.return_value = "some-uuid-{}".format(i) + pool_mock.info.return_value = [0, 1234, 5678, 123] + pool_mock.autostart.return_value = True + pool_mock.isPersistent.return_value = True +@@ -4257,7 +4344,6 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + """ + mock_pool = MagicMock() + mock_pool.delete = MagicMock(return_value=0) +- mock_pool.XMLDesc.return_value = "" + self.mock_conn.storagePoolLookupByName = MagicMock(return_value=mock_pool) + + res = virt.pool_delete("test-pool") +@@ -4271,12 +4357,12 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.mock_libvirt.VIR_STORAGE_POOL_DELETE_NORMAL + ) + +- def test_pool_delete_secret(self): ++ def test_pool_undefine_secret(self): + """ +- Test virt.pool_delete function where the pool has a secret ++ Test virt.pool_undefine function where the pool has a secret + """ + mock_pool = MagicMock() +- mock_pool.delete = MagicMock(return_value=0) ++ mock_pool.undefine = MagicMock(return_value=0) + mock_pool.XMLDesc.return_value = """ + + test-ses +@@ -4293,16 +4379,11 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + mock_undefine = MagicMock(return_value=0) + self.mock_conn.secretLookupByUsage.return_value.undefine = mock_undefine + +- res = virt.pool_delete("test-ses") ++ res = virt.pool_undefine("test-ses") + self.assertTrue(res) + + self.mock_conn.storagePoolLookupByName.assert_called_once_with("test-ses") +- +- # Shouldn't be called with another parameter so far since those are not implemented +- # and thus throwing exceptions. +- mock_pool.delete.assert_called_once_with( +- self.mock_libvirt.VIR_STORAGE_POOL_DELETE_NORMAL +- ) ++ mock_pool.undefine.assert_called_once_with() + + self.mock_conn.secretLookupByUsage.assert_called_once_with( + self.mock_libvirt.VIR_SECRET_USAGE_TYPE_CEPH, "pool_test-ses" +@@ -4571,24 +4652,6 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + """ + +- expected_xml = ( +- '' +- "default" +- "20fbe05c-ab40-418a-9afa-136d512f0ede" +- '1999421108224' +- '713207042048' +- '1286214066176' +- "" +- '' +- '' +- '' +- '' +- "" +- "iscsi-images" +- "" +- "" +- ) +- + mock_secret = MagicMock() + self.mock_conn.secretLookupByUUIDString = MagicMock(return_value=mock_secret) + +@@ -4609,6 +4672,23 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + self.mock_conn.storagePoolDefineXML.assert_not_called() + mock_secret.setValue.assert_called_once_with(b"secret") + ++ # Case where the secret can't be found ++ self.mock_conn.secretLookupByUUIDString = MagicMock( ++ side_effect=self.mock_libvirt.libvirtError("secret not found") ++ ) ++ self.assertFalse( ++ virt.pool_update( ++ "default", ++ "rbd", ++ source_name="iscsi-images", ++ source_hosts=["ses4.tf.local", "ses5.tf.local"], ++ source_auth={"username": "libvirt", "password": "c2VjcmV0"}, ++ ) ++ ) ++ self.mock_conn.storagePoolDefineXML.assert_not_called() ++ self.mock_conn.secretDefineXML.assert_called_once() ++ mock_secret.setValue.assert_called_once_with(b"secret") ++ + def test_pool_update_password_create(self): + """ + Test the pool_update function, where the password only is changed +@@ -4695,11 +4775,11 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + for idx, disk in enumerate(vms_disks): + vm = MagicMock() + # pylint: disable=no-member +- vm.name.return_value = "vm{0}".format(idx) ++ vm.name.return_value = "vm{}".format(idx) + vm.XMLDesc.return_value = """ + +- vm{0} +- {1} ++ vm{} ++ {} + + """.format( + idx, disk +@@ -4760,7 +4840,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + # pylint: disable=no-member + mock_volume.name.return_value = vol_data["name"] + mock_volume.key.return_value = vol_data["key"] +- mock_volume.path.return_value = "/path/to/{0}.qcow2".format( ++ mock_volume.path.return_value = "/path/to/{}.qcow2".format( + vol_data["name"] + ) + if vol_data["info"]: +@@ -4769,7 +4849,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + """ + + +- {0} ++ {} + + """.format( + vol_data["backingStore"] +@@ -5234,7 +5314,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + def create_mock_vm(idx): + mock_vm = MagicMock() +- mock_vm.name.return_value = "vm{0}".format(idx) ++ mock_vm.name.return_value = "vm{}".format(idx) + return mock_vm + + mock_vms = [create_mock_vm(idx) for idx in range(3)] +diff --git a/tests/unit/states/test_virt.py b/tests/unit/states/test_virt.py +index c76f8a5fc0..f03159334b 100644 +--- a/tests/unit/states/test_virt.py ++++ b/tests/unit/states/test_virt.py +@@ -1,9 +1,7 @@ +-# -*- coding: utf-8 -*- + """ + :codeauthor: Jayesh Kariya + """ + # Import Python libs +-from __future__ import absolute_import, print_function, unicode_literals + + import shutil + import tempfile +@@ -14,7 +12,6 @@ import salt.utils.files + from salt.exceptions import CommandExecutionError, SaltInvocationError + + # Import 3rd-party libs +-from salt.ext import six + from tests.support.mixins import LoaderModuleMockMixin + from tests.support.mock import MagicMock, mock_open, patch + +@@ -37,7 +34,7 @@ class LibvirtMock(MagicMock): # pylint: disable=too-many-ancestors + """ + Fake function return error message + """ +- return six.text_type(self) ++ return str(self) + + + class LibvirtTestCase(TestCase, LoaderModuleMockMixin): +@@ -341,6 +338,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + "myvm", + cpu=2, + mem=2048, ++ boot_dev="cdrom hd", + os_type="linux", + arch="i686", + vm_type="qemu", +@@ -363,6 +361,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + "myvm", + cpu=2, + mem=2048, ++ boot_dev="cdrom hd", + os_type="linux", + arch="i686", + disk="prod", +@@ -471,10 +470,13 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + "comment": "Domain myvm updated with live update(s) failures", + } + ) +- self.assertDictEqual(virt.defined("myvm", cpu=2), ret) ++ self.assertDictEqual( ++ virt.defined("myvm", cpu=2, boot_dev="cdrom hd"), ret ++ ) + update_mock.assert_called_with( + "myvm", + cpu=2, ++ boot_dev="cdrom hd", + mem=None, + disk_profile=None, + disks=None, +@@ -598,6 +600,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password=None, + boot=None, + test=True, ++ boot_dev=None, + ) + + # No changes case +@@ -632,6 +635,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password=None, + boot=None, + test=True, ++ boot_dev=None, + ) + + def test_running(self): +@@ -708,6 +712,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + install=True, + pub_key=None, + priv_key=None, ++ boot_dev=None, + connection=None, + username=None, + password=None, +@@ -769,6 +774,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + install=False, + pub_key="/path/to/key.pub", + priv_key="/path/to/key", ++ boot_dev="network hd", + connection="someconnection", + username="libvirtuser", + password="supersecret", +@@ -793,6 +799,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + start=False, + pub_key="/path/to/key.pub", + priv_key="/path/to/key", ++ boot_dev="network hd", + connection="someconnection", + username="libvirtuser", + password="supersecret", +@@ -937,6 +944,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password=None, + boot=None, + test=False, ++ boot_dev=None, + ) + + # Failed definition update case +@@ -1055,6 +1063,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password=None, + boot=None, + test=True, ++ boot_dev=None, + ) + start_mock.assert_not_called() + +@@ -1091,6 +1100,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password=None, + boot=None, + test=True, ++ boot_dev=None, + ) + + def test_stopped(self): +@@ -1978,6 +1988,72 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + password="secret", + ) + ++ # Define a pool that doesn't handle build ++ for mock in mocks: ++ mocks[mock].reset_mock() ++ with patch.dict( ++ virt.__salt__, ++ { # pylint: disable=no-member ++ "virt.pool_info": MagicMock( ++ side_effect=[ ++ {}, ++ {"mypool": {"state": "stopped", "autostart": True}}, ++ ] ++ ), ++ "virt.pool_define": mocks["define"], ++ "virt.pool_build": mocks["build"], ++ "virt.pool_set_autostart": mocks["autostart"], ++ }, ++ ): ++ ret.update( ++ { ++ "changes": {"mypool": "Pool defined, marked for autostart"}, ++ "comment": "Pool mypool defined, marked for autostart", ++ } ++ ) ++ self.assertDictEqual( ++ virt.pool_defined( ++ "mypool", ++ ptype="rbd", ++ source={ ++ "name": "libvirt-pool", ++ "hosts": ["ses2.tf.local", "ses3.tf.local"], ++ "auth": { ++ "username": "libvirt", ++ "password": "AQAz+PRdtquBBRAASMv7nlMZYfxIyLw3St65Xw==", ++ }, ++ }, ++ autostart=True, ++ ), ++ ret, ++ ) ++ mocks["define"].assert_called_with( ++ "mypool", ++ ptype="rbd", ++ target=None, ++ permissions=None, ++ source_devices=None, ++ source_dir=None, ++ source_adapter=None, ++ source_hosts=["ses2.tf.local", "ses3.tf.local"], ++ source_auth={ ++ "username": "libvirt", ++ "password": "AQAz+PRdtquBBRAASMv7nlMZYfxIyLw3St65Xw==", ++ }, ++ source_name="libvirt-pool", ++ source_format=None, ++ source_initiator=None, ++ start=False, ++ transient=False, ++ connection=None, ++ username=None, ++ password=None, ++ ) ++ mocks["autostart"].assert_called_with( ++ "mypool", state="on", connection=None, username=None, password=None, ++ ) ++ mocks["build"].assert_not_called() ++ + mocks["update"] = MagicMock(return_value=False) + for mock in mocks: + mocks[mock].reset_mock() +@@ -2027,6 +2103,9 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + for mock in mocks: + mocks[mock].reset_mock() + mocks["update"] = MagicMock(return_value=True) ++ mocks["build"] = MagicMock( ++ side_effect=self.mock_libvirt.libvirtError("Existing VG") ++ ) + with patch.dict( + virt.__salt__, + { # pylint: disable=no-member +@@ -2130,6 +2209,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + ), + ret, + ) ++ mocks["build"].assert_not_called() + mocks["update"].assert_called_with( + "mypool", + ptype="logical", +@@ -2477,8 +2557,8 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + ): + ret.update( + { +- "changes": {"mypool": "Pool updated, built, restarted"}, +- "comment": "Pool mypool updated, built, restarted", ++ "changes": {"mypool": "Pool updated, restarted"}, ++ "comment": "Pool mypool updated, restarted", + "result": True, + } + ) +@@ -2504,9 +2584,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): + mocks["start"].assert_called_with( + "mypool", connection=None, username=None, password=None + ) +- mocks["build"].assert_called_with( +- "mypool", connection=None, username=None, password=None +- ) ++ mocks["build"].assert_not_called() + mocks["update"].assert_called_with( + "mypool", + ptype="logical", +diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py +index 8fa352321c..8a6956d442 100644 +--- a/tests/unit/utils/test_data.py ++++ b/tests/unit/utils/test_data.py +@@ -1,38 +1,38 @@ +-# -*- coding: utf-8 -*- +-''' ++""" + Tests for salt.utils.data +-''' ++""" + + # Import Python libs +-from __future__ import absolute_import, print_function, unicode_literals ++ + import logging + + # Import Salt libs + import salt.utils.data + import salt.utils.stringutils +-from salt.utils.odict import OrderedDict +-from tests.support.unit import TestCase, LOREM_IPSUM +-from tests.support.mock import patch + + # Import 3rd party libs +-from salt.ext.six.moves import builtins # pylint: disable=import-error,redefined-builtin +-from salt.ext import six ++from salt.ext.six.moves import ( # pylint: disable=import-error,redefined-builtin ++ builtins, ++) ++from salt.utils.odict import OrderedDict ++from tests.support.mock import patch ++from tests.support.unit import LOREM_IPSUM, TestCase + + log = logging.getLogger(__name__) +-_b = lambda x: x.encode('utf-8') ++_b = lambda x: x.encode("utf-8") + _s = lambda x: salt.utils.stringutils.to_str(x, normalize=True) + # Some randomized data that will not decode +-BYTES = b'1\x814\x10' ++BYTES = b"1\x814\x10" + + # This is an example of a unicode string with й constructed using two separate + # code points. Do not modify it. +-EGGS = '\u044f\u0438\u0306\u0446\u0430' ++EGGS = "\u044f\u0438\u0306\u0446\u0430" + + + class DataTestCase(TestCase): + test_data = [ +- 'unicode_str', +- _b('питон'), ++ "unicode_str", ++ _b("питон"), + 123, + 456.789, + True, +@@ -40,71 +40,79 @@ class DataTestCase(TestCase): + None, + EGGS, + BYTES, +- [123, 456.789, _b('спам'), True, False, None, EGGS, BYTES], +- (987, 654.321, _b('яйца'), EGGS, None, (True, EGGS, BYTES)), +- {_b('str_key'): _b('str_val'), +- None: True, +- 123: 456.789, +- EGGS: BYTES, +- _b('subdict'): {'unicode_key': EGGS, +- _b('tuple'): (123, 'hello', _b('world'), True, EGGS, BYTES), +- _b('list'): [456, _b('спам'), False, EGGS, BYTES]}}, +- OrderedDict([(_b('foo'), 'bar'), (123, 456), (EGGS, BYTES)]) ++ [123, 456.789, _b("спам"), True, False, None, EGGS, BYTES], ++ (987, 654.321, _b("яйца"), EGGS, None, (True, EGGS, BYTES)), ++ { ++ _b("str_key"): _b("str_val"), ++ None: True, ++ 123: 456.789, ++ EGGS: BYTES, ++ _b("subdict"): { ++ "unicode_key": EGGS, ++ _b("tuple"): (123, "hello", _b("world"), True, EGGS, BYTES), ++ _b("list"): [456, _b("спам"), False, EGGS, BYTES], ++ }, ++ }, ++ OrderedDict([(_b("foo"), "bar"), (123, 456), (EGGS, BYTES)]), + ] + + def test_sorted_ignorecase(self): +- test_list = ['foo', 'Foo', 'bar', 'Bar'] +- expected_list = ['bar', 'Bar', 'foo', 'Foo'] +- self.assertEqual( +- salt.utils.data.sorted_ignorecase(test_list), expected_list) ++ test_list = ["foo", "Foo", "bar", "Bar"] ++ expected_list = ["bar", "Bar", "foo", "Foo"] ++ self.assertEqual(salt.utils.data.sorted_ignorecase(test_list), expected_list) + + def test_mysql_to_dict(self): +- test_mysql_output = ['+----+------+-----------+------+---------+------+-------+------------------+', +- '| Id | User | Host | db | Command | Time | State | Info |', +- '+----+------+-----------+------+---------+------+-------+------------------+', +- '| 7 | root | localhost | NULL | Query | 0 | init | show processlist |', +- '+----+------+-----------+------+---------+------+-------+------------------+'] ++ test_mysql_output = [ ++ "+----+------+-----------+------+---------+------+-------+------------------+", ++ "| Id | User | Host | db | Command | Time | State | Info |", ++ "+----+------+-----------+------+---------+------+-------+------------------+", ++ "| 7 | root | localhost | NULL | Query | 0 | init | show processlist |", ++ "+----+------+-----------+------+---------+------+-------+------------------+", ++ ] + +- ret = salt.utils.data.mysql_to_dict(test_mysql_output, 'Info') ++ ret = salt.utils.data.mysql_to_dict(test_mysql_output, "Info") + expected_dict = { +- 'show processlist': {'Info': 'show processlist', 'db': 'NULL', 'State': 'init', 'Host': 'localhost', +- 'Command': 'Query', 'User': 'root', 'Time': 0, 'Id': 7}} ++ "show processlist": { ++ "Info": "show processlist", ++ "db": "NULL", ++ "State": "init", ++ "Host": "localhost", ++ "Command": "Query", ++ "User": "root", ++ "Time": 0, ++ "Id": 7, ++ } ++ } + + self.assertDictEqual(ret, expected_dict) + + def test_subdict_match(self): +- test_two_level_dict = {'foo': {'bar': 'baz'}} +- test_two_level_comb_dict = {'foo': {'bar': 'baz:woz'}} ++ test_two_level_dict = {"foo": {"bar": "baz"}} ++ test_two_level_comb_dict = {"foo": {"bar": "baz:woz"}} + test_two_level_dict_and_list = { +- 'abc': ['def', 'ghi', {'lorem': {'ipsum': [{'dolor': 'sit'}]}}], ++ "abc": ["def", "ghi", {"lorem": {"ipsum": [{"dolor": "sit"}]}}], + } +- test_three_level_dict = {'a': {'b': {'c': 'v'}}} ++ test_three_level_dict = {"a": {"b": {"c": "v"}}} + + self.assertTrue( +- salt.utils.data.subdict_match( +- test_two_level_dict, 'foo:bar:baz' +- ) ++ salt.utils.data.subdict_match(test_two_level_dict, "foo:bar:baz") + ) + # In test_two_level_comb_dict, 'foo:bar' corresponds to 'baz:woz', not + # 'baz'. This match should return False. + self.assertFalse( +- salt.utils.data.subdict_match( +- test_two_level_comb_dict, 'foo:bar:baz' +- ) ++ salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz") + ) + # This tests matching with the delimiter in the value part (in other + # words, that the path 'foo:bar' corresponds to the string 'baz:woz'). + self.assertTrue( +- salt.utils.data.subdict_match( +- test_two_level_comb_dict, 'foo:bar:baz:woz' +- ) ++ salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz:woz") + ) + # This would match if test_two_level_comb_dict['foo']['bar'] was equal + # to 'baz:woz:wiz', or if there was more deep nesting. But it does not, + # so this should return False. + self.assertFalse( + salt.utils.data.subdict_match( +- test_two_level_comb_dict, 'foo:bar:baz:woz:wiz' ++ test_two_level_comb_dict, "foo:bar:baz:woz:wiz" + ) + ) + # This tests for cases when a key path corresponds to a list. The +@@ -115,189 +123,171 @@ class DataTestCase(TestCase): + # salt.utils.traverse_list_and_dict() so this particular assertion is a + # sanity check. + self.assertTrue( +- salt.utils.data.subdict_match( +- test_two_level_dict_and_list, 'abc:ghi' +- ) ++ salt.utils.data.subdict_match(test_two_level_dict_and_list, "abc:ghi") + ) + # This tests the use case of a dict embedded in a list, embedded in a + # list, embedded in a dict. This is a rather absurd case, but it + # confirms that match recursion works properly. + self.assertTrue( + salt.utils.data.subdict_match( +- test_two_level_dict_and_list, 'abc:lorem:ipsum:dolor:sit' ++ test_two_level_dict_and_list, "abc:lorem:ipsum:dolor:sit" + ) + ) + # Test four level dict match for reference +- self.assertTrue( +- salt.utils.data.subdict_match( +- test_three_level_dict, 'a:b:c:v' +- ) +- ) ++ self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:b:c:v")) + # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v' +- self.assertFalse( +- salt.utils.data.subdict_match( +- test_three_level_dict, 'a:c:v' +- ) +- ) ++ self.assertFalse(salt.utils.data.subdict_match(test_three_level_dict, "a:c:v")) + # Test wildcard match +- self.assertTrue( +- salt.utils.data.subdict_match( +- test_three_level_dict, 'a:*:c:v' +- ) +- ) ++ self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:*:c:v")) + + def test_subdict_match_with_wildcards(self): +- ''' ++ """ + Tests subdict matching when wildcards are used in the expression +- ''' +- data = { +- 'a': { +- 'b': { +- 'ç': 'd', +- 'é': ['eff', 'gee', '8ch'], +- 'ĩ': {'j': 'k'} +- } +- } +- } +- assert salt.utils.data.subdict_match(data, '*:*:*:*') +- assert salt.utils.data.subdict_match(data, 'a:*:*:*') +- assert salt.utils.data.subdict_match(data, 'a:b:*:*') +- assert salt.utils.data.subdict_match(data, 'a:b:ç:*') +- assert salt.utils.data.subdict_match(data, 'a:b:*:d') +- assert salt.utils.data.subdict_match(data, 'a:*:ç:d') +- assert salt.utils.data.subdict_match(data, '*:b:ç:d') +- assert salt.utils.data.subdict_match(data, '*:*:ç:d') +- assert salt.utils.data.subdict_match(data, '*:*:*:d') +- assert salt.utils.data.subdict_match(data, 'a:*:*:d') +- assert salt.utils.data.subdict_match(data, 'a:b:*:ef*') +- assert salt.utils.data.subdict_match(data, 'a:b:*:g*') +- assert salt.utils.data.subdict_match(data, 'a:b:*:j:*') +- assert salt.utils.data.subdict_match(data, 'a:b:*:j:k') +- assert salt.utils.data.subdict_match(data, 'a:b:*:*:k') +- assert salt.utils.data.subdict_match(data, 'a:b:*:*:*') ++ """ ++ data = {"a": {"b": {"ç": "d", "é": ["eff", "gee", "8ch"], "ĩ": {"j": "k"}}}} ++ assert salt.utils.data.subdict_match(data, "*:*:*:*") ++ assert salt.utils.data.subdict_match(data, "a:*:*:*") ++ assert salt.utils.data.subdict_match(data, "a:b:*:*") ++ assert salt.utils.data.subdict_match(data, "a:b:ç:*") ++ assert salt.utils.data.subdict_match(data, "a:b:*:d") ++ assert salt.utils.data.subdict_match(data, "a:*:ç:d") ++ assert salt.utils.data.subdict_match(data, "*:b:ç:d") ++ assert salt.utils.data.subdict_match(data, "*:*:ç:d") ++ assert salt.utils.data.subdict_match(data, "*:*:*:d") ++ assert salt.utils.data.subdict_match(data, "a:*:*:d") ++ assert salt.utils.data.subdict_match(data, "a:b:*:ef*") ++ assert salt.utils.data.subdict_match(data, "a:b:*:g*") ++ assert salt.utils.data.subdict_match(data, "a:b:*:j:*") ++ assert salt.utils.data.subdict_match(data, "a:b:*:j:k") ++ assert salt.utils.data.subdict_match(data, "a:b:*:*:k") ++ assert salt.utils.data.subdict_match(data, "a:b:*:*:*") + + def test_traverse_dict(self): +- test_two_level_dict = {'foo': {'bar': 'baz'}} ++ test_two_level_dict = {"foo": {"bar": "baz"}} + + self.assertDictEqual( +- {'not_found': 'nope'}, ++ {"not_found": "nope"}, + salt.utils.data.traverse_dict( +- test_two_level_dict, 'foo:bar:baz', {'not_found': 'nope'} +- ) ++ test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} ++ ), + ) + self.assertEqual( +- 'baz', ++ "baz", + salt.utils.data.traverse_dict( +- test_two_level_dict, 'foo:bar', {'not_found': 'not_found'} +- ) ++ test_two_level_dict, "foo:bar", {"not_found": "not_found"} ++ ), + ) + + def test_traverse_dict_and_list(self): +- test_two_level_dict = {'foo': {'bar': 'baz'}} ++ test_two_level_dict = {"foo": {"bar": "baz"}} + test_two_level_dict_and_list = { +- 'foo': ['bar', 'baz', {'lorem': {'ipsum': [{'dolor': 'sit'}]}}] ++ "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}] + } + + # Check traversing too far: salt.utils.data.traverse_dict_and_list() returns + # the value corresponding to a given key path, and baz is a value + # corresponding to the key path foo:bar. + self.assertDictEqual( +- {'not_found': 'nope'}, ++ {"not_found": "nope"}, + salt.utils.data.traverse_dict_and_list( +- test_two_level_dict, 'foo:bar:baz', {'not_found': 'nope'} +- ) ++ test_two_level_dict, "foo:bar:baz", {"not_found": "nope"} ++ ), + ) + # Now check to ensure that foo:bar corresponds to baz + self.assertEqual( +- 'baz', ++ "baz", + salt.utils.data.traverse_dict_and_list( +- test_two_level_dict, 'foo:bar', {'not_found': 'not_found'} +- ) ++ test_two_level_dict, "foo:bar", {"not_found": "not_found"} ++ ), + ) + # Check traversing too far + self.assertDictEqual( +- {'not_found': 'nope'}, ++ {"not_found": "nope"}, + salt.utils.data.traverse_dict_and_list( +- test_two_level_dict_and_list, 'foo:bar', {'not_found': 'nope'} +- ) ++ test_two_level_dict_and_list, "foo:bar", {"not_found": "nope"} ++ ), + ) + # Check index 1 (2nd element) of list corresponding to path 'foo' + self.assertEqual( +- 'baz', ++ "baz", + salt.utils.data.traverse_dict_and_list( +- test_two_level_dict_and_list, 'foo:1', {'not_found': 'not_found'} +- ) ++ test_two_level_dict_and_list, "foo:1", {"not_found": "not_found"} ++ ), + ) + # Traverse a couple times into dicts embedded in lists + self.assertEqual( +- 'sit', ++ "sit", + salt.utils.data.traverse_dict_and_list( + test_two_level_dict_and_list, +- 'foo:lorem:ipsum:dolor', +- {'not_found': 'not_found'} +- ) ++ "foo:lorem:ipsum:dolor", ++ {"not_found": "not_found"}, ++ ), + ) + + def test_compare_dicts(self): +- ret = salt.utils.data.compare_dicts(old={'foo': 'bar'}, new={'foo': 'bar'}) ++ ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "bar"}) + self.assertEqual(ret, {}) + +- ret = salt.utils.data.compare_dicts(old={'foo': 'bar'}, new={'foo': 'woz'}) +- expected_ret = {'foo': {'new': 'woz', 'old': 'bar'}} ++ ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "woz"}) ++ expected_ret = {"foo": {"new": "woz", "old": "bar"}} + self.assertDictEqual(ret, expected_ret) + + def test_compare_lists_no_change(self): +- ret = salt.utils.data.compare_lists(old=[1, 2, 3, 'a', 'b', 'c'], +- new=[1, 2, 3, 'a', 'b', 'c']) ++ ret = salt.utils.data.compare_lists( ++ old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3, "a", "b", "c"] ++ ) + expected = {} + self.assertDictEqual(ret, expected) + + def test_compare_lists_changes(self): +- ret = salt.utils.data.compare_lists(old=[1, 2, 3, 'a', 'b', 'c'], +- new=[1, 2, 4, 'x', 'y', 'z']) +- expected = {'new': [4, 'x', 'y', 'z'], 'old': [3, 'a', 'b', 'c']} ++ ret = salt.utils.data.compare_lists( ++ old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 4, "x", "y", "z"] ++ ) ++ expected = {"new": [4, "x", "y", "z"], "old": [3, "a", "b", "c"]} + self.assertDictEqual(ret, expected) + + def test_compare_lists_changes_new(self): +- ret = salt.utils.data.compare_lists(old=[1, 2, 3], +- new=[1, 2, 3, 'x', 'y', 'z']) +- expected = {'new': ['x', 'y', 'z']} ++ ret = salt.utils.data.compare_lists(old=[1, 2, 3], new=[1, 2, 3, "x", "y", "z"]) ++ expected = {"new": ["x", "y", "z"]} + self.assertDictEqual(ret, expected) + + def test_compare_lists_changes_old(self): +- ret = salt.utils.data.compare_lists(old=[1, 2, 3, 'a', 'b', 'c'], +- new=[1, 2, 3]) +- expected = {'old': ['a', 'b', 'c']} ++ ret = salt.utils.data.compare_lists(old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3]) ++ expected = {"old": ["a", "b", "c"]} + self.assertDictEqual(ret, expected) + + def test_decode(self): +- ''' ++ """ + Companion to test_decode_to_str, they should both be kept up-to-date + with one another. + + NOTE: This uses the lambda "_b" defined above in the global scope, + which encodes a string to a bytestring, assuming utf-8. +- ''' ++ """ + expected = [ +- 'unicode_str', +- 'питон', ++ "unicode_str", ++ "питон", + 123, + 456.789, + True, + False, + None, +- 'яйца', ++ "яйца", + BYTES, +- [123, 456.789, 'спам', True, False, None, 'яйца', BYTES], +- (987, 654.321, 'яйца', 'яйца', None, (True, 'яйца', BYTES)), +- {'str_key': 'str_val', +- None: True, +- 123: 456.789, +- 'яйца': BYTES, +- 'subdict': {'unicode_key': 'яйца', +- 'tuple': (123, 'hello', 'world', True, 'яйца', BYTES), +- 'list': [456, 'спам', False, 'яйца', BYTES]}}, +- OrderedDict([('foo', 'bar'), (123, 456), ('яйца', BYTES)]) ++ [123, 456.789, "спам", True, False, None, "яйца", BYTES], ++ (987, 654.321, "яйца", "яйца", None, (True, "яйца", BYTES)), ++ { ++ "str_key": "str_val", ++ None: True, ++ 123: 456.789, ++ "яйца": BYTES, ++ "subdict": { ++ "unicode_key": "яйца", ++ "tuple": (123, "hello", "world", True, "яйца", BYTES), ++ "list": [456, "спам", False, "яйца", BYTES], ++ }, ++ }, ++ OrderedDict([("foo", "bar"), (123, 456), ("яйца", BYTES)]), + ] + + ret = salt.utils.data.decode( +@@ -305,7 +295,8 @@ class DataTestCase(TestCase): + keep=True, + normalize=True, + preserve_dict_class=True, +- preserve_tuples=True) ++ preserve_tuples=True, ++ ) + self.assertEqual(ret, expected) + + # The binary data in the data structure should fail to decode, even +@@ -317,74 +308,100 @@ class DataTestCase(TestCase): + keep=False, + normalize=True, + preserve_dict_class=True, +- preserve_tuples=True) ++ preserve_tuples=True, ++ ) + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples +- expected[10] = [987, 654.321, 'яйца', 'яйца', None, [True, 'яйца', BYTES]] +- expected[11]['subdict']['tuple'] = [123, 'hello', 'world', True, 'яйца', BYTES] +- expected[12] = {'foo': 'bar', 123: 456, 'яйца': BYTES} ++ expected[10] = [987, 654.321, "яйца", "яйца", None, [True, "яйца", BYTES]] ++ expected[11]["subdict"]["tuple"] = [123, "hello", "world", True, "яйца", BYTES] ++ expected[12] = {"foo": "bar", 123: 456, "яйца": BYTES} + + ret = salt.utils.data.decode( + self.test_data, + keep=True, + normalize=True, + preserve_dict_class=False, +- preserve_tuples=False) ++ preserve_tuples=False, ++ ) + self.assertEqual(ret, expected) + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): +- log.debug('Testing decode of %s', item) ++ log.debug("Testing decode of %s", item) + self.assertEqual(salt.utils.data.decode(item), item) + + # Test single strings (not in a data structure) +- self.assertEqual(salt.utils.data.decode('foo'), 'foo') +- self.assertEqual(salt.utils.data.decode(_b('bar')), 'bar') +- self.assertEqual(salt.utils.data.decode(EGGS, normalize=True), 'яйца') ++ self.assertEqual(salt.utils.data.decode("foo"), "foo") ++ self.assertEqual(salt.utils.data.decode(_b("bar")), "bar") ++ self.assertEqual(salt.utils.data.decode(EGGS, normalize=True), "яйца") + self.assertEqual(salt.utils.data.decode(EGGS, normalize=False), EGGS) + + # Test binary blob + self.assertEqual(salt.utils.data.decode(BYTES, keep=True), BYTES) +- self.assertRaises( +- UnicodeDecodeError, +- salt.utils.data.decode, +- BYTES, +- keep=False) ++ self.assertRaises(UnicodeDecodeError, salt.utils.data.decode, BYTES, keep=False) ++ ++ def test_circular_refs_dicts(self): ++ test_dict = {"key": "value", "type": "test1"} ++ test_dict["self"] = test_dict ++ ret = salt.utils.data._remove_circular_refs(ob=test_dict) ++ self.assertDictEqual(ret, {"key": "value", "type": "test1", "self": None}) ++ ++ def test_circular_refs_lists(self): ++ test_list = { ++ "foo": [], ++ } ++ test_list["foo"].append((test_list,)) ++ ret = salt.utils.data._remove_circular_refs(ob=test_list) ++ self.assertDictEqual(ret, {"foo": [(None,)]}) ++ ++ def test_circular_refs_tuple(self): ++ test_dup = {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} ++ ret = salt.utils.data._remove_circular_refs(ob=test_dup) ++ self.assertDictEqual( ++ ret, {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1} ++ ) + + def test_decode_to_str(self): +- ''' ++ """ + Companion to test_decode, they should both be kept up-to-date with one + another. + + NOTE: This uses the lambda "_s" defined above in the global scope, + which converts the string/bytestring to a str type. +- ''' ++ """ + expected = [ +- _s('unicode_str'), +- _s('питон'), ++ _s("unicode_str"), ++ _s("питон"), + 123, + 456.789, + True, + False, + None, +- _s('яйца'), ++ _s("яйца"), + BYTES, +- [123, 456.789, _s('спам'), True, False, None, _s('яйца'), BYTES], +- (987, 654.321, _s('яйца'), _s('яйца'), None, (True, _s('яйца'), BYTES)), ++ [123, 456.789, _s("спам"), True, False, None, _s("яйца"), BYTES], ++ (987, 654.321, _s("яйца"), _s("яйца"), None, (True, _s("яйца"), BYTES)), + { +- _s('str_key'): _s('str_val'), ++ _s("str_key"): _s("str_val"), + None: True, + 123: 456.789, +- _s('яйца'): BYTES, +- _s('subdict'): { +- _s('unicode_key'): _s('яйца'), +- _s('tuple'): (123, _s('hello'), _s('world'), True, _s('яйца'), BYTES), +- _s('list'): [456, _s('спам'), False, _s('яйца'), BYTES] +- } ++ _s("яйца"): BYTES, ++ _s("subdict"): { ++ _s("unicode_key"): _s("яйца"), ++ _s("tuple"): ( ++ 123, ++ _s("hello"), ++ _s("world"), ++ True, ++ _s("яйца"), ++ BYTES, ++ ), ++ _s("list"): [456, _s("спам"), False, _s("яйца"), BYTES], ++ }, + }, +- OrderedDict([(_s('foo'), _s('bar')), (123, 456), (_s('яйца'), BYTES)]) ++ OrderedDict([(_s("foo"), _s("bar")), (123, 456), (_s("яйца"), BYTES)]), + ] + + ret = salt.utils.data.decode( +@@ -393,27 +410,42 @@ class DataTestCase(TestCase): + normalize=True, + preserve_dict_class=True, + preserve_tuples=True, +- to_str=True) ++ to_str=True, ++ ) + self.assertEqual(ret, expected) + +- if six.PY3: +- # The binary data in the data structure should fail to decode, even +- # using the fallback, and raise an exception. +- self.assertRaises( +- UnicodeDecodeError, +- salt.utils.data.decode, +- self.test_data, +- keep=False, +- normalize=True, +- preserve_dict_class=True, +- preserve_tuples=True, +- to_str=True) ++ # The binary data in the data structure should fail to decode, even ++ # using the fallback, and raise an exception. ++ self.assertRaises( ++ UnicodeDecodeError, ++ salt.utils.data.decode, ++ self.test_data, ++ keep=False, ++ normalize=True, ++ preserve_dict_class=True, ++ preserve_tuples=True, ++ to_str=True, ++ ) + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples +- expected[10] = [987, 654.321, _s('яйца'), _s('яйца'), None, [True, _s('яйца'), BYTES]] +- expected[11][_s('subdict')][_s('tuple')] = [123, _s('hello'), _s('world'), True, _s('яйца'), BYTES] +- expected[12] = {_s('foo'): _s('bar'), 123: 456, _s('яйца'): BYTES} ++ expected[10] = [ ++ 987, ++ 654.321, ++ _s("яйца"), ++ _s("яйца"), ++ None, ++ [True, _s("яйца"), BYTES], ++ ] ++ expected[11][_s("subdict")][_s("tuple")] = [ ++ 123, ++ _s("hello"), ++ _s("world"), ++ True, ++ _s("яйца"), ++ BYTES, ++ ] ++ expected[12] = {_s("foo"): _s("bar"), 123: 456, _s("яйца"): BYTES} + + ret = salt.utils.data.decode( + self.test_data, +@@ -421,47 +453,41 @@ class DataTestCase(TestCase): + normalize=True, + preserve_dict_class=False, + preserve_tuples=False, +- to_str=True) ++ to_str=True, ++ ) + self.assertEqual(ret, expected) + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): +- log.debug('Testing decode of %s', item) ++ log.debug("Testing decode of %s", item) + self.assertEqual(salt.utils.data.decode(item, to_str=True), item) + + # Test single strings (not in a data structure) +- self.assertEqual(salt.utils.data.decode('foo', to_str=True), _s('foo')) +- self.assertEqual(salt.utils.data.decode(_b('bar'), to_str=True), _s('bar')) ++ self.assertEqual(salt.utils.data.decode("foo", to_str=True), _s("foo")) ++ self.assertEqual(salt.utils.data.decode(_b("bar"), to_str=True), _s("bar")) + + # Test binary blob +- self.assertEqual( +- salt.utils.data.decode(BYTES, keep=True, to_str=True), +- BYTES ++ self.assertEqual(salt.utils.data.decode(BYTES, keep=True, to_str=True), BYTES) ++ self.assertRaises( ++ UnicodeDecodeError, salt.utils.data.decode, BYTES, keep=False, to_str=True, + ) +- if six.PY3: +- self.assertRaises( +- UnicodeDecodeError, +- salt.utils.data.decode, +- BYTES, +- keep=False, +- to_str=True) + + def test_decode_fallback(self): +- ''' ++ """ + Test fallback to utf-8 +- ''' +- with patch.object(builtins, '__salt_system_encoding__', 'ascii'): +- self.assertEqual(salt.utils.data.decode(_b('яйца')), 'яйца') ++ """ ++ with patch.object(builtins, "__salt_system_encoding__", "ascii"): ++ self.assertEqual(salt.utils.data.decode(_b("яйца")), "яйца") + + def test_encode(self): +- ''' ++ """ + NOTE: This uses the lambda "_b" defined above in the global scope, + which encodes a string to a bytestring, assuming utf-8. +- ''' ++ """ + expected = [ +- _b('unicode_str'), +- _b('питон'), ++ _b("unicode_str"), ++ _b("питон"), + 123, + 456.789, + True, +@@ -469,67 +495,71 @@ class DataTestCase(TestCase): + None, + _b(EGGS), + BYTES, +- [123, 456.789, _b('спам'), True, False, None, _b(EGGS), BYTES], +- (987, 654.321, _b('яйца'), _b(EGGS), None, (True, _b(EGGS), BYTES)), ++ [123, 456.789, _b("спам"), True, False, None, _b(EGGS), BYTES], ++ (987, 654.321, _b("яйца"), _b(EGGS), None, (True, _b(EGGS), BYTES)), + { +- _b('str_key'): _b('str_val'), ++ _b("str_key"): _b("str_val"), + None: True, + 123: 456.789, + _b(EGGS): BYTES, +- _b('subdict'): { +- _b('unicode_key'): _b(EGGS), +- _b('tuple'): (123, _b('hello'), _b('world'), True, _b(EGGS), BYTES), +- _b('list'): [456, _b('спам'), False, _b(EGGS), BYTES] +- } ++ _b("subdict"): { ++ _b("unicode_key"): _b(EGGS), ++ _b("tuple"): (123, _b("hello"), _b("world"), True, _b(EGGS), BYTES), ++ _b("list"): [456, _b("спам"), False, _b(EGGS), BYTES], ++ }, + }, +- OrderedDict([(_b('foo'), _b('bar')), (123, 456), (_b(EGGS), BYTES)]) ++ OrderedDict([(_b("foo"), _b("bar")), (123, 456), (_b(EGGS), BYTES)]), + ] + + # Both keep=True and keep=False should work because the BYTES data is + # already bytes. + ret = salt.utils.data.encode( +- self.test_data, +- keep=True, +- preserve_dict_class=True, +- preserve_tuples=True) ++ self.test_data, keep=True, preserve_dict_class=True, preserve_tuples=True ++ ) + self.assertEqual(ret, expected) + ret = salt.utils.data.encode( +- self.test_data, +- keep=False, +- preserve_dict_class=True, +- preserve_tuples=True) ++ self.test_data, keep=False, preserve_dict_class=True, preserve_tuples=True ++ ) + self.assertEqual(ret, expected) + + # Now munge the expected data so that we get what we would expect if we + # disable preservation of dict class and tuples +- expected[10] = [987, 654.321, _b('яйца'), _b(EGGS), None, [True, _b(EGGS), BYTES]] +- expected[11][_b('subdict')][_b('tuple')] = [ +- 123, _b('hello'), _b('world'), True, _b(EGGS), BYTES ++ expected[10] = [ ++ 987, ++ 654.321, ++ _b("яйца"), ++ _b(EGGS), ++ None, ++ [True, _b(EGGS), BYTES], + ] +- expected[12] = {_b('foo'): _b('bar'), 123: 456, _b(EGGS): BYTES} ++ expected[11][_b("subdict")][_b("tuple")] = [ ++ 123, ++ _b("hello"), ++ _b("world"), ++ True, ++ _b(EGGS), ++ BYTES, ++ ] ++ expected[12] = {_b("foo"): _b("bar"), 123: 456, _b(EGGS): BYTES} + + ret = salt.utils.data.encode( +- self.test_data, +- keep=True, +- preserve_dict_class=False, +- preserve_tuples=False) ++ self.test_data, keep=True, preserve_dict_class=False, preserve_tuples=False ++ ) + self.assertEqual(ret, expected) + ret = salt.utils.data.encode( +- self.test_data, +- keep=False, +- preserve_dict_class=False, +- preserve_tuples=False) ++ self.test_data, keep=False, preserve_dict_class=False, preserve_tuples=False ++ ) + self.assertEqual(ret, expected) + + # Now test single non-string, non-data-structure items, these should + # return the same value when passed to this function + for item in (123, 4.56, True, False, None): +- log.debug('Testing encode of %s', item) ++ log.debug("Testing encode of %s", item) + self.assertEqual(salt.utils.data.encode(item), item) + + # Test single strings (not in a data structure) +- self.assertEqual(salt.utils.data.encode('foo'), _b('foo')) +- self.assertEqual(salt.utils.data.encode(_b('bar')), _b('bar')) ++ self.assertEqual(salt.utils.data.encode("foo"), _b("foo")) ++ self.assertEqual(salt.utils.data.encode(_b("bar")), _b("bar")) + + # Test binary blob, nothing should happen even when keep=False since + # the data is already bytes +@@ -537,41 +567,43 @@ class DataTestCase(TestCase): + self.assertEqual(salt.utils.data.encode(BYTES, keep=False), BYTES) + + def test_encode_keep(self): +- ''' ++ """ + Whereas we tested the keep argument in test_decode, it is much easier + to do a more comprehensive test of keep in its own function where we + can force the encoding. +- ''' +- unicode_str = 'питон' +- encoding = 'ascii' ++ """ ++ unicode_str = "питон" ++ encoding = "ascii" + + # Test single string + self.assertEqual( +- salt.utils.data.encode(unicode_str, encoding, keep=True), +- unicode_str) ++ salt.utils.data.encode(unicode_str, encoding, keep=True), unicode_str ++ ) + self.assertRaises( + UnicodeEncodeError, + salt.utils.data.encode, + unicode_str, + encoding, +- keep=False) ++ keep=False, ++ ) + + data = [ + unicode_str, +- [b'foo', [unicode_str], {b'key': unicode_str}, (unicode_str,)], +- {b'list': [b'foo', unicode_str], +- b'dict': {b'key': unicode_str}, +- b'tuple': (b'foo', unicode_str)}, +- ([b'foo', unicode_str], {b'key': unicode_str}, (unicode_str,)) ++ [b"foo", [unicode_str], {b"key": unicode_str}, (unicode_str,)], ++ { ++ b"list": [b"foo", unicode_str], ++ b"dict": {b"key": unicode_str}, ++ b"tuple": (b"foo", unicode_str), ++ }, ++ ([b"foo", unicode_str], {b"key": unicode_str}, (unicode_str,)), + ] + + # Since everything was a bytestring aside from the bogus data, the + # return data should be identical. We don't need to test recursive + # decoding, that has already been tested in test_encode. + self.assertEqual( +- salt.utils.data.encode(data, encoding, +- keep=True, preserve_tuples=True), +- data ++ salt.utils.data.encode(data, encoding, keep=True, preserve_tuples=True), ++ data, + ) + self.assertRaises( + UnicodeEncodeError, +@@ -579,13 +611,15 @@ class DataTestCase(TestCase): + data, + encoding, + keep=False, +- preserve_tuples=True) ++ preserve_tuples=True, ++ ) + + for index, _ in enumerate(data): + self.assertEqual( +- salt.utils.data.encode(data[index], encoding, +- keep=True, preserve_tuples=True), +- data[index] ++ salt.utils.data.encode( ++ data[index], encoding, keep=True, preserve_tuples=True ++ ), ++ data[index], + ) + self.assertRaises( + UnicodeEncodeError, +@@ -593,31 +627,36 @@ class DataTestCase(TestCase): + data[index], + encoding, + keep=False, +- preserve_tuples=True) ++ preserve_tuples=True, ++ ) + + def test_encode_fallback(self): +- ''' ++ """ + Test fallback to utf-8 +- ''' +- with patch.object(builtins, '__salt_system_encoding__', 'ascii'): +- self.assertEqual(salt.utils.data.encode('яйца'), _b('яйца')) +- with patch.object(builtins, '__salt_system_encoding__', 'CP1252'): +- self.assertEqual(salt.utils.data.encode('Ψ'), _b('Ψ')) ++ """ ++ with patch.object(builtins, "__salt_system_encoding__", "ascii"): ++ self.assertEqual(salt.utils.data.encode("яйца"), _b("яйца")) ++ with patch.object(builtins, "__salt_system_encoding__", "CP1252"): ++ self.assertEqual(salt.utils.data.encode("Ψ"), _b("Ψ")) + + def test_repack_dict(self): +- list_of_one_element_dicts = [{'dict_key_1': 'dict_val_1'}, +- {'dict_key_2': 'dict_val_2'}, +- {'dict_key_3': 'dict_val_3'}] +- expected_ret = {'dict_key_1': 'dict_val_1', +- 'dict_key_2': 'dict_val_2', +- 'dict_key_3': 'dict_val_3'} ++ list_of_one_element_dicts = [ ++ {"dict_key_1": "dict_val_1"}, ++ {"dict_key_2": "dict_val_2"}, ++ {"dict_key_3": "dict_val_3"}, ++ ] ++ expected_ret = { ++ "dict_key_1": "dict_val_1", ++ "dict_key_2": "dict_val_2", ++ "dict_key_3": "dict_val_3", ++ } + ret = salt.utils.data.repack_dictlist(list_of_one_element_dicts) + self.assertDictEqual(ret, expected_ret) + + # Try with yaml +- yaml_key_val_pair = '- key1: val1' ++ yaml_key_val_pair = "- key1: val1" + ret = salt.utils.data.repack_dictlist(yaml_key_val_pair) +- self.assertDictEqual(ret, {'key1': 'val1'}) ++ self.assertDictEqual(ret, {"key1": "val1"}) + + # Make sure we handle non-yaml junk data + ret = salt.utils.data.repack_dictlist(LOREM_IPSUM) +@@ -626,43 +665,47 @@ class DataTestCase(TestCase): + def test_stringify(self): + self.assertRaises(TypeError, salt.utils.data.stringify, 9) + self.assertEqual( +- salt.utils.data.stringify(['one', 'two', str('three'), 4, 5]), # future lint: disable=blacklisted-function +- ['one', 'two', 'three', '4', '5'] ++ salt.utils.data.stringify( ++ ["one", "two", "three", 4, 5] ++ ), # future lint: disable=blacklisted-function ++ ["one", "two", "three", "4", "5"], + ) + + def test_json_query(self): + # Raises exception if jmespath module is not found +- with patch('salt.utils.data.jmespath', None): ++ with patch("salt.utils.data.jmespath", None): + self.assertRaisesRegex( +- RuntimeError, 'requires jmespath', +- salt.utils.data.json_query, {}, '@' ++ RuntimeError, "requires jmespath", salt.utils.data.json_query, {}, "@" + ) + + # Test search + user_groups = { +- 'user1': {'groups': ['group1', 'group2', 'group3']}, +- 'user2': {'groups': ['group1', 'group2']}, +- 'user3': {'groups': ['group3']}, ++ "user1": {"groups": ["group1", "group2", "group3"]}, ++ "user2": {"groups": ["group1", "group2"]}, ++ "user3": {"groups": ["group3"]}, + } +- expression = '*.groups[0]' +- primary_groups = ['group1', 'group1', 'group3'] ++ expression = "*.groups[0]" ++ primary_groups = ["group1", "group1", "group3"] + self.assertEqual( +- sorted(salt.utils.data.json_query(user_groups, expression)), +- primary_groups ++ sorted(salt.utils.data.json_query(user_groups, expression)), primary_groups + ) + + + class FilterFalseyTestCase(TestCase): +- ''' ++ """ + Test suite for salt.utils.data.filter_falsey +- ''' ++ """ + + def test_nop(self): +- ''' ++ """ + Test cases where nothing will be done. +- ''' ++ """ + # Test with dictionary without recursion +- old_dict = {'foo': 'bar', 'bar': {'baz': {'qux': 'quux'}}, 'baz': ['qux', {'foo': 'bar'}]} ++ old_dict = { ++ "foo": "bar", ++ "bar": {"baz": {"qux": "quux"}}, ++ "baz": ["qux", {"foo": "bar"}], ++ } + new_dict = salt.utils.data.filter_falsey(old_dict) + self.assertEqual(old_dict, new_dict) + # Check returned type equality +@@ -671,23 +714,25 @@ class FilterFalseyTestCase(TestCase): + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + self.assertEqual(old_dict, new_dict) + # Test with list +- old_list = ['foo', 'bar'] ++ old_list = ["foo", "bar"] + new_list = salt.utils.data.filter_falsey(old_list) + self.assertEqual(old_list, new_list) + # Check returned type equality + self.assertIs(type(old_list), type(new_list)) + # Test with set +- old_set = set(['foo', 'bar']) ++ old_set = {"foo", "bar"} + new_set = salt.utils.data.filter_falsey(old_set) + self.assertEqual(old_set, new_set) + # Check returned type equality + self.assertIs(type(old_set), type(new_set)) + # Test with OrderedDict +- old_dict = OrderedDict([ +- ('foo', 'bar'), +- ('bar', OrderedDict([('qux', 'quux')])), +- ('baz', ['qux', OrderedDict([('foo', 'bar')])]) +- ]) ++ old_dict = OrderedDict( ++ [ ++ ("foo", "bar"), ++ ("bar", OrderedDict([("qux", "quux")])), ++ ("baz", ["qux", OrderedDict([("foo", "bar")])]), ++ ] ++ ) + new_dict = salt.utils.data.filter_falsey(old_dict) + self.assertEqual(old_dict, new_dict) + self.assertIs(type(old_dict), type(new_dict)) +@@ -696,8 +741,8 @@ class FilterFalseyTestCase(TestCase): + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type(0)]) + self.assertEqual(old_list, new_list) + # Test excluding str (or unicode) (or both) +- old_list = [''] +- new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type('')]) ++ old_list = [""] ++ new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type("")]) + self.assertEqual(old_list, new_list) + # Test excluding list + old_list = [[]] +@@ -709,185 +754,264 @@ class FilterFalseyTestCase(TestCase): + self.assertEqual(old_list, new_list) + + def test_filter_dict_no_recurse(self): +- ''' ++ """ + Test filtering a dictionary without recursing. + This will only filter out key-values where the values are falsey. +- ''' +- old_dict = {'foo': None, +- 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, +- 'baz': ['qux'], +- 'qux': {}, +- 'quux': []} ++ """ ++ old_dict = { ++ "foo": None, ++ "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, ++ "baz": ["qux"], ++ "qux": {}, ++ "quux": [], ++ } + new_dict = salt.utils.data.filter_falsey(old_dict) +- expect_dict = {'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']} ++ expect_dict = { ++ "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, ++ "baz": ["qux"], ++ } + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) + + def test_filter_dict_recurse(self): +- ''' ++ """ + Test filtering a dictionary with recursing. + This will filter out any key-values where the values are falsey or when + the values *become* falsey after filtering their contents (in case they + are lists or dicts). +- ''' +- old_dict = {'foo': None, +- 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, +- 'baz': ['qux'], +- 'qux': {}, +- 'quux': []} ++ """ ++ old_dict = { ++ "foo": None, ++ "bar": {"baz": {"qux": None, "quux": "", "foo": []}}, ++ "baz": ["qux"], ++ "qux": {}, ++ "quux": [], ++ } + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) +- expect_dict = {'baz': ['qux']} ++ expect_dict = {"baz": ["qux"]} + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) + + def test_filter_list_no_recurse(self): +- ''' ++ """ + Test filtering a list without recursing. + This will only filter out items which are falsey. +- ''' +- old_list = ['foo', None, [], {}, 0, ''] ++ """ ++ old_list = ["foo", None, [], {}, 0, ""] + new_list = salt.utils.data.filter_falsey(old_list) +- expect_list = ['foo'] ++ expect_list = ["foo"] + self.assertEqual(expect_list, new_list) + self.assertIs(type(expect_list), type(new_list)) + # Ensure nested values are *not* filtered out. + old_list = [ +- 'foo', +- ['foo'], +- ['foo', None], +- {'foo': 0}, +- {'foo': 'bar', 'baz': []}, +- [{'foo': ''}], ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey(old_list) + self.assertEqual(old_list, new_list) + self.assertIs(type(old_list), type(new_list)) + + def test_filter_list_recurse(self): +- ''' ++ """ + Test filtering a list with recursing. + This will filter out any items which are falsey, or which become falsey + after filtering their contents (in case they are lists or dicts). +- ''' ++ """ + old_list = [ +- 'foo', +- ['foo'], +- ['foo', None], +- {'foo': 0}, +- {'foo': 'bar', 'baz': []}, +- [{'foo': ''}] ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], + ] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) +- expect_list = ['foo', ['foo'], ['foo'], {'foo': 'bar'}] ++ expect_list = ["foo", ["foo"], ["foo"], {"foo": "bar"}] + self.assertEqual(expect_list, new_list) + self.assertIs(type(expect_list), type(new_list)) + + def test_filter_set_no_recurse(self): +- ''' ++ """ + Test filtering a set without recursing. + Note that a set cannot contain unhashable types, so recursion is not possible. +- ''' +- old_set = set([ +- 'foo', +- None, +- 0, +- '', +- ]) ++ """ ++ old_set = {"foo", None, 0, ""} + new_set = salt.utils.data.filter_falsey(old_set) +- expect_set = set(['foo']) ++ expect_set = {"foo"} + self.assertEqual(expect_set, new_set) + self.assertIs(type(expect_set), type(new_set)) + + def test_filter_ordereddict_no_recurse(self): +- ''' ++ """ + Test filtering an OrderedDict without recursing. +- ''' +- old_dict = OrderedDict([ +- ('foo', None), +- ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), +- ('baz', ['qux']), +- ('qux', {}), +- ('quux', []) +- ]) ++ """ ++ old_dict = OrderedDict( ++ [ ++ ("foo", None), ++ ( ++ "bar", ++ OrderedDict( ++ [ ++ ( ++ "baz", ++ OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), ++ ) ++ ] ++ ), ++ ), ++ ("baz", ["qux"]), ++ ("qux", {}), ++ ("quux", []), ++ ] ++ ) + new_dict = salt.utils.data.filter_falsey(old_dict) +- expect_dict = OrderedDict([ +- ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), +- ('baz', ['qux']), +- ]) ++ expect_dict = OrderedDict( ++ [ ++ ( ++ "bar", ++ OrderedDict( ++ [ ++ ( ++ "baz", ++ OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), ++ ) ++ ] ++ ), ++ ), ++ ("baz", ["qux"]), ++ ] ++ ) + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) + + def test_filter_ordereddict_recurse(self): +- ''' ++ """ + Test filtering an OrderedDict with recursing. +- ''' +- old_dict = OrderedDict([ +- ('foo', None), +- ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), +- ('baz', ['qux']), +- ('qux', {}), +- ('quux', []) +- ]) ++ """ ++ old_dict = OrderedDict( ++ [ ++ ("foo", None), ++ ( ++ "bar", ++ OrderedDict( ++ [ ++ ( ++ "baz", ++ OrderedDict([("qux", None), ("quux", ""), ("foo", [])]), ++ ) ++ ] ++ ), ++ ), ++ ("baz", ["qux"]), ++ ("qux", {}), ++ ("quux", []), ++ ] ++ ) + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) +- expect_dict = OrderedDict([ +- ('baz', ['qux']), +- ]) ++ expect_dict = OrderedDict([("baz", ["qux"])]) + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) + + def test_filter_list_recurse_limit(self): +- ''' ++ """ + Test filtering a list with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. +- ''' ++ """ + old_list = [None, [None, [None, [None]]]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2) + self.assertEqual([[[[None]]]], new_list) + + def test_filter_dict_recurse_limit(self): +- ''' ++ """ + Test filtering a dict with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. +- ''' +- old_dict = {'one': None, +- 'foo': {'two': None, 'bar': {'three': None, 'baz': {'four': None}}}} ++ """ ++ old_dict = { ++ "one": None, ++ "foo": {"two": None, "bar": {"three": None, "baz": {"four": None}}}, ++ } + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2) +- self.assertEqual({'foo': {'bar': {'baz': {'four': None}}}}, new_dict) ++ self.assertEqual({"foo": {"bar": {"baz": {"four": None}}}}, new_dict) + + def test_filter_exclude_types(self): +- ''' ++ """ + Test filtering a list recursively, but also ignoring (i.e. not filtering) + out certain types that can be falsey. +- ''' ++ """ + # Ignore int, unicode +- old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] +- new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(0), type('')]) +- self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 0}, {'foo': 'bar'}, [{'foo': ''}]], new_list) ++ old_list = [ ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], ++ ] ++ new_list = salt.utils.data.filter_falsey( ++ old_list, recurse_depth=3, ignore_types=[type(0), type("")] ++ ) ++ self.assertEqual( ++ ["foo", ["foo"], ["foo"], {"foo": 0}, {"foo": "bar"}, [{"foo": ""}]], ++ new_list, ++ ) + # Ignore list +- old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] +- new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type([])]) +- self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar', 'baz': []}, []], new_list) ++ old_list = [ ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], ++ ] ++ new_list = salt.utils.data.filter_falsey( ++ old_list, recurse_depth=3, ignore_types=[type([])] ++ ) ++ self.assertEqual( ++ ["foo", ["foo"], ["foo"], {"foo": "bar", "baz": []}, []], new_list ++ ) + # Ignore dict +- old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] +- new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type({})]) +- self.assertEqual(['foo', ['foo'], ['foo'], {}, {'foo': 'bar'}, [{}]], new_list) ++ old_list = [ ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], ++ ] ++ new_list = salt.utils.data.filter_falsey( ++ old_list, recurse_depth=3, ignore_types=[type({})] ++ ) ++ self.assertEqual(["foo", ["foo"], ["foo"], {}, {"foo": "bar"}, [{}]], new_list) + # Ignore NoneType +- old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] +- new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(None)]) +- self.assertEqual(['foo', ['foo'], ['foo', None], {'foo': 'bar'}], new_list) ++ old_list = [ ++ "foo", ++ ["foo"], ++ ["foo", None], ++ {"foo": 0}, ++ {"foo": "bar", "baz": []}, ++ [{"foo": ""}], ++ ] ++ new_list = salt.utils.data.filter_falsey( ++ old_list, recurse_depth=3, ignore_types=[type(None)] ++ ) ++ self.assertEqual(["foo", ["foo"], ["foo", None], {"foo": "bar"}], new_list) + + + class FilterRecursiveDiff(TestCase): +- ''' ++ """ + Test suite for salt.utils.data.recursive_diff +- ''' ++ """ + + def test_list_equality(self): +- ''' ++ """ + Test cases where equal lists are compared. +- ''' ++ """ + test_list = [0, 1, 2] + self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list)) + +@@ -895,392 +1019,455 @@ class FilterRecursiveDiff(TestCase): + self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list)) + + def test_dict_equality(self): +- ''' ++ """ + Test cases where equal dicts are compared. +- ''' +- test_dict = {'foo': 'bar', 'bar': {'baz': {'qux': 'quux'}}, 'frop': 0} ++ """ ++ test_dict = {"foo": "bar", "bar": {"baz": {"qux": "quux"}}, "frop": 0} + self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict)) + + def test_ordereddict_equality(self): +- ''' ++ """ + Test cases where equal OrderedDicts are compared. +- ''' +- test_dict = OrderedDict([ +- ('foo', 'bar'), +- ('bar', OrderedDict([('baz', OrderedDict([('qux', 'quux')]))])), +- ('frop', 0)]) ++ """ ++ test_dict = OrderedDict( ++ [ ++ ("foo", "bar"), ++ ("bar", OrderedDict([("baz", OrderedDict([("qux", "quux")]))])), ++ ("frop", 0), ++ ] ++ ) + self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict)) + + def test_mixed_equality(self): +- ''' ++ """ + Test cases where mixed nested lists and dicts are compared. +- ''' ++ """ + test_data = { +- 'foo': 'bar', +- 'baz': [0, 1, 2], +- 'bar': {'baz': [{'qux': 'quux'}, {'froop', 0}]} ++ "foo": "bar", ++ "baz": [0, 1, 2], ++ "bar": {"baz": [{"qux": "quux"}, {"froop", 0}]}, + } + self.assertEqual({}, salt.utils.data.recursive_diff(test_data, test_data)) + + def test_set_equality(self): +- ''' ++ """ + Test cases where equal sets are compared. +- ''' +- test_set = set([0, 1, 2, 3, 'foo']) ++ """ ++ test_set = {0, 1, 2, 3, "foo"} + self.assertEqual({}, salt.utils.data.recursive_diff(test_set, test_set)) + + # This is a bit of an oddity, as python seems to sort the sets in memory + # so both sets end up with the same ordering (0..3). +- set_one = set([0, 1, 2, 3]) +- set_two = set([3, 2, 1, 0]) ++ set_one = {0, 1, 2, 3} ++ set_two = {3, 2, 1, 0} + self.assertEqual({}, salt.utils.data.recursive_diff(set_one, set_two)) + + def test_tuple_equality(self): +- ''' ++ """ + Test cases where equal tuples are compared. +- ''' +- test_tuple = (0, 1, 2, 3, 'foo') ++ """ ++ test_tuple = (0, 1, 2, 3, "foo") + self.assertEqual({}, salt.utils.data.recursive_diff(test_tuple, test_tuple)) + + def test_list_inequality(self): +- ''' ++ """ + Test cases where two inequal lists are compared. +- ''' ++ """ + list_one = [0, 1, 2] +- list_two = ['foo', 'bar', 'baz'] +- expected_result = {'old': list_one, 'new': list_two} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_one, list_two)) +- expected_result = {'new': list_one, 'old': list_two} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_two, list_one)) +- +- list_one = [0, 'foo', 1, 'bar'] +- list_two = [1, 'foo', 1, 'qux'] +- expected_result = {'old': [0, 'bar'], 'new': [1, 'qux']} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_one, list_two)) +- expected_result = {'new': [0, 'bar'], 'old': [1, 'qux']} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_two, list_one)) ++ list_two = ["foo", "bar", "baz"] ++ expected_result = {"old": list_one, "new": list_two} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_one, list_two) ++ ) ++ expected_result = {"new": list_one, "old": list_two} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_two, list_one) ++ ) ++ ++ list_one = [0, "foo", 1, "bar"] ++ list_two = [1, "foo", 1, "qux"] ++ expected_result = {"old": [0, "bar"], "new": [1, "qux"]} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_one, list_two) ++ ) ++ expected_result = {"new": [0, "bar"], "old": [1, "qux"]} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_two, list_one) ++ ) + + list_one = [0, 1, [2, 3]] +- list_two = [0, 1, ['foo', 'bar']] +- expected_result = {'old': [[2, 3]], 'new': [['foo', 'bar']]} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_one, list_two)) +- expected_result = {'new': [[2, 3]], 'old': [['foo', 'bar']]} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_two, list_one)) ++ list_two = [0, 1, ["foo", "bar"]] ++ expected_result = {"old": [[2, 3]], "new": [["foo", "bar"]]} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_one, list_two) ++ ) ++ expected_result = {"new": [[2, 3]], "old": [["foo", "bar"]]} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_two, list_one) ++ ) + + def test_dict_inequality(self): +- ''' ++ """ + Test cases where two inequal dicts are compared. +- ''' +- dict_one = {'foo': 1, 'bar': 2, 'baz': 3} +- dict_two = {'foo': 2, 1: 'bar', 'baz': 3} +- expected_result = {'old': {'foo': 1, 'bar': 2}, 'new': {'foo': 2, 1: 'bar'}} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)) +- expected_result = {'new': {'foo': 1, 'bar': 2}, 'old': {'foo': 2, 1: 'bar'}} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)) +- +- dict_one = {'foo': {'bar': {'baz': 1}}} +- dict_two = {'foo': {'qux': {'baz': 1}}} +- expected_result = {'old': dict_one, 'new': dict_two} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)) +- expected_result = {'new': dict_one, 'old': dict_two} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)) ++ """ ++ dict_one = {"foo": 1, "bar": 2, "baz": 3} ++ dict_two = {"foo": 2, 1: "bar", "baz": 3} ++ expected_result = {"old": {"foo": 1, "bar": 2}, "new": {"foo": 2, 1: "bar"}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) ++ ) ++ expected_result = {"new": {"foo": 1, "bar": 2}, "old": {"foo": 2, 1: "bar"}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) ++ ) ++ ++ dict_one = {"foo": {"bar": {"baz": 1}}} ++ dict_two = {"foo": {"qux": {"baz": 1}}} ++ expected_result = {"old": dict_one, "new": dict_two} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) ++ ) ++ expected_result = {"new": dict_one, "old": dict_two} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) ++ ) + + def test_ordereddict_inequality(self): +- ''' ++ """ + Test cases where two inequal OrderedDicts are compared. +- ''' +- odict_one = OrderedDict([('foo', 'bar'), ('bar', 'baz')]) +- odict_two = OrderedDict([('bar', 'baz'), ('foo', 'bar')]) +- expected_result = {'old': odict_one, 'new': odict_two} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(odict_one, odict_two)) ++ """ ++ odict_one = OrderedDict([("foo", "bar"), ("bar", "baz")]) ++ odict_two = OrderedDict([("bar", "baz"), ("foo", "bar")]) ++ expected_result = {"old": odict_one, "new": odict_two} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(odict_one, odict_two) ++ ) + + def test_set_inequality(self): +- ''' ++ """ + Test cases where two inequal sets are compared. + Tricky as the sets are compared zipped, so shuffled sets of equal values + are considered different. +- ''' +- set_one = set([0, 1, 2, 4]) +- set_two = set([0, 1, 3, 4]) +- expected_result = {'old': set([2]), 'new': set([3])} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(set_one, set_two)) +- expected_result = {'new': set([2]), 'old': set([3])} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(set_two, set_one)) ++ """ ++ set_one = {0, 1, 2, 4} ++ set_two = {0, 1, 3, 4} ++ expected_result = {"old": {2}, "new": {3}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(set_one, set_two) ++ ) ++ expected_result = {"new": {2}, "old": {3}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(set_two, set_one) ++ ) + + # It is unknown how different python versions will store sets in memory. + # Python 2.7 seems to sort it (i.e. set_one below becomes {0, 1, 'foo', 'bar'} + # However Python 3.6.8 stores it differently each run. + # So just test for "not equal" here. +- set_one = set([0, 'foo', 1, 'bar']) +- set_two = set(['foo', 1, 'bar', 2]) ++ set_one = {0, "foo", 1, "bar"} ++ set_two = {"foo", 1, "bar", 2} + expected_result = {} +- self.assertNotEqual(expected_result, salt.utils.data.recursive_diff(set_one, set_two)) ++ self.assertNotEqual( ++ expected_result, salt.utils.data.recursive_diff(set_one, set_two) ++ ) + + def test_mixed_inequality(self): +- ''' ++ """ + Test cases where two mixed dicts/iterables that are different are compared. +- ''' +- dict_one = {'foo': [1, 2, 3]} +- dict_two = {'foo': [3, 2, 1]} +- expected_result = {'old': {'foo': [1, 3]}, 'new': {'foo': [3, 1]}} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)) +- expected_result = {'new': {'foo': [1, 3]}, 'old': {'foo': [3, 1]}} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)) +- +- list_one = [1, 2, {'foo': ['bar', {'foo': 1, 'bar': 2}]}] +- list_two = [3, 4, {'foo': ['qux', {'foo': 1, 'bar': 2}]}] +- expected_result = {'old': [1, 2, {'foo': ['bar']}], 'new': [3, 4, {'foo': ['qux']}]} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_one, list_two)) +- expected_result = {'new': [1, 2, {'foo': ['bar']}], 'old': [3, 4, {'foo': ['qux']}]} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(list_two, list_one)) +- +- mixed_one = {'foo': set([0, 1, 2]), 'bar': [0, 1, 2]} +- mixed_two = {'foo': set([1, 2, 3]), 'bar': [1, 2, 3]} ++ """ ++ dict_one = {"foo": [1, 2, 3]} ++ dict_two = {"foo": [3, 2, 1]} ++ expected_result = {"old": {"foo": [1, 3]}, "new": {"foo": [3, 1]}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) ++ ) ++ expected_result = {"new": {"foo": [1, 3]}, "old": {"foo": [3, 1]}} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(dict_two, dict_one) ++ ) ++ ++ list_one = [1, 2, {"foo": ["bar", {"foo": 1, "bar": 2}]}] ++ list_two = [3, 4, {"foo": ["qux", {"foo": 1, "bar": 2}]}] + expected_result = { +- 'old': {'foo': set([0]), 'bar': [0, 1, 2]}, +- 'new': {'foo': set([3]), 'bar': [1, 2, 3]} ++ "old": [1, 2, {"foo": ["bar"]}], ++ "new": [3, 4, {"foo": ["qux"]}], + } +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two)) ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_one, list_two) ++ ) ++ expected_result = { ++ "new": [1, 2, {"foo": ["bar"]}], ++ "old": [3, 4, {"foo": ["qux"]}], ++ } ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(list_two, list_one) ++ ) ++ ++ mixed_one = {"foo": {0, 1, 2}, "bar": [0, 1, 2]} ++ mixed_two = {"foo": {1, 2, 3}, "bar": [1, 2, 3]} + expected_result = { +- 'new': {'foo': set([0]), 'bar': [0, 1, 2]}, +- 'old': {'foo': set([3]), 'bar': [1, 2, 3]} ++ "old": {"foo": {0}, "bar": [0, 1, 2]}, ++ "new": {"foo": {3}, "bar": [1, 2, 3]}, + } +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one)) ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two) ++ ) ++ expected_result = { ++ "new": {"foo": {0}, "bar": [0, 1, 2]}, ++ "old": {"foo": {3}, "bar": [1, 2, 3]}, ++ } ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one) ++ ) + + def test_tuple_inequality(self): +- ''' ++ """ + Test cases where two tuples that are different are compared. +- ''' ++ """ + tuple_one = (1, 2, 3) + tuple_two = (3, 2, 1) +- expected_result = {'old': (1, 3), 'new': (3, 1)} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two)) ++ expected_result = {"old": (1, 3), "new": (3, 1)} ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two) ++ ) + + def test_list_vs_set(self): +- ''' ++ """ + Test case comparing a list with a set, will be compared unordered. +- ''' ++ """ + mixed_one = [1, 2, 3] +- mixed_two = set([3, 2, 1]) ++ mixed_two = {3, 2, 1} + expected_result = {} +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two)) +- self.assertEqual(expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one)) ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two) ++ ) ++ self.assertEqual( ++ expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one) ++ ) + + def test_dict_vs_ordereddict(self): +- ''' ++ """ + Test case comparing a dict with an ordereddict, will be compared unordered. +- ''' +- test_dict = {'foo': 'bar', 'bar': 'baz'} +- test_odict = OrderedDict([('foo', 'bar'), ('bar', 'baz')]) ++ """ ++ test_dict = {"foo": "bar", "bar": "baz"} ++ test_odict = OrderedDict([("foo", "bar"), ("bar", "baz")]) + self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict)) + self.assertEqual({}, salt.utils.data.recursive_diff(test_odict, test_dict)) + +- test_odict2 = OrderedDict([('bar', 'baz'), ('foo', 'bar')]) ++ test_odict2 = OrderedDict([("bar", "baz"), ("foo", "bar")]) + self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict2)) + self.assertEqual({}, salt.utils.data.recursive_diff(test_odict2, test_dict)) + + def test_list_ignore_ignored(self): +- ''' ++ """ + Test case comparing two lists with ignore-list supplied (which is not used + when comparing lists). +- ''' ++ """ + list_one = [1, 2, 3] + list_two = [3, 2, 1] +- expected_result = {'old': [1, 3], 'new': [3, 1]} ++ expected_result = {"old": [1, 3], "new": [3, 1]} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(list_one, list_two, ignore_keys=[1, 3]) ++ salt.utils.data.recursive_diff(list_one, list_two, ignore_keys=[1, 3]), + ) + + def test_dict_ignore(self): +- ''' ++ """ + Test case comparing two dicts with ignore-list supplied. +- ''' +- dict_one = {'foo': 1, 'bar': 2, 'baz': 3} +- dict_two = {'foo': 3, 'bar': 2, 'baz': 1} +- expected_result = {'old': {'baz': 3}, 'new': {'baz': 1}} ++ """ ++ dict_one = {"foo": 1, "bar": 2, "baz": 3} ++ dict_two = {"foo": 3, "bar": 2, "baz": 1} ++ expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=['foo']) ++ salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]), + ) + + def test_ordereddict_ignore(self): +- ''' ++ """ + Test case comparing two OrderedDicts with ignore-list supplied. +- ''' +- odict_one = OrderedDict([('foo', 1), ('bar', 2), ('baz', 3)]) +- odict_two = OrderedDict([('baz', 1), ('bar', 2), ('foo', 3)]) ++ """ ++ odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) ++ odict_two = OrderedDict([("baz", 1), ("bar", 2), ("foo", 3)]) + # The key 'foo' will be ignored, which means the key from the other OrderedDict + # will always be considered "different" since OrderedDicts are compared ordered. +- expected_result = {'old': OrderedDict([('baz', 3)]), 'new': OrderedDict([('baz', 1)])} ++ expected_result = { ++ "old": OrderedDict([("baz", 3)]), ++ "new": OrderedDict([("baz", 1)]), ++ } + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(odict_one, odict_two, ignore_keys=['foo']) ++ salt.utils.data.recursive_diff(odict_one, odict_two, ignore_keys=["foo"]), + ) + + def test_dict_vs_ordereddict_ignore(self): +- ''' ++ """ + Test case comparing a dict with an OrderedDict with ignore-list supplied. +- ''' +- dict_one = {'foo': 1, 'bar': 2, 'baz': 3} +- odict_two = OrderedDict([('foo', 3), ('bar', 2), ('baz', 1)]) +- expected_result = {'old': {'baz': 3}, 'new': OrderedDict([('baz', 1)])} ++ """ ++ dict_one = {"foo": 1, "bar": 2, "baz": 3} ++ odict_two = OrderedDict([("foo", 3), ("bar", 2), ("baz", 1)]) ++ expected_result = {"old": {"baz": 3}, "new": OrderedDict([("baz", 1)])} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, odict_two, ignore_keys=['foo']) ++ salt.utils.data.recursive_diff(dict_one, odict_two, ignore_keys=["foo"]), + ) + + def test_mixed_nested_ignore(self): +- ''' ++ """ + Test case comparing mixed, nested items with ignore-list supplied. +- ''' +- dict_one = {'foo': [1], 'bar': {'foo': 1, 'bar': 2}, 'baz': 3} +- dict_two = {'foo': [2], 'bar': {'foo': 3, 'bar': 2}, 'baz': 1} +- expected_result = {'old': {'baz': 3}, 'new': {'baz': 1}} ++ """ ++ dict_one = {"foo": [1], "bar": {"foo": 1, "bar": 2}, "baz": 3} ++ dict_two = {"foo": [2], "bar": {"foo": 3, "bar": 2}, "baz": 1} ++ expected_result = {"old": {"baz": 3}, "new": {"baz": 1}} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=['foo']) ++ salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]), + ) + + def test_ordered_dict_unequal_length(self): +- ''' ++ """ + Test case comparing two OrderedDicts of unequal length. +- ''' +- odict_one = OrderedDict([('foo', 1), ('bar', 2), ('baz', 3)]) +- odict_two = OrderedDict([('foo', 1), ('bar', 2)]) +- expected_result = {'old': OrderedDict([('baz', 3)]), 'new': {}} ++ """ ++ odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) ++ odict_two = OrderedDict([("foo", 1), ("bar", 2)]) ++ expected_result = {"old": OrderedDict([("baz", 3)]), "new": {}} + self.assertEqual( +- expected_result, +- salt.utils.data.recursive_diff(odict_one, odict_two) ++ expected_result, salt.utils.data.recursive_diff(odict_one, odict_two) + ) + + def test_list_unequal_length(self): +- ''' ++ """ + Test case comparing two lists of unequal length. +- ''' ++ """ + list_one = [1, 2, 3] + list_two = [1, 2, 3, 4] +- expected_result = {'old': [], 'new': [4]} ++ expected_result = {"old": [], "new": [4]} + self.assertEqual( +- expected_result, +- salt.utils.data.recursive_diff(list_one, list_two) ++ expected_result, salt.utils.data.recursive_diff(list_one, list_two) + ) + + def test_set_unequal_length(self): +- ''' ++ """ + Test case comparing two sets of unequal length. + This does not do anything special, as it is unordered. +- ''' +- set_one = set([1, 2, 3]) +- set_two = set([4, 3, 2, 1]) +- expected_result = {'old': set([]), 'new': set([4])} ++ """ ++ set_one = {1, 2, 3} ++ set_two = {4, 3, 2, 1} ++ expected_result = {"old": set(), "new": {4}} + self.assertEqual( +- expected_result, +- salt.utils.data.recursive_diff(set_one, set_two) ++ expected_result, salt.utils.data.recursive_diff(set_one, set_two) + ) + + def test_tuple_unequal_length(self): +- ''' ++ """ + Test case comparing two tuples of unequal length. + This should be the same as comparing two ordered lists. +- ''' ++ """ + tuple_one = (1, 2, 3) + tuple_two = (1, 2, 3, 4) +- expected_result = {'old': (), 'new': (4,)} ++ expected_result = {"old": (), "new": (4,)} + self.assertEqual( +- expected_result, +- salt.utils.data.recursive_diff(tuple_one, tuple_two) ++ expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two) + ) + + def test_list_unordered(self): +- ''' ++ """ + Test case comparing two lists unordered. +- ''' ++ """ + list_one = [1, 2, 3, 4] + list_two = [4, 3, 2] +- expected_result = {'old': [1], 'new': []} ++ expected_result = {"old": [1], "new": []} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(list_one, list_two, ignore_order=True) ++ salt.utils.data.recursive_diff(list_one, list_two, ignore_order=True), + ) + + def test_mixed_nested_unordered(self): +- ''' ++ """ + Test case comparing nested dicts/lists unordered. +- ''' +- dict_one = {'foo': {'bar': [1, 2, 3]}, 'bar': [{'foo': 4}, 0]} +- dict_two = {'foo': {'bar': [3, 2, 1]}, 'bar': [0, {'foo': 4}]} ++ """ ++ dict_one = {"foo": {"bar": [1, 2, 3]}, "bar": [{"foo": 4}, 0]} ++ dict_two = {"foo": {"bar": [3, 2, 1]}, "bar": [0, {"foo": 4}]} + expected_result = {} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_order=True) ++ salt.utils.data.recursive_diff(dict_one, dict_two, ignore_order=True), + ) + expected_result = { +- 'old': {'foo': {'bar': [1, 3]}, 'bar': [{'foo': 4}, 0]}, +- 'new': {'foo': {'bar': [3, 1]}, 'bar': [0, {'foo': 4}]}, ++ "old": {"foo": {"bar": [1, 3]}, "bar": [{"foo": 4}, 0]}, ++ "new": {"foo": {"bar": [3, 1]}, "bar": [0, {"foo": 4}]}, + } + self.assertEqual( +- expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two) ++ expected_result, salt.utils.data.recursive_diff(dict_one, dict_two) + ) + + def test_ordered_dict_unordered(self): +- ''' ++ """ + Test case comparing OrderedDicts unordered. +- ''' +- odict_one = OrderedDict([('foo', 1), ('bar', 2), ('baz', 3)]) +- odict_two = OrderedDict([('baz', 3), ('bar', 2), ('foo', 1)]) ++ """ ++ odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) ++ odict_two = OrderedDict([("baz", 3), ("bar", 2), ("foo", 1)]) + expected_result = {} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(odict_one, odict_two, ignore_order=True) ++ salt.utils.data.recursive_diff(odict_one, odict_two, ignore_order=True), + ) + + def test_ignore_missing_keys_dict(self): +- ''' ++ """ + Test case ignoring missing keys on a comparison of dicts. +- ''' +- dict_one = {'foo': 1, 'bar': 2, 'baz': 3} +- dict_two = {'bar': 3} +- expected_result = {'old': {'bar': 2}, 'new': {'bar': 3}} ++ """ ++ dict_one = {"foo": 1, "bar": 2, "baz": 3} ++ dict_two = {"bar": 3} ++ expected_result = {"old": {"bar": 2}, "new": {"bar": 3}} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_missing_keys=True) ++ salt.utils.data.recursive_diff( ++ dict_one, dict_two, ignore_missing_keys=True ++ ), + ) + + def test_ignore_missing_keys_ordered_dict(self): +- ''' ++ """ + Test case not ignoring missing keys on a comparison of OrderedDicts. +- ''' +- odict_one = OrderedDict([('foo', 1), ('bar', 2), ('baz', 3)]) +- odict_two = OrderedDict([('bar', 3)]) +- expected_result = {'old': odict_one, 'new': odict_two} ++ """ ++ odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)]) ++ odict_two = OrderedDict([("bar", 3)]) ++ expected_result = {"old": odict_one, "new": odict_two} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(odict_one, odict_two, ignore_missing_keys=True) ++ salt.utils.data.recursive_diff( ++ odict_one, odict_two, ignore_missing_keys=True ++ ), + ) + + def test_ignore_missing_keys_recursive(self): +- ''' ++ """ + Test case ignoring missing keys on a comparison of nested dicts. +- ''' +- dict_one = {'foo': {'bar': 2, 'baz': 3}} +- dict_two = {'foo': {'baz': 3}} ++ """ ++ dict_one = {"foo": {"bar": 2, "baz": 3}} ++ dict_two = {"foo": {"baz": 3}} + expected_result = {} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_missing_keys=True) ++ salt.utils.data.recursive_diff( ++ dict_one, dict_two, ignore_missing_keys=True ++ ), + ) + # Compare from dict-in-dict + dict_two = {} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_missing_keys=True) ++ salt.utils.data.recursive_diff( ++ dict_one, dict_two, ignore_missing_keys=True ++ ), + ) + # Compare from dict-in-list +- dict_one = {'foo': ['bar', {'baz': 3}]} +- dict_two = {'foo': ['bar', {}]} ++ dict_one = {"foo": ["bar", {"baz": 3}]} ++ dict_two = {"foo": ["bar", {}]} + self.assertEqual( + expected_result, +- salt.utils.data.recursive_diff(dict_one, dict_two, ignore_missing_keys=True) ++ salt.utils.data.recursive_diff( ++ dict_one, dict_two, ignore_missing_keys=True ++ ), + ) +diff --git a/tests/unit/utils/test_xmlutil.py b/tests/unit/utils/test_xmlutil.py +index c04f39498e..cbf73861e5 100644 +--- a/tests/unit/utils/test_xmlutil.py ++++ b/tests/unit/utils/test_xmlutil.py +@@ -1,148 +1,170 @@ +-# -*- coding: utf-8 -*- +-''' ++""" + tests.unit.xmlutil_test + ~~~~~~~~~~~~~~~~~~~~ +-''' +-from __future__ import absolute_import, print_function, unicode_literals +-# Import Salt Testing libs +-from tests.support.unit import TestCase ++""" ++import salt.utils.xmlutil as xml + + # Import Salt libs + from salt._compat import ElementTree as ET +-import salt.utils.xmlutil as xml ++ ++# Import Salt Testing libs ++from tests.support.unit import TestCase + + + class XMLUtilTestCase(TestCase): +- ''' ++ """ + Tests that salt.utils.xmlutil properly parses XML data and returns as a properly formatted + dictionary. The default method of parsing will ignore attributes and return only the child + items. The full method will include parsing attributes. +- ''' ++ """ + + def setUp(self): + + # Populate our use cases for specific XML formats. + self.cases = { +- 'a': { +- 'xml': 'data', +- 'legacy': {'parent': 'data'}, +- 'full': 'data' ++ "a": { ++ "xml": "data", ++ "legacy": {"parent": "data"}, ++ "full": "data", + }, +- 'b': { +- 'xml': 'data', +- 'legacy': {'parent': 'data'}, +- 'full': {'parent': 'data', 'value': 'data'} ++ "b": { ++ "xml": 'data', ++ "legacy": {"parent": "data"}, ++ "full": {"parent": "data", "value": "data"}, + }, +- 'c': { +- 'xml': 'datadata' +- '', +- 'legacy': {'child': ['data', {'child': 'data'}, {'child': None}, {'child': None}]}, +- 'full': {'child': ['data', {'child': 'data', 'value': 'data'}, {'value': 'data'}, None]} ++ "c": { ++ "xml": 'datadata' ++ '', ++ "legacy": { ++ "child": [ ++ "data", ++ {"child": "data"}, ++ {"child": None}, ++ {"child": None}, ++ ] ++ }, ++ "full": { ++ "child": [ ++ "data", ++ {"child": "data", "value": "data"}, ++ {"value": "data"}, ++ None, ++ ] ++ }, + }, +- 'd': { +- 'xml': 'data', +- 'legacy': {'child': 'data'}, +- 'full': {'child': 'data', 'another': 'data', 'value': 'data'} ++ "d": { ++ "xml": 'data', ++ "legacy": {"child": "data"}, ++ "full": {"child": "data", "another": "data", "value": "data"}, + }, +- 'e': { +- 'xml': 'data', +- 'legacy': {'child': 'data'}, +- 'full': {'child': {'child': 'data', 'value': 'data'}, 'another': 'data', 'value': 'data'} ++ "e": { ++ "xml": 'data', ++ "legacy": {"child": "data"}, ++ "full": { ++ "child": {"child": "data", "value": "data"}, ++ "another": "data", ++ "value": "data", ++ }, + }, +- 'f': { +- 'xml': 'data' +- 'data', +- 'legacy': {'child': [{'sub-child': 'data'}, {'child': 'data'}]}, +- 'full': {'child': [{'sub-child': {'value': 'data', 'sub-child': 'data'}}, 'data']} ++ "f": { ++ "xml": 'data' ++ "data", ++ "legacy": {"child": [{"sub-child": "data"}, {"child": "data"}]}, ++ "full": { ++ "child": [ ++ {"sub-child": {"value": "data", "sub-child": "data"}}, ++ "data", ++ ] ++ }, + }, + } + + def test_xml_case_a(self): +- xmldata = ET.fromstring(self.cases['a']['xml']) ++ xmldata = ET.fromstring(self.cases["a"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['a']['legacy']) ++ self.assertEqual(defaultdict, self.cases["a"]["legacy"]) + + def test_xml_case_a_legacy(self): +- xmldata = ET.fromstring(self.cases['a']['xml']) ++ xmldata = ET.fromstring(self.cases["a"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['a']['legacy']) ++ self.assertEqual(defaultdict, self.cases["a"]["legacy"]) + + def test_xml_case_a_full(self): +- xmldata = ET.fromstring(self.cases['a']['xml']) ++ xmldata = ET.fromstring(self.cases["a"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['a']['full']) ++ self.assertEqual(defaultdict, self.cases["a"]["full"]) + + def test_xml_case_b(self): +- xmldata = ET.fromstring(self.cases['b']['xml']) ++ xmldata = ET.fromstring(self.cases["b"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['b']['legacy']) ++ self.assertEqual(defaultdict, self.cases["b"]["legacy"]) + + def test_xml_case_b_legacy(self): +- xmldata = ET.fromstring(self.cases['b']['xml']) ++ xmldata = ET.fromstring(self.cases["b"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['b']['legacy']) ++ self.assertEqual(defaultdict, self.cases["b"]["legacy"]) + + def test_xml_case_b_full(self): +- xmldata = ET.fromstring(self.cases['b']['xml']) ++ xmldata = ET.fromstring(self.cases["b"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['b']['full']) ++ self.assertEqual(defaultdict, self.cases["b"]["full"]) + + def test_xml_case_c(self): +- xmldata = ET.fromstring(self.cases['c']['xml']) ++ xmldata = ET.fromstring(self.cases["c"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['c']['legacy']) ++ self.assertEqual(defaultdict, self.cases["c"]["legacy"]) + + def test_xml_case_c_legacy(self): +- xmldata = ET.fromstring(self.cases['c']['xml']) ++ xmldata = ET.fromstring(self.cases["c"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['c']['legacy']) ++ self.assertEqual(defaultdict, self.cases["c"]["legacy"]) + + def test_xml_case_c_full(self): +- xmldata = ET.fromstring(self.cases['c']['xml']) ++ xmldata = ET.fromstring(self.cases["c"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['c']['full']) ++ self.assertEqual(defaultdict, self.cases["c"]["full"]) + + def test_xml_case_d(self): +- xmldata = ET.fromstring(self.cases['d']['xml']) ++ xmldata = ET.fromstring(self.cases["d"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['d']['legacy']) ++ self.assertEqual(defaultdict, self.cases["d"]["legacy"]) + + def test_xml_case_d_legacy(self): +- xmldata = ET.fromstring(self.cases['d']['xml']) ++ xmldata = ET.fromstring(self.cases["d"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['d']['legacy']) ++ self.assertEqual(defaultdict, self.cases["d"]["legacy"]) + + def test_xml_case_d_full(self): +- xmldata = ET.fromstring(self.cases['d']['xml']) ++ xmldata = ET.fromstring(self.cases["d"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['d']['full']) ++ self.assertEqual(defaultdict, self.cases["d"]["full"]) + + def test_xml_case_e(self): +- xmldata = ET.fromstring(self.cases['e']['xml']) ++ xmldata = ET.fromstring(self.cases["e"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['e']['legacy']) ++ self.assertEqual(defaultdict, self.cases["e"]["legacy"]) + + def test_xml_case_e_legacy(self): +- xmldata = ET.fromstring(self.cases['e']['xml']) ++ xmldata = ET.fromstring(self.cases["e"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['e']['legacy']) ++ self.assertEqual(defaultdict, self.cases["e"]["legacy"]) + + def test_xml_case_e_full(self): +- xmldata = ET.fromstring(self.cases['e']['xml']) ++ xmldata = ET.fromstring(self.cases["e"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['e']['full']) ++ self.assertEqual(defaultdict, self.cases["e"]["full"]) + + def test_xml_case_f(self): +- xmldata = ET.fromstring(self.cases['f']['xml']) ++ xmldata = ET.fromstring(self.cases["f"]["xml"]) + defaultdict = xml.to_dict(xmldata) +- self.assertEqual(defaultdict, self.cases['f']['legacy']) ++ self.assertEqual(defaultdict, self.cases["f"]["legacy"]) + + def test_xml_case_f_legacy(self): +- xmldata = ET.fromstring(self.cases['f']['xml']) ++ xmldata = ET.fromstring(self.cases["f"]["xml"]) + defaultdict = xml.to_dict(xmldata, False) +- self.assertEqual(defaultdict, self.cases['f']['legacy']) ++ self.assertEqual(defaultdict, self.cases["f"]["legacy"]) + + def test_xml_case_f_full(self): +- xmldata = ET.fromstring(self.cases['f']['xml']) ++ xmldata = ET.fromstring(self.cases["f"]["xml"]) + defaultdict = xml.to_dict(xmldata, True) +- self.assertEqual(defaultdict, self.cases['f']['full']) ++ self.assertEqual(defaultdict, self.cases["f"]["full"]) +-- +2.28.0 + + diff --git a/do-not-raise-streamclosederror-traceback-but-only-lo.patch b/do-not-raise-streamclosederror-traceback-but-only-lo.patch new file mode 100644 index 0000000..bf180aa --- /dev/null +++ b/do-not-raise-streamclosederror-traceback-but-only-lo.patch @@ -0,0 +1,27 @@ +From b651c2cd8b719a72e66b63afd9061739624763e1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Wed, 26 Aug 2020 10:24:58 +0100 +Subject: [PATCH] Do not raise StreamClosedError traceback but only log + it (bsc#1175549) + +--- + salt/transport/ipc.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py +index 33ee3d4182..624eca5a9c 100644 +--- a/salt/transport/ipc.py ++++ b/salt/transport/ipc.py +@@ -667,7 +667,6 @@ class IPCMessageSubscriber(IPCClient): + except StreamClosedError as exc: + log.trace('Subscriber disconnected from IPC %s', self.socket_path) + self._read_stream_future = None +- exc_to_raise = exc + except Exception as exc: # pylint: disable=broad-except + log.error('Exception occurred in Subscriber while handling stream: %s', exc) + self._read_stream_future = None +-- +2.28.0 + + diff --git a/opensuse-3000.2-virt-backports-236-257.patch b/opensuse-3000.2-virt-backports-236-257.patch new file mode 100644 index 0000000..6a3dce8 --- /dev/null +++ b/opensuse-3000.2-virt-backports-236-257.patch @@ -0,0 +1,21134 @@ +From de6c0e021f366ac7e0c4f6f103ece6624ed71a77 Mon Sep 17 00:00:00 2001 +From: Cedric Bosdonnat +Date: Tue, 1 Sep 2020 10:38:39 +0200 +Subject: [PATCH] Opensuse 3000.2 virt backports (#236) (#257) + +* Revert "virt._get_domain: don't raise an exception if there is no VM" + +This reverts commit 9d433776fc252ab872b331cfa7fc940c42452f83. + +* Revert "openSUSE-3000 virt-defined-states (#222)" + +This reverts commit 900fdc4d5bcfdac933f9a411d38b92ef47dd1ef5. + +* Revert "Add virt.all_capabilities" + +This reverts commit 96e8307b531f7e51f5587f66a51dc52ad246c8c1. + +* Blacken salt -- virt only + +* virt._get_domain: don't raise an exception if there is no VM + +Raising an exception if there is no VM in _get_domain makes sense if +looking for some VMs, but not when listing all VMs. + +* fixed bug on update existing boot parameters + +* add support to boot vm with UEFI + +* virt: don't depend on ElementTree.tostring to compare XML fragments + +ElementTree.tostring() implementation starts to keep the attribute order +defined by the user in python 3.8. This can lead to equal elements to be +considered different. Use xmlutil.to_dict(element, True) to compare the +elements. + +This also uncovered a wrong behavior of pool_update when only changing +the password. + +* openSUSE-3000 virt-defined-states (#222) + +* Create virt.pool_defined state out of virt.pool_running + +Users may want to use states to ensure a virtual storage pool is defined +and not enforce it to be running. Extract the code that performs the +pool definition / update from virt.pool_running state into a +virt.pool_defined. + +Obviously the virt.pool_running state calls the virt.pool_defined one. +In such a case no additionnal test is needed for virt.pool_defined since +this is already tested with virt.pool_running. + +* Add virt.update test parameter + +In order to allow running dry-runs of virt.update module add a test +parameter. This will later be used by the virt states. + +* Extract virt.defined state from virt.running + +In order to ensure a virtual guest is defined independently of its +status, extract the corresponding code from the virt.running state. + +This commit also handles the __opts__['test'] for the running state. +Since the update call only performs changes if needed, deprecate the +update parameter. + +* Extract virt.network_defined from virt.network_running + +Just like domains and storage pools, users may want to ensure a network +is defined without influencing it's status. Extract the code from +network_running state defining the network into a network_defined state. + +While at it, support __opt__['test'] == True in these states. Updating +the network definition in the pool_defined state will come in a future +PR. + +* Fix virt.update to handle None mem and cpu + +virt.running state now may call virt.update with None mem and cpu +parameters. This was not handled in _gen_xml(). Also add some more tests +cases matching this for virt.update. + +* Add support to virt for libvirt loader + +* Add bhyve compatibility to virt + +* Fix test_get_hypervisor: mock bhyve + +* Add virt.all_capabilities + +In order to get all possible capabilities from a host, the user has to +call virt.capabilities, and then loop over the guests and domains +before calling virt.domain_capabilities for each of them. + +This commit embeds all this logic to get them all in a single +virt.all_capabilities call. + +* Virt init disks support (PR#56666) + +* Fix pylint warning in test_virt.py after blackening + +* Add pool parameter to virt.define_vol_xml_str + +* Remove useless default values for disks and vm_name in _disk_profile + +* virt._gen_vol_xml: move all esx-specifics outside + +In the near future gen_vol_xml will be able to handle many volume types, +not only for ESX volumes. For this, clean up the function from all the +ESX-specifics code and move them to the caller code. + +The volume key and target path values have also been removed since those +are read-only elements that should not be provided for volume creation +as per https://libvirt.org/formatstorage.html#StorageVol + +* virt: add more properties to generate volume XML + +In order to generate almost all the volumes that libvirt can handle, add +the type, backingStore, permissions and allocation parameters to the +virt._gen_vol_xml() function. + +Also make the format parameter optional since libvirt has default values +depending on the storage backend. + +* virt.define_vol_xml_str variant using an existing libvirt connection + +In order to avoid connection multiple times when reusing this function +in the virt module, create _define_vol_xml_str not caring about the +connection opening and closing. + +* Add virt.volume_define function + +In the same vein than pool_define and network_define, expose a +volume_define function to let users create a volume without needing to +know all of libvirt's XML details. + +* Add virt.volume_upload function + +When using volumes the user can just copy the template disk image into +the target path. In such a case, the image needs to be uploaded into the +volume. + +* virt.capabilities refactoring + +Extract the libvirt-handling code from virt.capabilities into a +virt._capabilities function accepting an opened libvirt connection. +This allows reusing the code in other functions with easy connection +handling. + +* Extract virt.pool_capabilities logic for use with a libvirt connection + +Te virt.pool_capabilities function computes a lot of interesting values +on the virtual storage pool types. Extract the logic of it into +virt._pool_capabilities for reuse. + +* Share libvirt connection in virt.init + +Since the next commits will introduce more uses of the libvirt +connection in virt.init(), start sharing it now. + +* Add disk volumes support in virt.init + +In order to support creating VMs with disk on more storage pools like +iSCSI, disks, LVM, RBD, etc, virt.init needs to handle disks not only as +files, but also as libvirt volumes. + +* fix libvirtError use + +libvirtError is not defined, libvirt.libvirtError should be used +instead. + +* virt: let libvirt generate MAC addresses + +There is no need to generate MAC addresses in the virt module if the +user hasn't provided any. This only makes it harder to make the +difference between a real mac address change from the user and a new +generated one. + +Now the mac address is not written in the domain XML definition if not +provided by the user. This avoids unnecessary changes when applying +virt.running. + +* virt.update handle disk volumes + +* virt.volume_infos: output backing store as well + +Since it could be useful to know whether a volume has a backing store, +output the path and format of the backing store if any is defined. + +* virt.volume_infos: output disk format + +Since the format of a volume may be of interest and could help to tell +if two volumes are similar, output this information in the +virt.volume_infos function. + +* Add virt.volume_defined state + +In order to help creating storage volumes in virtual storage pools from +states, add a virt.volume_defined state. + +* Add volume support to virt._get_disks + +If a virtual machine has disks of volume types, they will now be +reported by the virt._get_disk() function and all the user-exposed +functions using it like virt.get_disks(), virt.vm_info() and +virt.full_info(). + +* Add volume disks support to virt.purge() + +virt.purge will now remove not only the file disks, but also the disk volumes. + +* Handle RBD volumes as network disks in VM definition + +libvirt doesn't support attaching RBD storage pool volumes to a VM. +For instance, a disk with such a source is invalid: + + + +And needs to be replaced with: + + + + + + + + +This means that we need to fetch the connection data from the pool +definition and convert the volume disk definition into a network one +when creating the VM. + +This also means that when purging the VM we need to search for the +pool by looking in every pool's XML to delete the volume. + +* virt: fix VM creating with disk volume + +Since volumes in a virtual storage pool of type 'disk' are partitions, +then need to be named after the disk name with sequential numbers rather +than using the VM and disk names. + +Also, the format passed by the user is the one used when creating the volume. +However in the VM definition for logical and disk volumes, the format should +be set to raw. + +* virt: handle cdrom remote images + +Libvirt allows to use network images for cdroms. Use them if the image +is a remote URL for a cdrom device. + +* virt.update properly handle removable devices + +Live attaching / detaching removable devices results in failures. +To change the source of those, we need to call updateDeviceFlags instead. + +* improve boot parameter documentation (PR#57086) + +* Remove buggy start parameter from virt.pool_running docstring + +As mentioned by issue #57275, the start parameter in virt.pool_running +documentation is not implemented at all. Remove it from the doc. + +* virt: fix removable_changes definition place + +* virt: show the proper pool name in get_disks + +From the user point of view the internal RBD pool name doesn't make +sense when listing the disks of a VM. We need then to resolve it to the +libvirt pool name if possible. + +Move the corresponding code from purge to get_disks. + +* virt: fix updating VM with backing store disks + +libvirt adds the index attribute in the XML definition when there is a +backing store file for a disk. We need to elude it before comparing the +sources to avoid trying to recreate disks in such cases. + +* virt: default cdrom model to ide + +cdrom devices can't use virtio. Set the default bus to ide for cdroms. + +* virt.get_disk: output the full URL for cdroms with remote image + +virt._gen_xml converts the cdroms with remote URL into a network device, +but to be coherent with the user input the virt.get_disk() function +needs to reassemble the URL in thoses cases. + +* virt.pool_delete: remove also secret + +Some pool type require a secret for the authentication on the remote +source. Remove the secrets that were added by pool_defined but leave the +others in place. + +* virt.init: cdrom format default to raw + +cdrom sources can't be of format qcow2. Force raw as the default if +needed when creating VM with source_file set for the cdrom. + +* virt.init: fix disk target names + +Fixes issue #57477. + +* virt.update: handle changing cdrom images for devices with remote source + +When a DVD device on a VM has a remote source, virt.update needs to be +able to handle detaching it and attaching a file image live. + +* virt.init: fix the name of volumes reused in disk-types pools + +Only compute the partition name if no source_file was provided by the +user for a pool of disk type. + +Co-authored-by: Blacken Salt +Co-authored-by: Firefly +Co-authored-by: Jeroen Schutrup +Co-authored-by: ch3ll + +Co-authored-by: Blacken Salt +Co-authored-by: Firefly +Co-authored-by: Jeroen Schutrup +Co-authored-by: ch3ll +--- + changelog/57005.added | 1 + + changelog/57275.fixed | 1 + + changelog/57477.fixed | 1 + + changelog/57497.fixed | 1 + + salt/modules/virt.py | 4761 ++++++++++-------- + salt/states/virt.py | 1440 +++--- + salt/templates/virt/libvirt_domain.jinja | 29 +- + salt/templates/virt/libvirt_volume.jinja | 40 +- + tests/unit/modules/test_virt.py | 5564 +++++++++++++++------- + tests/unit/states/test_virt.py | 4638 +++++++++++------- + 10 files changed, 10558 insertions(+), 5918 deletions(-) + create mode 100644 changelog/57005.added + create mode 100644 changelog/57275.fixed + create mode 100644 changelog/57477.fixed + create mode 100644 changelog/57497.fixed + +diff --git a/changelog/57005.added b/changelog/57005.added +new file mode 100644 +index 0000000000..6704541509 +--- /dev/null ++++ b/changelog/57005.added +@@ -0,0 +1 @@ ++Add support for disks volumes in virt.running state +diff --git a/changelog/57275.fixed b/changelog/57275.fixed +new file mode 100644 +index 0000000000..6efbe48315 +--- /dev/null ++++ b/changelog/57275.fixed +@@ -0,0 +1 @@ ++Remove buggy start parameter from virt.pool_running docstring +diff --git a/changelog/57477.fixed b/changelog/57477.fixed +new file mode 100644 +index 0000000000..f32f32fdfc +--- /dev/null ++++ b/changelog/57477.fixed +@@ -0,0 +1 @@ ++virt.init fix the disk target names +diff --git a/changelog/57497.fixed b/changelog/57497.fixed +new file mode 100644 +index 0000000000..24697e108d +--- /dev/null ++++ b/changelog/57497.fixed +@@ -0,0 +1 @@ ++Fix volume name for disk-typed pools in virt.defined +diff --git a/salt/modules/virt.py b/salt/modules/virt.py +index 1652f3fca1..a78c21e323 100644 +--- a/salt/modules/virt.py ++++ b/salt/modules/virt.py +@@ -1,5 +1,5 @@ + # -*- coding: utf-8 -*- +-''' ++""" + Work with virtual machines managed by libvirt + + :depends: libvirt Python module +@@ -68,7 +68,7 @@ The calls not using the libvirt connection setup are: + - `libvirt URI format `_ + - `libvirt authentication configuration `_ + +-''' ++""" + # Special Thanks to Michael Dehann, many of the concepts, and a few structures + # of his in the virt func module have been used + +@@ -76,31 +76,21 @@ The calls not using the libvirt connection setup are: + from __future__ import absolute_import, print_function, unicode_literals + import base64 + import copy ++import datetime ++import logging + import os + import re +-import sys + import shutil +-import subprocess + import string # pylint: disable=deprecated-module +-import logging ++import subprocess ++import sys + import time +-import datetime + from xml.etree import ElementTree ++from xml.sax import saxutils + + # Import third party libs + import jinja2 + import jinja2.exceptions +-try: +- import libvirt # pylint: disable=import-error +- +- # pylint: disable=no-name-in-module +- from libvirt import libvirtError +- +- # pylint: enable=no-name-in-module +- +- HAS_LIBVIRT = True +-except ImportError: +- HAS_LIBVIRT = False + + # Import salt libs + import salt.utils.files +@@ -111,49 +101,65 @@ import salt.utils.stringutils + import salt.utils.templates + import salt.utils.validate.net + import salt.utils.versions ++import salt.utils.xmlutil as xmlutil + import salt.utils.yaml +- +-from salt.utils.virt import check_remote, download_remote ++from salt._compat import ipaddress + from salt.exceptions import CommandExecutionError, SaltInvocationError + from salt.ext import six + from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +-from salt._compat import ipaddress ++from salt.ext.six.moves.urllib.parse import urlparse, urlunparse ++from salt.utils.virt import check_remote, download_remote ++ ++try: ++ import libvirt # pylint: disable=import-error ++ ++ # pylint: disable=no-name-in-module ++ from libvirt import libvirtError ++ ++ # pylint: enable=no-name-in-module ++ ++ HAS_LIBVIRT = True ++except ImportError: ++ HAS_LIBVIRT = False ++ + + log = logging.getLogger(__name__) + + # Set up template environment + JINJA = jinja2.Environment( + loader=jinja2.FileSystemLoader( +- os.path.join(salt.utils.templates.TEMPLATE_DIRNAME, 'virt') ++ os.path.join(salt.utils.templates.TEMPLATE_DIRNAME, "virt") + ) + ) + +-CACHE_DIR = '/var/lib/libvirt/saltinst' ++CACHE_DIR = "/var/lib/libvirt/saltinst" + +-VIRT_STATE_NAME_MAP = {0: 'running', +- 1: 'running', +- 2: 'running', +- 3: 'paused', +- 4: 'shutdown', +- 5: 'shutdown', +- 6: 'crashed'} ++VIRT_STATE_NAME_MAP = { ++ 0: "running", ++ 1: "running", ++ 2: "running", ++ 3: "paused", ++ 4: "shutdown", ++ 5: "shutdown", ++ 6: "crashed", ++} + + + def __virtual__(): + if not HAS_LIBVIRT: +- return (False, 'Unable to locate or import python libvirt library.') +- return 'virt' ++ return (False, "Unable to locate or import python libvirt library.") ++ return "virt" + + + def __get_request_auth(username, password): +- ''' ++ """ + Get libvirt.openAuth callback with username, password values overriding + the configuration ones. +- ''' ++ """ + + # pylint: disable=unused-argument + def __request_auth(credentials, user_data): +- '''Callback method passed to libvirt.openAuth(). ++ """Callback method passed to libvirt.openAuth(). + + The credentials argument is a list of credentials that libvirt + would like to request. An element of this list is a list containing +@@ -165,21 +171,31 @@ def __get_request_auth(username, password): + - a place to store the actual result for the request + + The user_data argument is currently not set in the openAuth call. +- ''' ++ """ + for credential in credentials: + if credential[0] == libvirt.VIR_CRED_AUTHNAME: +- credential[4] = username if username else \ +- __salt__['config.get']('virt:connection:auth:username', credential[3]) ++ credential[4] = ( ++ username ++ if username ++ else __salt__["config.get"]( ++ "virt:connection:auth:username", credential[3] ++ ) ++ ) + elif credential[0] == libvirt.VIR_CRED_NOECHOPROMPT: +- credential[4] = password if password else \ +- __salt__['config.get']('virt:connection:auth:password', credential[3]) ++ credential[4] = ( ++ password ++ if password ++ else __salt__["config.get"]( ++ "virt:connection:auth:password", credential[3] ++ ) ++ ) + else: +- log.info('Unhandled credential type: %s', credential[0]) ++ log.info("Unhandled credential type: %s", credential[0]) + return 0 + + + def __get_conn(**kwargs): +- ''' ++ """ + Detects what type of dom this node is and attempts to connect to the + correct hypervisor via libvirt. + +@@ -187,99 +203,65 @@ def __get_conn(**kwargs): + :param username: username to connect with, overriding defaults + :param password: password to connect with, overriding defaults + +- ''' ++ """ + # This has only been tested on kvm and xen, it needs to be expanded to + # support all vm layers supported by libvirt ++ # Connection string works on bhyve, but auth is not tested. + +- username = kwargs.get('username', None) +- password = kwargs.get('password', None) +- conn_str = kwargs.get('connection', None) ++ username = kwargs.get("username", None) ++ password = kwargs.get("password", None) ++ conn_str = kwargs.get("connection", None) + if not conn_str: +- conn_str = __salt__['config.get']('virt.connect', None) +- if conn_str is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'virt.connect\' configuration property has been deprecated in favor ' +- 'of \'virt:connection:uri\'. \'virt.connect\' will stop being used in ' +- '{version}.' +- ) +- else: +- conn_str = __salt__['config.get']('libvirt:connection', None) +- if conn_str is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'libvirt.connection\' configuration property has been deprecated in favor ' +- 'of \'virt:connection:uri\'. \'libvirt.connection\' will stop being used in ' +- '{version}.' +- ) +- +- conn_str = __salt__['config.get']('virt:connection:uri', conn_str) +- +- hypervisor = __salt__['config.get']('libvirt:hypervisor', None) +- if hypervisor is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'libvirt.hypervisor\' configuration property has been deprecated. ' +- 'Rather use the \'virt:connection:uri\' to properly define the libvirt ' +- 'URI or alias of the host to connect to. \'libvirt:hypervisor\' will ' +- 'stop being used in {version}.' +- ) +- +- if hypervisor == 'esxi' and conn_str is None: +- salt.utils.versions.warn_until( +- 'Sodium', +- 'esxi hypervisor default with no default connection URI detected, ' +- 'please set \'virt:connection:uri\' to \'esx\' for keep the legacy ' +- 'behavior. Will default to libvirt guess once \'libvirt:hypervisor\' ' +- 'configuration is removed in {version}.' +- ) +- conn_str = 'esx' ++ conn_str = __salt__["config.get"]("virt:connection:uri", conn_str) + + try: +- auth_types = [libvirt.VIR_CRED_AUTHNAME, +- libvirt.VIR_CRED_NOECHOPROMPT, +- libvirt.VIR_CRED_ECHOPROMPT, +- libvirt.VIR_CRED_PASSPHRASE, +- libvirt.VIR_CRED_EXTERNAL] +- conn = libvirt.openAuth(conn_str, [auth_types, __get_request_auth(username, password), None], 0) ++ auth_types = [ ++ libvirt.VIR_CRED_AUTHNAME, ++ libvirt.VIR_CRED_NOECHOPROMPT, ++ libvirt.VIR_CRED_ECHOPROMPT, ++ libvirt.VIR_CRED_PASSPHRASE, ++ libvirt.VIR_CRED_EXTERNAL, ++ ] ++ conn = libvirt.openAuth( ++ conn_str, [auth_types, __get_request_auth(username, password), None], 0 ++ ) + except Exception: # pylint: disable=broad-except + raise CommandExecutionError( +- 'Sorry, {0} failed to open a connection to the hypervisor ' +- 'software at {1}'.format( +- __grains__['fqdn'], +- conn_str +- ) ++ "Sorry, {0} failed to open a connection to the hypervisor " ++ "software at {1}".format(__grains__["fqdn"], conn_str) + ) + return conn + + + def _get_domain(conn, *vms, **kwargs): +- ''' ++ """ + Return a domain object for the named VM or return domain object for all VMs. + + :params conn: libvirt connection object + :param vms: list of domain names to look for + :param iterable: True to return an array in all cases +- ''' ++ """ + ret = list() + lookup_vms = list() + + all_vms = [] +- if kwargs.get('active', True): ++ if kwargs.get("active", True): + for id_ in conn.listDomainsID(): + all_vms.append(conn.lookupByID(id_).name()) + +- if kwargs.get('inactive', True): ++ if kwargs.get("inactive", True): + for id_ in conn.listDefinedDomains(): + all_vms.append(id_) + + if vms and not all_vms: +- raise CommandExecutionError('No virtual machines found.') ++ raise CommandExecutionError("No virtual machines found.") + + if vms: + for name in vms: + if name not in all_vms: +- raise CommandExecutionError('The VM "{name}" is not present'.format(name=name)) ++ raise CommandExecutionError( ++ 'The VM "{name}" is not present'.format(name=name) ++ ) + else: + lookup_vms.append(name) + else: +@@ -288,52 +270,69 @@ def _get_domain(conn, *vms, **kwargs): + for name in lookup_vms: + ret.append(conn.lookupByName(name)) + +- return len(ret) == 1 and not kwargs.get('iterable') and ret[0] or ret ++ return len(ret) == 1 and not kwargs.get("iterable") and ret[0] or ret + + + def _parse_qemu_img_info(info): +- ''' ++ """ + Parse qemu-img info JSON output into disk infos dictionary +- ''' ++ """ + raw_infos = salt.utils.json.loads(info) + disks = [] + for disk_infos in raw_infos: + disk = { +- 'file': disk_infos['filename'], +- 'file format': disk_infos['format'], +- 'disk size': disk_infos['actual-size'], +- 'virtual size': disk_infos['virtual-size'], +- 'cluster size': disk_infos['cluster-size'] if 'cluster-size' in disk_infos else None, +- } +- +- if 'full-backing-filename' in disk_infos.keys(): +- disk['backing file'] = format(disk_infos['full-backing-filename']) +- +- if 'snapshots' in disk_infos.keys(): +- disk['snapshots'] = [ +- { +- 'id': snapshot['id'], +- 'tag': snapshot['name'], +- 'vmsize': snapshot['vm-state-size'], +- 'date': datetime.datetime.fromtimestamp( +- float('{}.{}'.format(snapshot['date-sec'], snapshot['date-nsec']))).isoformat(), +- 'vmclock': datetime.datetime.utcfromtimestamp( +- float('{}.{}'.format(snapshot['vm-clock-sec'], +- snapshot['vm-clock-nsec']))).time().isoformat() +- } for snapshot in disk_infos['snapshots']] ++ "file": disk_infos["filename"], ++ "file format": disk_infos["format"], ++ "disk size": disk_infos["actual-size"], ++ "virtual size": disk_infos["virtual-size"], ++ "cluster size": disk_infos["cluster-size"] ++ if "cluster-size" in disk_infos ++ else None, ++ } ++ ++ if "full-backing-filename" in disk_infos.keys(): ++ disk["backing file"] = format(disk_infos["full-backing-filename"]) ++ ++ if "snapshots" in disk_infos.keys(): ++ disk["snapshots"] = [ ++ { ++ "id": snapshot["id"], ++ "tag": snapshot["name"], ++ "vmsize": snapshot["vm-state-size"], ++ "date": datetime.datetime.fromtimestamp( ++ float( ++ "{}.{}".format(snapshot["date-sec"], snapshot["date-nsec"]) ++ ) ++ ).isoformat(), ++ "vmclock": datetime.datetime.utcfromtimestamp( ++ float( ++ "{}.{}".format( ++ snapshot["vm-clock-sec"], snapshot["vm-clock-nsec"] ++ ) ++ ) ++ ) ++ .time() ++ .isoformat(), ++ } ++ for snapshot in disk_infos["snapshots"] ++ ] + disks.append(disk) + + for disk in disks: +- if 'backing file' in disk.keys(): +- candidates = [info for info in disks if 'file' in info.keys() and info['file'] == disk['backing file']] ++ if "backing file" in disk.keys(): ++ candidates = [ ++ info ++ for info in disks ++ if "file" in info.keys() and info["file"] == disk["backing file"] ++ ] + if candidates: +- disk['backing file'] = candidates[0] ++ disk["backing file"] = candidates[0] + + return disks[0] + + + def _get_uuid(dom): +- ''' ++ """ + Return a uuid from the named vm + + CLI Example: +@@ -341,12 +340,12 @@ def _get_uuid(dom): + .. code-block:: bash + + salt '*' virt.get_uuid +- ''' +- return ElementTree.fromstring(get_xml(dom)).find('uuid').text ++ """ ++ return ElementTree.fromstring(get_xml(dom)).find("uuid").text + + + def _get_on_poweroff(dom): +- ''' ++ """ + Return `on_poweroff` setting from the named vm + + CLI Example: +@@ -354,13 +353,13 @@ def _get_on_poweroff(dom): + .. code-block:: bash + + salt '*' virt.get_on_restart +- ''' +- node = ElementTree.fromstring(get_xml(dom)).find('on_poweroff') +- return node.text if node is not None else '' ++ """ ++ node = ElementTree.fromstring(get_xml(dom)).find("on_poweroff") ++ return node.text if node is not None else "" + + + def _get_on_reboot(dom): +- ''' ++ """ + Return `on_reboot` setting from the named vm + + CLI Example: +@@ -368,13 +367,13 @@ def _get_on_reboot(dom): + .. code-block:: bash + + salt '*' virt.get_on_reboot +- ''' +- node = ElementTree.fromstring(get_xml(dom)).find('on_reboot') +- return node.text if node is not None else '' ++ """ ++ node = ElementTree.fromstring(get_xml(dom)).find("on_reboot") ++ return node.text if node is not None else "" + + + def _get_on_crash(dom): +- ''' ++ """ + Return `on_crash` setting from the named vm + + CLI Example: +@@ -382,460 +381,714 @@ def _get_on_crash(dom): + .. code-block:: bash + + salt '*' virt.get_on_crash +- ''' +- node = ElementTree.fromstring(get_xml(dom)).find('on_crash') +- return node.text if node is not None else '' ++ """ ++ node = ElementTree.fromstring(get_xml(dom)).find("on_crash") ++ return node.text if node is not None else "" + + + def _get_nics(dom): +- ''' ++ """ + Get domain network interfaces from a libvirt domain object. +- ''' ++ """ + nics = {} + doc = ElementTree.fromstring(dom.XMLDesc(0)) +- for iface_node in doc.findall('devices/interface'): ++ for iface_node in doc.findall("devices/interface"): + nic = {} +- nic['type'] = iface_node.get('type') ++ nic["type"] = iface_node.get("type") + for v_node in iface_node: +- if v_node.tag == 'mac': +- nic['mac'] = v_node.get('address') +- if v_node.tag == 'model': +- nic['model'] = v_node.get('type') +- if v_node.tag == 'target': +- nic['target'] = v_node.get('dev') ++ if v_node.tag == "mac": ++ nic["mac"] = v_node.get("address") ++ if v_node.tag == "model": ++ nic["model"] = v_node.get("type") ++ if v_node.tag == "target": ++ nic["target"] = v_node.get("dev") + # driver, source, and match can all have optional attributes +- if re.match('(driver|source|address)', v_node.tag): ++ if re.match("(driver|source|address)", v_node.tag): + temp = {} + for key, value in six.iteritems(v_node.attrib): + temp[key] = value + nic[v_node.tag] = temp + # virtualport needs to be handled separately, to pick up the + # type attribute of the virtualport itself +- if v_node.tag == 'virtualport': ++ if v_node.tag == "virtualport": + temp = {} +- temp['type'] = v_node.get('type') ++ temp["type"] = v_node.get("type") + for key, value in six.iteritems(v_node.attrib): + temp[key] = value +- nic['virtualport'] = temp +- if 'mac' not in nic: ++ nic["virtualport"] = temp ++ if "mac" not in nic: + continue +- nics[nic['mac']] = nic ++ nics[nic["mac"]] = nic + return nics + + + def _get_graphics(dom): +- ''' ++ """ + Get domain graphics from a libvirt domain object. +- ''' +- out = {'autoport': 'None', +- 'keymap': 'None', +- 'listen': 'None', +- 'port': 'None', +- 'type': 'None'} ++ """ ++ out = { ++ "autoport": "None", ++ "keymap": "None", ++ "listen": "None", ++ "port": "None", ++ "type": "None", ++ } ++ doc = ElementTree.fromstring(dom.XMLDesc(0)) ++ for g_node in doc.findall("devices/graphics"): ++ for key, value in six.iteritems(g_node.attrib): ++ out[key] = value ++ return out ++ ++ ++def _get_loader(dom): ++ """ ++ Get domain loader from a libvirt domain object. ++ """ ++ out = {"path": "None"} + doc = ElementTree.fromstring(dom.XMLDesc(0)) +- for g_node in doc.findall('devices/graphics'): ++ for g_node in doc.findall("os/loader"): ++ out["path"] = g_node.text + for key, value in six.iteritems(g_node.attrib): + out[key] = value + return out + + +-def _get_disks(dom): +- ''' ++def _get_disks(conn, dom): ++ """ + Get domain disks from a libvirt domain object. +- ''' ++ """ + disks = {} + doc = ElementTree.fromstring(dom.XMLDesc(0)) +- for elem in doc.findall('devices/disk'): +- source = elem.find('source') ++ for elem in doc.findall("devices/disk"): ++ source = elem.find("source") + if source is None: + continue +- target = elem.find('target') ++ target = elem.find("target") ++ driver = elem.find("driver") + if target is None: + continue +- if 'dev' in target.attrib: +- qemu_target = source.get('file', '') +- if not qemu_target: +- qemu_target = source.get('dev', '') +- if not qemu_target and 'protocol' in source.attrib and 'name' in source.attrib: # for rbd network +- qemu_target = '{0}:{1}'.format( +- source.get('protocol'), +- source.get('name')) ++ qemu_target = None ++ extra_properties = None ++ if "dev" in target.attrib: ++ disk_type = elem.get("type") ++ if disk_type == "file": ++ qemu_target = source.get("file", "") ++ if qemu_target.startswith("/dev/zvol/"): ++ disks[target.get("dev")] = {"file": qemu_target, "zfs": True} ++ continue ++ # Extract disk sizes, snapshots, backing files ++ if elem.get("device", "disk") != "cdrom": ++ try: ++ stdout = subprocess.Popen( ++ [ ++ "qemu-img", ++ "info", ++ "-U", ++ "--output", ++ "json", ++ "--backing-chain", ++ qemu_target, ++ ], ++ shell=False, ++ stdout=subprocess.PIPE, ++ ).communicate()[0] ++ qemu_output = salt.utils.stringutils.to_str(stdout) ++ output = _parse_qemu_img_info(qemu_output) ++ extra_properties = output ++ except TypeError: ++ disk.update({"file": "Does not exist"}) ++ elif disk_type == "block": ++ qemu_target = source.get("dev", "") ++ elif disk_type == "network": ++ qemu_target = source.get("protocol") ++ source_name = source.get("name") ++ if source_name: ++ qemu_target = "{0}:{1}".format(qemu_target, source_name) ++ ++ # Reverse the magic for the rbd and gluster pools ++ if source.get("protocol") in ["rbd", "gluster"]: ++ for pool_i in conn.listAllStoragePools(): ++ pool_i_xml = ElementTree.fromstring(pool_i.XMLDesc()) ++ name_node = pool_i_xml.find("source/name") ++ if name_node is not None and source_name.startswith( ++ "{}/".format(name_node.text) ++ ): ++ qemu_target = "{}{}".format( ++ pool_i.name(), source_name[len(name_node.text) :] ++ ) ++ break ++ ++ # Reverse the magic for cdroms with remote URLs ++ if elem.get("device", "disk") == "cdrom": ++ host_node = source.find("host") ++ if host_node is not None: ++ hostname = host_node.get("name") ++ port = host_node.get("port") ++ qemu_target = urlunparse( ++ ( ++ source.get("protocol"), ++ "{}:{}".format(hostname, port) if port else hostname, ++ source_name, ++ "", ++ saxutils.unescape(source.get("query", "")), ++ "", ++ ) ++ ) ++ elif disk_type == "volume": ++ pool_name = source.get("pool") ++ volume_name = source.get("volume") ++ qemu_target = "{}/{}".format(pool_name, volume_name) ++ pool = conn.storagePoolLookupByName(pool_name) ++ vol = pool.storageVolLookupByName(volume_name) ++ vol_info = vol.info() ++ extra_properties = { ++ "virtual size": vol_info[1], ++ "disk size": vol_info[2], ++ } ++ ++ backing_files = [ ++ { ++ "file": node.find("source").get("file"), ++ "file format": node.find("format").get("type"), ++ } ++ for node in elem.findall(".//backingStore[source]") ++ ] ++ ++ if backing_files: ++ # We had the backing files in a flat list, nest them again. ++ extra_properties["backing file"] = backing_files[0] ++ parent = extra_properties["backing file"] ++ for sub_backing_file in backing_files[1:]: ++ parent["backing file"] = sub_backing_file ++ parent = sub_backing_file ++ ++ else: ++ # In some cases the backing chain is not displayed by the domain definition ++ # Try to see if we have some of it in the volume definition. ++ vol_desc = ElementTree.fromstring(vol.XMLDesc()) ++ backing_path = vol_desc.find("./backingStore/path") ++ backing_format = vol_desc.find("./backingStore/format") ++ if backing_path is not None: ++ extra_properties["backing file"] = {"file": backing_path.text} ++ if backing_format is not None: ++ extra_properties["backing file"][ ++ "file format" ++ ] = backing_format.get("type") ++ + if not qemu_target: + continue + +- disk = {'file': qemu_target, 'type': elem.get('device')} +- +- driver = elem.find('driver') +- if driver is not None and driver.get('type') == 'qcow2': +- try: +- stdout = subprocess.Popen( +- ['qemu-img', 'info', '-U', '--output', 'json', '--backing-chain', disk['file']], +- shell=False, +- stdout=subprocess.PIPE).communicate()[0] +- qemu_output = salt.utils.stringutils.to_str(stdout) +- output = _parse_qemu_img_info(qemu_output) +- disk.update(output) +- except TypeError: +- disk.update({'file': 'Does not exist'}) +- +- disks[target.get('dev')] = disk ++ disk = { ++ "file": qemu_target, ++ "type": elem.get("device"), ++ } ++ if driver is not None and "type" in driver.attrib: ++ disk["file format"] = driver.get("type") ++ if extra_properties: ++ disk.update(extra_properties) ++ ++ disks[target.get("dev")] = disk + return disks + + + def _libvirt_creds(): +- ''' ++ """ + Returns the user and group that the disk images should be owned by +- ''' +- g_cmd = 'grep ^\\s*group /etc/libvirt/qemu.conf' +- u_cmd = 'grep ^\\s*user /etc/libvirt/qemu.conf' ++ """ ++ g_cmd = "grep ^\\s*group /etc/libvirt/qemu.conf" ++ u_cmd = "grep ^\\s*user /etc/libvirt/qemu.conf" + try: +- stdout = subprocess.Popen(g_cmd, +- shell=True, +- stdout=subprocess.PIPE).communicate()[0] ++ stdout = subprocess.Popen( ++ g_cmd, shell=True, stdout=subprocess.PIPE ++ ).communicate()[0] + group = salt.utils.stringutils.to_str(stdout).split('"')[1] + except IndexError: +- group = 'root' ++ group = "root" + try: +- stdout = subprocess.Popen(u_cmd, +- shell=True, +- stdout=subprocess.PIPE).communicate()[0] ++ stdout = subprocess.Popen( ++ u_cmd, shell=True, stdout=subprocess.PIPE ++ ).communicate()[0] + user = salt.utils.stringutils.to_str(stdout).split('"')[1] + except IndexError: +- user = 'root' +- return {'user': user, 'group': group} ++ user = "root" ++ return {"user": user, "group": group} + + + def _get_migrate_command(): +- ''' ++ """ + Returns the command shared by the different migration types +- ''' +- tunnel = __salt__['config.option']('virt.tunnel') +- if tunnel: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'virt.tunnel\' has been deprecated in favor of ' +- '\'virt:tunnel\'. \'virt.tunnel\' will stop ' +- 'being used in {version}.') +- else: +- tunnel = __salt__['config.get']('virt:tunnel') ++ """ ++ tunnel = __salt__["config.get"]("virt:tunnel") + if tunnel: +- return ('virsh migrate --p2p --tunnelled --live --persistent ' +- '--undefinesource ') +- return 'virsh migrate --live --persistent --undefinesource ' ++ return ( ++ "virsh migrate --p2p --tunnelled --live --persistent " "--undefinesource " ++ ) ++ return "virsh migrate --live --persistent --undefinesource " + + + def _get_target(target, ssh): +- ''' ++ """ + Compute libvirt URL for target migration host. +- ''' +- proto = 'qemu' ++ """ ++ proto = "qemu" + if ssh: +- proto += '+ssh' +- return ' {0}://{1}/{2}'.format(proto, target, 'system') +- +- +-def _gen_xml(name, +- cpu, +- mem, +- diskp, +- nicp, +- hypervisor, +- os_type, +- arch, +- graphics=None, +- boot=None, +- **kwargs): +- ''' ++ proto += "+ssh" ++ return " {0}://{1}/{2}".format(proto, target, "system") ++ ++ ++def _gen_xml( ++ conn, ++ name, ++ cpu, ++ mem, ++ diskp, ++ nicp, ++ hypervisor, ++ os_type, ++ arch, ++ graphics=None, ++ boot=None, ++ **kwargs ++): ++ """ + Generate the XML string to define a libvirt VM +- ''' ++ """ + mem = int(mem) * 1024 # MB + context = { +- 'hypervisor': hypervisor, +- 'name': name, +- 'cpu': six.text_type(cpu), +- 'mem': six.text_type(mem), ++ "hypervisor": hypervisor, ++ "name": name, ++ "cpu": six.text_type(cpu), ++ "mem": six.text_type(mem), + } +- if hypervisor in ['qemu', 'kvm']: +- context['controller_model'] = False +- elif hypervisor == 'vmware': ++ if hypervisor in ["qemu", "kvm"]: ++ context["controller_model"] = False ++ elif hypervisor == "vmware": + # TODO: make bus and model parameterized, this works for 64-bit Linux +- context['controller_model'] = 'lsilogic' ++ context["controller_model"] = "lsilogic" + + # By default, set the graphics to listen to all addresses + if graphics: +- if 'listen' not in graphics: +- graphics['listen'] = {'type': 'address', 'address': '0.0.0.0'} +- elif 'address' not in graphics['listen'] and graphics['listen']['type'] == 'address': +- graphics['listen']['address'] = '0.0.0.0' ++ if "listen" not in graphics: ++ graphics["listen"] = {"type": "address", "address": "0.0.0.0"} ++ elif ( ++ "address" not in graphics["listen"] ++ and graphics["listen"]["type"] == "address" ++ ): ++ graphics["listen"]["address"] = "0.0.0.0" + + # Graphics of type 'none' means no graphics device at all +- if graphics.get('type', 'none') == 'none': ++ if graphics.get("type", "none") == "none": + graphics = None +- context['graphics'] = graphics ++ context["graphics"] = graphics + +- if 'boot_dev' in kwargs: +- context['boot_dev'] = [] +- for dev in kwargs['boot_dev'].split(): +- context['boot_dev'].append(dev) ++ if "boot_dev" in kwargs: ++ context["boot_dev"] = [] ++ for dev in kwargs["boot_dev"].split(): ++ context["boot_dev"].append(dev) + else: +- context['boot_dev'] = ['hd'] ++ context["boot_dev"] = ["hd"] + +- context['boot'] = boot if boot else {} ++ context["boot"] = boot if boot else {} + +- if os_type == 'xen': ++ if os_type == "xen": + # Compute the Xen PV boot method +- if __grains__['os_family'] == 'Suse': +- if not boot or not boot.get('kernel', None): +- context['boot']['kernel'] = \ +- '/usr/lib/grub2/x86_64-xen/grub.xen' +- context['boot_dev'] = [] +- +- if 'serial_type' in kwargs: +- context['serial_type'] = kwargs['serial_type'] +- if 'serial_type' in context and context['serial_type'] == 'tcp': +- if 'telnet_port' in kwargs: +- context['telnet_port'] = kwargs['telnet_port'] ++ if __grains__["os_family"] == "Suse": ++ if not boot or not boot.get("kernel", None): ++ context["boot"]["kernel"] = "/usr/lib/grub2/x86_64-xen/grub.xen" ++ context["boot_dev"] = [] ++ ++ if "serial_type" in kwargs: ++ context["serial_type"] = kwargs["serial_type"] ++ if "serial_type" in context and context["serial_type"] == "tcp": ++ if "telnet_port" in kwargs: ++ context["telnet_port"] = kwargs["telnet_port"] + else: +- context['telnet_port'] = 23023 # FIXME: use random unused port +- if 'serial_type' in context: +- if 'console' in kwargs: +- context['console'] = kwargs['console'] ++ context["telnet_port"] = 23023 # FIXME: use random unused port ++ if "serial_type" in context: ++ if "console" in kwargs: ++ context["console"] = kwargs["console"] + else: +- context['console'] = True ++ context["console"] = True + +- context['disks'] = [] +- disk_bus_map = {'virtio': 'vd', 'xen': 'xvd', 'fdc': 'fd', 'ide': 'hd'} ++ context["disks"] = [] ++ disk_bus_map = {"virtio": "vd", "xen": "xvd", "fdc": "fd", "ide": "hd"} ++ targets = [] + for i, disk in enumerate(diskp): +- prefix = disk_bus_map.get(disk['model'], 'sd') ++ prefix = disk_bus_map.get(disk["model"], "sd") + disk_context = { +- 'device': disk.get('device', 'disk'), +- 'target_dev': '{0}{1}'.format(prefix, string.ascii_lowercase[i]), +- 'disk_bus': disk['model'], +- 'type': disk['format'], +- 'index': six.text_type(i), ++ "device": disk.get("device", "disk"), ++ "target_dev": _get_disk_target(targets, len(diskp), prefix), ++ "disk_bus": disk["model"], ++ "format": disk.get("format", "raw"), ++ "index": six.text_type(i), + } +- if 'source_file' and disk['source_file']: +- disk_context['source_file'] = disk['source_file'] +- +- if hypervisor in ['qemu', 'kvm', 'bhyve', 'xen']: +- disk_context['address'] = False +- disk_context['driver'] = True +- elif hypervisor in ['esxi', 'vmware']: +- disk_context['address'] = True +- disk_context['driver'] = False +- context['disks'].append(disk_context) +- context['nics'] = nicp +- +- context['os_type'] = os_type +- context['arch'] = arch +- +- fn_ = 'libvirt_domain.jinja' ++ targets.append(disk_context["target_dev"]) ++ if disk.get("source_file"): ++ url = urlparse(disk["source_file"]) ++ if not url.scheme or not url.hostname: ++ disk_context["source_file"] = disk["source_file"] ++ disk_context["type"] = "file" ++ elif url.scheme in ["http", "https", "ftp", "ftps", "tftp"]: ++ disk_context["type"] = "network" ++ disk_context["protocol"] = url.scheme ++ disk_context["volume"] = url.path ++ disk_context["query"] = saxutils.escape(url.query) ++ disk_context["hosts"] = [{"name": url.hostname, "port": url.port}] ++ ++ elif disk.get("pool"): ++ disk_context["volume"] = disk["filename"] ++ # If we had no source_file, then we want a volume ++ pool_xml = ElementTree.fromstring( ++ conn.storagePoolLookupByName(disk["pool"]).XMLDesc() ++ ) ++ pool_type = pool_xml.get("type") ++ if pool_type in ["rbd", "gluster", "sheepdog"]: ++ # libvirt can't handle rbd, gluster and sheepdog as volumes ++ disk_context["type"] = "network" ++ disk_context["protocol"] = pool_type ++ # Copy the hosts from the pool definition ++ disk_context["hosts"] = [ ++ {"name": host.get("name"), "port": host.get("port")} ++ for host in pool_xml.findall(".//host") ++ ] ++ dir_node = pool_xml.find("./source/dir") ++ # Gluster and RBD need pool/volume name ++ name_node = pool_xml.find("./source/name") ++ if name_node is not None: ++ disk_context["volume"] = "{}/{}".format( ++ name_node.text, disk_context["volume"] ++ ) ++ # Copy the authentication if any for RBD ++ auth_node = pool_xml.find("./source/auth") ++ if auth_node is not None: ++ username = auth_node.get("username") ++ secret_node = auth_node.find("./secret") ++ usage = secret_node.get("usage") ++ if not usage: ++ # Get the usage from the UUID ++ uuid = secret_node.get("uuid") ++ usage = conn.secretLookupByUUIDString(uuid).usageID() ++ disk_context["auth"] = { ++ "type": "ceph", ++ "username": username, ++ "usage": usage, ++ } ++ else: ++ if pool_type in ["disk", "logical"]: ++ # The volume format for these types doesn't match the driver format in the VM ++ disk_context["format"] = "raw" ++ disk_context["type"] = "volume" ++ disk_context["pool"] = disk["pool"] ++ ++ else: ++ # No source and no pool is a removable device, use file type ++ disk_context["type"] = "file" ++ ++ if hypervisor in ["qemu", "kvm", "bhyve", "xen"]: ++ disk_context["address"] = False ++ disk_context["driver"] = True ++ elif hypervisor in ["esxi", "vmware"]: ++ disk_context["address"] = True ++ disk_context["driver"] = False ++ context["disks"].append(disk_context) ++ context["nics"] = nicp ++ ++ context["os_type"] = os_type ++ context["arch"] = arch ++ ++ fn_ = "libvirt_domain.jinja" + try: + template = JINJA.get_template(fn_) + except jinja2.exceptions.TemplateNotFound: +- log.error('Could not load template %s', fn_) +- return '' ++ log.error("Could not load template %s", fn_) ++ return "" + + return template.render(**context) + + +-def _gen_vol_xml(vmname, +- diskname, +- disktype, +- size, +- pool): +- ''' ++def _gen_vol_xml( ++ name, ++ size, ++ format=None, ++ allocation=0, ++ type=None, ++ permissions=None, ++ backing_store=None, ++ nocow=False, ++): ++ """ + Generate the XML string to define a libvirt storage volume +- ''' ++ """ + size = int(size) * 1024 # MB + context = { +- 'name': vmname, +- 'filename': '{0}.{1}'.format(diskname, disktype), +- 'volname': diskname, +- 'disktype': disktype, +- 'size': six.text_type(size), +- 'pool': pool, ++ "type": type, ++ "name": name, ++ "target": {"permissions": permissions, "nocow": nocow}, ++ "format": format, ++ "size": six.text_type(size), ++ "allocation": six.text_type(int(allocation) * 1024), ++ "backingStore": backing_store, + } +- fn_ = 'libvirt_volume.jinja' ++ fn_ = "libvirt_volume.jinja" + try: + template = JINJA.get_template(fn_) + except jinja2.exceptions.TemplateNotFound: +- log.error('Could not load template %s', fn_) +- return '' ++ log.error("Could not load template %s", fn_) ++ return "" + return template.render(**context) + + +-def _gen_net_xml(name, +- bridge, +- forward, +- vport, +- tag=None, +- ip_configs=None): +- ''' ++def _gen_net_xml(name, bridge, forward, vport, tag=None, ip_configs=None): ++ """ + Generate the XML string to define a libvirt network +- ''' ++ """ + context = { +- 'name': name, +- 'bridge': bridge, +- 'forward': forward, +- 'vport': vport, +- 'tag': tag, +- 'ip_configs': [{ +- 'address': ipaddress.ip_network(config['cidr']), +- 'dhcp_ranges': config.get('dhcp_ranges', []), +- } for config in ip_configs or []], ++ "name": name, ++ "bridge": bridge, ++ "forward": forward, ++ "vport": vport, ++ "tag": tag, ++ "ip_configs": [ ++ { ++ "address": ipaddress.ip_network(config["cidr"]), ++ "dhcp_ranges": config.get("dhcp_ranges", []), ++ } ++ for config in ip_configs or [] ++ ], + } +- fn_ = 'libvirt_network.jinja' ++ fn_ = "libvirt_network.jinja" + try: + template = JINJA.get_template(fn_) + except jinja2.exceptions.TemplateNotFound: +- log.error('Could not load template %s', fn_) +- return '' ++ log.error("Could not load template %s", fn_) ++ return "" + return template.render(**context) + + +-def _gen_pool_xml(name, +- ptype, +- target=None, +- permissions=None, +- source_devices=None, +- source_dir=None, +- source_adapter=None, +- source_hosts=None, +- source_auth=None, +- source_name=None, +- source_format=None, +- source_initiator=None): +- ''' ++def _gen_pool_xml( ++ name, ++ ptype, ++ target=None, ++ permissions=None, ++ source_devices=None, ++ source_dir=None, ++ source_adapter=None, ++ source_hosts=None, ++ source_auth=None, ++ source_name=None, ++ source_format=None, ++ source_initiator=None, ++): ++ """ + Generate the XML string to define a libvirt storage pool +- ''' +- hosts = [host.split(':') for host in source_hosts or []] ++ """ ++ hosts = [host.split(":") for host in source_hosts or []] + source = None +- if any([source_devices, source_dir, source_adapter, hosts, source_auth, source_name, source_format, +- source_initiator]): ++ if any( ++ [ ++ source_devices, ++ source_dir, ++ source_adapter, ++ hosts, ++ source_auth, ++ source_name, ++ source_format, ++ source_initiator, ++ ] ++ ): + source = { +- 'devices': source_devices or [], +- 'dir': source_dir if source_format != 'cifs' or not source_dir else source_dir.lstrip('/'), +- 'adapter': source_adapter, +- 'hosts': [{'name': host[0], 'port': host[1] if len(host) > 1 else None} for host in hosts], +- 'auth': source_auth, +- 'name': source_name, +- 'format': source_format, +- 'initiator': source_initiator, ++ "devices": source_devices or [], ++ "dir": source_dir ++ if source_format != "cifs" or not source_dir ++ else source_dir.lstrip("/"), ++ "adapter": source_adapter, ++ "hosts": [ ++ {"name": host[0], "port": host[1] if len(host) > 1 else None} ++ for host in hosts ++ ], ++ "auth": source_auth, ++ "name": source_name, ++ "format": source_format, ++ "initiator": source_initiator, + } + + context = { +- 'name': name, +- 'ptype': ptype, +- 'target': {'path': target, 'permissions': permissions}, +- 'source': source ++ "name": name, ++ "ptype": ptype, ++ "target": {"path": target, "permissions": permissions}, ++ "source": source, + } +- fn_ = 'libvirt_pool.jinja' ++ fn_ = "libvirt_pool.jinja" + try: + template = JINJA.get_template(fn_) + except jinja2.exceptions.TemplateNotFound: +- log.error('Could not load template %s', fn_) +- return '' ++ log.error("Could not load template %s", fn_) ++ return "" + return template.render(**context) + + + def _gen_secret_xml(auth_type, usage, description): +- ''' ++ """ + Generate a libvirt secret definition XML +- ''' ++ """ + context = { +- 'type': auth_type, +- 'usage': usage, +- 'description': description, ++ "type": auth_type, ++ "usage": usage, ++ "description": description, + } +- fn_ = 'libvirt_secret.jinja' ++ fn_ = "libvirt_secret.jinja" + try: + template = JINJA.get_template(fn_) + except jinja2.exceptions.TemplateNotFound: +- log.error('Could not load template %s', fn_) +- return '' ++ log.error("Could not load template %s", fn_) ++ return "" + return template.render(**context) + + + def _get_images_dir(): +- ''' ++ """ + Extract the images dir from the configuration. First attempts to + find legacy virt.images, then tries virt:images. +- ''' +- img_dir = __salt__['config.option']('virt.images') +- if img_dir: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'virt.images\' has been deprecated in favor of ' +- '\'virt:images\'. \'virt.images\' will stop ' +- 'being used in {version}.') +- else: +- img_dir = __salt__['config.get']('virt:images') +- +- log.debug('Image directory from config option `virt:images`' +- ' is %s', img_dir) ++ """ ++ img_dir = __salt__["config.get"]("virt:images") ++ log.debug("Image directory from config option `virt:images`" " is %s", img_dir) + return img_dir + + +-def _qemu_image_create(disk, create_overlay=False, saltenv='base'): +- ''' ++def _zfs_image_create( ++ vm_name, ++ pool, ++ disk_name, ++ hostname_property_name, ++ sparse_volume, ++ disk_size, ++ disk_image_name, ++): ++ """ ++ Clones an existing image, or creates a new one. ++ ++ When cloning an image, disk_image_name refers to the source ++ of the clone. If not specified, disk_size is used for creating ++ a new zvol, and sparse_volume determines whether to create ++ a thin provisioned volume. ++ ++ The cloned or new volume can have a ZFS property set containing ++ the vm_name. Use hostname_property_name for specifying the key ++ of this ZFS property. ++ """ ++ if not disk_image_name and not disk_size: ++ raise CommandExecutionError( ++ "Unable to create new disk {0}, please specify" ++ " the disk image name or disk size argument".format(disk_name) ++ ) ++ ++ if not pool: ++ raise CommandExecutionError( ++ "Unable to create new disk {0}, please specify" ++ " the disk pool name".format(disk_name) ++ ) ++ ++ destination_fs = os.path.join(pool, "{0}.{1}".format(vm_name, disk_name)) ++ log.debug("Image destination will be %s", destination_fs) ++ ++ existing_disk = __salt__["zfs.list"](name=pool) ++ if "error" in existing_disk: ++ raise CommandExecutionError( ++ "Unable to create new disk {0}. {1}".format( ++ destination_fs, existing_disk["error"] ++ ) ++ ) ++ elif destination_fs in existing_disk: ++ log.info( ++ "ZFS filesystem {0} already exists. Skipping creation".format( ++ destination_fs ++ ) ++ ) ++ blockdevice_path = os.path.join("/dev/zvol", pool, vm_name) ++ return blockdevice_path ++ ++ properties = {} ++ if hostname_property_name: ++ properties[hostname_property_name] = vm_name ++ ++ if disk_image_name: ++ __salt__["zfs.clone"]( ++ name_a=disk_image_name, name_b=destination_fs, properties=properties ++ ) ++ ++ elif disk_size: ++ __salt__["zfs.create"]( ++ name=destination_fs, ++ properties=properties, ++ volume_size=disk_size, ++ sparse=sparse_volume, ++ ) ++ ++ blockdevice_path = os.path.join( ++ "/dev/zvol", pool, "{0}.{1}".format(vm_name, disk_name) ++ ) ++ log.debug("Image path will be %s", blockdevice_path) ++ return blockdevice_path ++ ++ ++def _qemu_image_create(disk, create_overlay=False, saltenv="base"): ++ """ + Create the image file using specified disk_size or/and disk_image + + Return path to the created image file +- ''' +- disk_size = disk.get('size', None) +- disk_image = disk.get('image', None) ++ """ ++ disk_size = disk.get("size", None) ++ disk_image = disk.get("image", None) + + if not disk_size and not disk_image: + raise CommandExecutionError( +- 'Unable to create new disk {0}, please specify' +- ' disk size and/or disk image argument' +- .format(disk['filename']) ++ "Unable to create new disk {0}, please specify" ++ " disk size and/or disk image argument".format(disk["filename"]) + ) + +- img_dest = disk['source_file'] +- log.debug('Image destination will be %s', img_dest) ++ img_dest = disk["source_file"] ++ log.debug("Image destination will be %s", img_dest) + img_dir = os.path.dirname(img_dest) +- log.debug('Image destination directory is %s', img_dir) ++ log.debug("Image destination directory is %s", img_dir) + if not os.path.exists(img_dir): + os.makedirs(img_dir) + + if disk_image: +- log.debug('Create disk from specified image %s', disk_image) +- sfn = __salt__['cp.cache_file'](disk_image, saltenv) ++ log.debug("Create disk from specified image %s", disk_image) ++ sfn = __salt__["cp.cache_file"](disk_image, saltenv) + + qcow2 = False +- if salt.utils.path.which('qemu-img'): +- res = __salt__['cmd.run']('qemu-img info "{}"'.format(sfn)) ++ if salt.utils.path.which("qemu-img"): ++ res = __salt__["cmd.run"]('qemu-img info "{}"'.format(sfn)) + imageinfo = salt.utils.yaml.safe_load(res) +- qcow2 = imageinfo['file format'] == 'qcow2' ++ qcow2 = imageinfo["file format"] == "qcow2" + try: + if create_overlay and qcow2: +- log.info('Cloning qcow2 image %s using copy on write', sfn) +- __salt__['cmd.run']( +- 'qemu-img create -f qcow2 -o backing_file="{0}" "{1}"' +- .format(sfn, img_dest).split()) ++ log.info("Cloning qcow2 image %s using copy on write", sfn) ++ __salt__["cmd.run"]( ++ 'qemu-img create -f qcow2 -o backing_file="{0}" "{1}"'.format( ++ sfn, img_dest ++ ).split() ++ ) + else: +- log.debug('Copying %s to %s', sfn, img_dest) ++ log.debug("Copying %s to %s", sfn, img_dest) + salt.utils.files.copyfile(sfn, img_dest) + + mask = salt.utils.files.get_umask() + + if disk_size and qcow2: +- log.debug('Resize qcow2 image to %sM', disk_size) +- __salt__['cmd.run']( +- 'qemu-img resize "{0}" {1}M' +- .format(img_dest, disk_size) ++ log.debug("Resize qcow2 image to %sM", disk_size) ++ __salt__["cmd.run"]( ++ 'qemu-img resize "{0}" {1}M'.format(img_dest, disk_size) + ) + +- log.debug('Apply umask and remove exec bit') ++ log.debug("Apply umask and remove exec bit") + mode = (0o0777 ^ mask) & 0o0666 + os.chmod(img_dest, mode) + + except (IOError, OSError) as err: + raise CommandExecutionError( +- 'Problem while copying image. {0} - {1}' +- .format(disk_image, err) ++ "Problem while copying image. {0} - {1}".format(disk_image, err) + ) + + else: +@@ -844,33 +1097,105 @@ def _qemu_image_create(disk, create_overlay=False, saltenv='base'): + mask = salt.utils.files.get_umask() + + if disk_size: +- log.debug('Create empty image with size %sM', disk_size) +- __salt__['cmd.run']( +- 'qemu-img create -f {0} "{1}" {2}M' +- .format(disk.get('format', 'qcow2'), img_dest, disk_size) ++ log.debug("Create empty image with size %sM", disk_size) ++ __salt__["cmd.run"]( ++ 'qemu-img create -f {0} "{1}" {2}M'.format( ++ disk.get("format", "qcow2"), img_dest, disk_size ++ ) + ) + else: + raise CommandExecutionError( +- 'Unable to create new disk {0},' +- ' please specify argument' +- .format(img_dest) ++ "Unable to create new disk {0}," ++ " please specify argument".format(img_dest) + ) + +- log.debug('Apply umask and remove exec bit') ++ log.debug("Apply umask and remove exec bit") + mode = (0o0777 ^ mask) & 0o0666 + os.chmod(img_dest, mode) + + except (IOError, OSError) as err: + raise CommandExecutionError( +- 'Problem while creating volume {0} - {1}' +- .format(img_dest, err) ++ "Problem while creating volume {0} - {1}".format(img_dest, err) + ) + + return img_dest + + +-def _disk_profile(profile, hypervisor, disks=None, vm_name=None, image=None, pool=None, **kwargs): +- ''' ++def _seed_image(seed_cmd, img_path, name, config, install, pub_key, priv_key): ++ """ ++ Helper function to seed an existing image. Note that this doesn't ++ handle volumes. ++ """ ++ log.debug("Seeding image") ++ __salt__[seed_cmd]( ++ img_path, ++ id_=name, ++ config=config, ++ install=install, ++ pub_key=pub_key, ++ priv_key=priv_key, ++ ) ++ ++ ++def _disk_volume_create(conn, disk, seeder=None, saltenv="base"): ++ """ ++ Create a disk volume for use in a VM ++ """ ++ if disk.get("overlay_image"): ++ raise SaltInvocationError( ++ "Disk overlay_image property is not supported when creating volumes," ++ "use backing_store_path and backing_store_format instead." ++ ) ++ ++ pool = conn.storagePoolLookupByName(disk["pool"]) ++ ++ # Use existing volume if possible ++ if disk["filename"] in pool.listVolumes(): ++ return ++ ++ pool_type = ElementTree.fromstring(pool.XMLDesc()).get("type") ++ ++ backing_path = disk.get("backing_store_path") ++ backing_format = disk.get("backing_store_format") ++ backing_store = None ++ if ( ++ backing_path ++ and backing_format ++ and (disk.get("format") == "qcow2" or pool_type == "logical") ++ ): ++ backing_store = {"path": backing_path, "format": backing_format} ++ ++ if backing_store and disk.get("image"): ++ raise SaltInvocationError( ++ "Using a template image with a backing store is not possible, " ++ "choose either of them." ++ ) ++ ++ vol_xml = _gen_vol_xml( ++ disk["filename"], ++ disk.get("size", 0), ++ format=disk.get("format"), ++ backing_store=backing_store, ++ ) ++ _define_vol_xml_str(conn, vol_xml, disk.get("pool")) ++ ++ if disk.get("image"): ++ log.debug("Caching disk template image: %s", disk.get("image")) ++ cached_path = __salt__["cp.cache_file"](disk.get("image"), saltenv) ++ ++ if seeder: ++ seeder(cached_path) ++ _volume_upload( ++ conn, ++ disk["pool"], ++ disk["filename"], ++ cached_path, ++ sparse=disk.get("format") == "qcow2", ++ ) ++ ++ ++def _disk_profile(conn, profile, hypervisor, disks, vm_name): ++ """ + Gather the disk profile from the config or apply the default based + on the active hypervisor + +@@ -907,22 +1232,16 @@ def _disk_profile(profile, hypervisor, disks=None, vm_name=None, image=None, poo + + The ``format`` and ``model`` parameters are optional, and will + default to whatever is best suitable for the active hypervisor. +- ''' +- default = [{'system': +- {'size': 8192}}] +- if hypervisor == 'vmware': +- overlay = {'format': 'vmdk', +- 'model': 'scsi', +- 'device': 'disk', +- 'pool': '[{0}] '.format(pool if pool else '0')} +- elif hypervisor in ['qemu', 'kvm']: +- overlay = {'format': 'qcow2', +- 'device': 'disk', +- 'model': 'virtio'} +- elif hypervisor == 'xen': +- overlay = {'format': 'qcow2', +- 'device': 'disk', +- 'model': 'xen'} ++ """ ++ default = [{"system": {"size": 8192}}] ++ if hypervisor == "vmware": ++ overlay = {"format": "vmdk", "model": "scsi", "device": "disk"} ++ elif hypervisor in ["qemu", "kvm"]: ++ overlay = {"device": "disk", "model": "virtio"} ++ elif hypervisor == "xen": ++ overlay = {"device": "disk", "model": "xen"} ++ elif hypervisor in ["bhyve"]: ++ overlay = {"format": "raw", "model": "virtio", "sparse_volume": False} + else: + overlay = {} + +@@ -930,89 +1249,153 @@ def _disk_profile(profile, hypervisor, disks=None, vm_name=None, image=None, poo + disklist = [] + if profile: + disklist = copy.deepcopy( +- __salt__['config.get']('virt:disk', {}).get(profile, default)) ++ __salt__["config.get"]("virt:disk", {}).get(profile, default) ++ ) + + # Transform the list to remove one level of dictionnary and add the name as a property + disklist = [dict(d, name=name) for disk in disklist for name, d in disk.items()] + +- # Add the image to the first disk if there is one +- if image: +- # If image is specified in module arguments, then it will be used +- # for the first disk instead of the image from the disk profile +- log.debug('%s image from module arguments will be used for disk "%s"' +- ' instead of %s', image, disklist[0]['name'], disklist[0].get('image', "")) +- disklist[0]['image'] = image +- + # Merge with the user-provided disks definitions + if disks: + for udisk in disks: +- if 'name' in udisk: +- found = [disk for disk in disklist if udisk['name'] == disk['name']] ++ if "name" in udisk: ++ found = [disk for disk in disklist if udisk["name"] == disk["name"]] + if found: + found[0].update(udisk) + else: + disklist.append(udisk) + ++ # Get pool capabilities once to get default format later ++ pool_caps = _pool_capabilities(conn) ++ + for disk in disklist: ++ # Set default model for cdrom devices before the overlay sets the wrong one ++ if disk.get("device", "disk") == "cdrom" and "model" not in disk: ++ disk["model"] = "ide" ++ + # Add the missing properties that have defaults + for key, val in six.iteritems(overlay): + if key not in disk: + disk[key] = val + + # We may have an already computed source_file (i.e. image not created by our module) +- if 'source_file' in disk and disk['source_file']: +- disk['filename'] = os.path.basename(disk['source_file']) +- elif 'source_file' not in disk: +- _fill_disk_filename(vm_name, disk, hypervisor, **kwargs) ++ if disk.get("source_file") and os.path.exists(disk["source_file"]): ++ disk["filename"] = os.path.basename(disk["source_file"]) ++ if not disk.get("format"): ++ disk["format"] = ( ++ "qcow2" if disk.get("device", "disk") != "cdrom" else "raw" ++ ) ++ elif disk.get("device", "disk") == "disk": ++ _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps) + + return disklist + + +-def _fill_disk_filename(vm_name, disk, hypervisor, **kwargs): +- ''' ++def _fill_disk_filename(conn, vm_name, disk, hypervisor, pool_caps): ++ """ + Compute the disk file name and update it in the disk value. +- ''' +- base_dir = disk.get('pool', None) +- if hypervisor in ['qemu', 'kvm', 'xen']: ++ """ ++ # Compute the filename without extension since it may not make sense for some pool types ++ disk["filename"] = "{0}_{1}".format(vm_name, disk["name"]) ++ ++ # Compute the source file path ++ base_dir = disk.get("pool", None) ++ if hypervisor in ["qemu", "kvm", "xen"]: + # Compute the base directory from the pool property. We may have either a path + # or a libvirt pool name there. +- # If the pool is a known libvirt one with a target path, use it as target path + if not base_dir: + base_dir = _get_images_dir() ++ ++ # If the pool is a known libvirt one, skip the filename since a libvirt volume will be created later ++ if base_dir not in conn.listStoragePools(): ++ # For path-based disks, keep the qcow2 default format ++ if not disk.get("format"): ++ disk["format"] = "qcow2" ++ disk["filename"] = "{0}.{1}".format(disk["filename"], disk["format"]) ++ disk["source_file"] = os.path.join(base_dir, disk["filename"]) + else: +- if not base_dir.startswith('/'): +- # The pool seems not to be a path, lookup for pool infos +- infos = pool_info(base_dir, **kwargs) +- pool = infos[base_dir] if base_dir in infos else None +- if not pool or not pool['target_path'] or pool['target_path'].startswith('/dev'): +- raise CommandExecutionError( +- 'Unable to create new disk {0}, specified pool {1} does not exist ' +- 'or is unsupported'.format(disk['name'], base_dir)) +- base_dir = pool['target_path'] ++ if "pool" not in disk: ++ disk["pool"] = base_dir ++ pool_obj = conn.storagePoolLookupByName(base_dir) ++ pool_xml = ElementTree.fromstring(pool_obj.XMLDesc()) ++ pool_type = pool_xml.get("type") ++ ++ # Disk pools volume names are partition names, they need to be named based on the device name ++ if pool_type == "disk": ++ device = pool_xml.find("./source/device").get("path") ++ all_volumes = pool_obj.listVolumes() ++ if disk.get("source_file") not in all_volumes: ++ indexes = [ ++ int(re.sub("[a-z]+", "", vol_name)) for vol_name in all_volumes ++ ] or [0] ++ index = min( ++ [ ++ idx ++ for idx in range(1, max(indexes) + 2) ++ if idx not in indexes ++ ] ++ ) ++ disk["filename"] = "{}{}".format(os.path.basename(device), index) ++ ++ # Is the user wanting to reuse an existing volume? ++ if disk.get("source_file"): ++ if not disk.get("source_file") in pool_obj.listVolumes(): ++ raise SaltInvocationError( ++ "{} volume doesn't exist in pool {}".format( ++ disk.get("source_file"), base_dir ++ ) ++ ) ++ disk["filename"] = disk["source_file"] ++ del disk["source_file"] ++ ++ # Get the default format from the pool capabilities ++ if not disk.get("format"): ++ volume_options = ( ++ [ ++ type_caps.get("options", {}).get("volume", {}) ++ for type_caps in pool_caps.get("pool_types") ++ if type_caps["name"] == pool_type ++ ] ++ or [{}] ++ )[0] ++ # Still prefer qcow2 if possible ++ if "qcow2" in volume_options.get("targetFormatType", []): ++ disk["format"] = "qcow2" ++ else: ++ disk["format"] = volume_options.get("default_format", None) + +- # Compute the filename and source file properties if possible +- if vm_name: +- disk['filename'] = '{0}_{1}.{2}'.format(vm_name, disk['name'], disk['format']) +- disk['source_file'] = os.path.join(base_dir, disk['filename']) ++ elif hypervisor == "bhyve" and vm_name: ++ disk["filename"] = "{0}.{1}".format(vm_name, disk["name"]) ++ disk["source_file"] = os.path.join( ++ "/dev/zvol", base_dir or "", disk["filename"] ++ ) ++ ++ elif hypervisor in ["esxi", "vmware"]: ++ if not base_dir: ++ base_dir = __salt__["config.get"]("virt:storagepool", "[0] ") ++ disk["filename"] = "{0}.{1}".format(disk["filename"], disk["format"]) ++ disk["source_file"] = "{0}{1}".format(base_dir, disk["filename"]) + + +-def _complete_nics(interfaces, hypervisor, dmac=None): +- ''' ++def _complete_nics(interfaces, hypervisor): ++ """ + Complete missing data for network interfaces. +- ''' ++ """ + +- vmware_overlay = {'type': 'bridge', 'source': 'DEFAULT', 'model': 'e1000'} +- kvm_overlay = {'type': 'bridge', 'source': 'br0', 'model': 'virtio'} +- xen_overlay = {'type': 'bridge', 'source': 'br0', 'model': None} ++ vmware_overlay = {"type": "bridge", "source": "DEFAULT", "model": "e1000"} ++ kvm_overlay = {"type": "bridge", "source": "br0", "model": "virtio"} ++ xen_overlay = {"type": "bridge", "source": "br0", "model": None} ++ bhyve_overlay = {"type": "bridge", "source": "bridge0", "model": "virtio"} + overlays = { +- 'xen': xen_overlay, +- 'kvm': kvm_overlay, +- 'qemu': kvm_overlay, +- 'vmware': vmware_overlay, +- } ++ "xen": xen_overlay, ++ "kvm": kvm_overlay, ++ "qemu": kvm_overlay, ++ "vmware": vmware_overlay, ++ "bhyve": bhyve_overlay, ++ } + + def _normalize_net_types(attributes): +- ''' ++ """ + Guess which style of definition: + + bridge: br0 +@@ -1025,84 +1408,49 @@ def _complete_nics(interfaces, hypervisor, dmac=None): + + type: network + source: net0 +- ''' +- for type_ in ['bridge', 'network']: ++ """ ++ for type_ in ["bridge", "network"]: + if type_ in attributes: +- attributes['type'] = type_ ++ attributes["type"] = type_ + # we want to discard the original key +- attributes['source'] = attributes.pop(type_) ++ attributes["source"] = attributes.pop(type_) + +- attributes['type'] = attributes.get('type', None) +- attributes['source'] = attributes.get('source', None) ++ attributes["type"] = attributes.get("type", None) ++ attributes["source"] = attributes.get("source", None) + + def _apply_default_overlay(attributes): +- ''' ++ """ + Apply the default overlay to attributes +- ''' ++ """ + for key, value in six.iteritems(overlays[hypervisor]): + if key not in attributes or not attributes[key]: + attributes[key] = value + +- def _assign_mac(attributes, hypervisor): +- ''' +- Compute mac address for NIC depending on hypervisor +- ''' +- if dmac is not None: +- log.debug('Default MAC address is %s', dmac) +- if salt.utils.validate.net.mac(dmac): +- attributes['mac'] = dmac +- else: +- msg = 'Malformed MAC address: {0}'.format(dmac) +- raise CommandExecutionError(msg) +- else: +- if hypervisor in ['qemu', 'kvm']: +- attributes['mac'] = salt.utils.network.gen_mac( +- prefix='52:54:00') +- else: +- attributes['mac'] = salt.utils.network.gen_mac() +- + for interface in interfaces: + _normalize_net_types(interface) +- if interface.get('mac', None) is None: +- _assign_mac(interface, hypervisor) + if hypervisor in overlays: + _apply_default_overlay(interface) + + return interfaces + + +-def _nic_profile(profile_name, hypervisor, dmac=None): +- ''' ++def _nic_profile(profile_name, hypervisor): ++ """ + Compute NIC data based on profile +- ''' +- +- default = [{'eth0': {}}] +- +- # support old location +- config_data = __salt__['config.option']('virt.nic', {}).get( +- profile_name, None ++ """ ++ config_data = __salt__["config.get"]("virt:nic", {}).get( ++ profile_name, [{"eth0": {}}] + ) + +- if config_data is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'virt.nic\' has been deprecated in favor of \'virt:nic\'. ' +- '\'virt.nic\' will stop being used in {version}.' +- ) +- else: +- config_data = __salt__['config.get']('virt:nic', {}).get( +- profile_name, default +- ) +- + interfaces = [] + + # pylint: disable=invalid-name + def append_dict_profile_to_interface_list(profile_dict): +- ''' ++ """ + Append dictionary profile data to interfaces list +- ''' ++ """ + for interface_name, attributes in six.iteritems(profile_dict): +- attributes['name'] = interface_name ++ attributes["name"] = interface_name + interfaces.append(attributes) + + # old style dicts (top-level dicts) +@@ -1139,25 +1487,24 @@ def _nic_profile(profile_name, hypervisor, dmac=None): + else: + interfaces.append(interface) + +- # dmac can only be used from init() +- return _complete_nics(interfaces, hypervisor, dmac=dmac) ++ return _complete_nics(interfaces, hypervisor) + + +-def _get_merged_nics(hypervisor, profile, interfaces=None, dmac=None): +- ''' ++def _get_merged_nics(hypervisor, profile, interfaces=None): ++ """ + Get network devices from the profile and merge uer defined ones with them. +- ''' +- nicp = _nic_profile(profile, hypervisor, dmac=dmac) if profile else [] +- log.debug('NIC profile is %s', nicp) ++ """ ++ nicp = _nic_profile(profile, hypervisor) if profile else [] ++ log.debug("NIC profile is %s", nicp) + if interfaces: + users_nics = _complete_nics(interfaces, hypervisor) + for unic in users_nics: +- found = [nic for nic in nicp if nic['name'] == unic['name']] ++ found = [nic for nic in nicp if nic["name"] == unic["name"]] + if found: + found[0].update(unic) + else: + nicp.append(unic) +- log.debug('Merged NICs: %s', nicp) ++ log.debug("Merged NICs: %s", nicp) + return nicp + + +@@ -1173,62 +1520,60 @@ def _handle_remote_boot_params(orig_boot): + """ + saltinst_dir = None + new_boot = orig_boot.copy() ++ keys = orig_boot.keys() ++ cases = [ ++ {"loader", "nvram"}, ++ {"kernel", "initrd"}, ++ {"kernel", "initrd", "cmdline"}, ++ {"loader", "nvram", "kernel", "initrd"}, ++ {"loader", "nvram", "kernel", "initrd", "cmdline"}, ++ ] + + try: +- for key in ['kernel', 'initrd']: +- if check_remote(orig_boot.get(key)): +- if saltinst_dir is None: +- os.makedirs(CACHE_DIR) +- saltinst_dir = CACHE_DIR +- +- new_boot[key] = download_remote(orig_boot.get(key), +- saltinst_dir) +- +- return new_boot ++ if keys in cases: ++ for key in keys: ++ if orig_boot.get(key) is not None and check_remote(orig_boot.get(key)): ++ if saltinst_dir is None: ++ os.makedirs(CACHE_DIR) ++ saltinst_dir = CACHE_DIR ++ new_boot[key] = download_remote(orig_boot.get(key), saltinst_dir) ++ return new_boot ++ else: ++ raise SaltInvocationError( ++ "Invalid boot parameters, (kernel, initrd) or/and (loader, nvram) must be both present" ++ ) + except Exception as err: # pylint: disable=broad-except + raise err + + +-def init(name, +- cpu, +- mem, +- image=None, +- nic='default', +- interfaces=None, +- hypervisor=None, +- start=True, # pylint: disable=redefined-outer-name +- disk='default', +- disks=None, +- saltenv='base', +- seed=True, +- install=True, +- pub_key=None, +- priv_key=None, +- seed_cmd='seed.apply', +- enable_vnc=False, +- enable_qcow=False, +- graphics=None, +- os_type=None, +- arch=None, +- boot=None, +- **kwargs): +- ''' ++def init( ++ name, ++ cpu, ++ mem, ++ nic="default", ++ interfaces=None, ++ hypervisor=None, ++ start=True, # pylint: disable=redefined-outer-name ++ disk="default", ++ disks=None, ++ saltenv="base", ++ seed=True, ++ install=True, ++ pub_key=None, ++ priv_key=None, ++ seed_cmd="seed.apply", ++ graphics=None, ++ os_type=None, ++ arch=None, ++ boot=None, ++ **kwargs ++): ++ """ + Initialize a new vm + + :param name: name of the virtual machine to create + :param cpu: Number of virtual CPUs to assign to the virtual machine + :param mem: Amount of memory to allocate to the virtual machine in MiB. +- :param image: Path to a disk image to use as the first disk (Default: ``None``). +- Deprecated in favor of the ``disks`` parameter. To set (or change) the image of a +- disk, add the following to the disks definitions: +- +- .. code-block:: python +- +- { +- 'name': 'name_of_disk_to_change', +- 'image': '/path/to/the/image' +- } +- + :param nic: NIC profile to use (Default: ``'default'``). + The profile interfaces can be customized / extended with the interfaces parameter. + If set to ``None``, no profile will be used. +@@ -1255,17 +1600,6 @@ def init(name, + :param pub_key: public key to seed with (Default: ``None``) + :param priv_key: public key to seed with (Default: ``None``) + :param seed_cmd: Salt command to execute to seed the image. (Default: ``'seed.apply'``) +- :param enable_vnc: +- ``True`` to setup a vnc display for the VM (Default: ``False``) +- +- Deprecated in favor of the ``graphics`` parameter. Could be replaced with +- the following: +- +- .. code-block:: python +- +- graphics={'type': 'vnc'} +- +- .. deprecated:: 2019.2.0 + :param graphics: + Dictionary providing details on the graphics device to create. (Default: ``None``) + See :ref:`init-graphics-def` for more details on the possible values. +@@ -1281,51 +1615,6 @@ def init(name, + but ``x86_64`` is prefed over ``i686``. + + .. versionadded:: 2019.2.0 +- :param enable_qcow: +- ``True`` to create a QCOW2 overlay image, rather than copying the image +- (Default: ``False``). +- +- Deprecated in favor of ``disks`` parameter. Add the following to the disks +- definitions to create an overlay image of a template disk image with an +- image set: +- +- .. code-block:: python +- +- { +- 'name': 'name_of_disk_to_change', +- 'overlay_image': True +- } +- +- .. deprecated:: 2019.2.0 +- :param pool: +- Path of the folder where the image files are located for vmware/esx hypervisors. +- +- Deprecated in favor of ``disks`` parameter. Add the following to the disks +- definitions to set the vmware datastore of a disk image: +- +- .. code-block:: python +- +- { +- 'name': 'name_of_disk_to_change', +- 'pool': 'mydatastore' +- } +- +- .. deprecated:: Flurorine +- :param dmac: +- Default MAC address to use for the network interfaces. By default MAC addresses are +- automatically generated. +- +- Deprecated in favor of ``interfaces`` parameter. Add the following to the interfaces +- definitions to force the mac address of a NIC: +- +- .. code-block:: python +- +- { +- 'name': 'name_of_nic_to_change', +- 'mac': 'MY:MA:CC:ADD:RE:SS' +- } +- +- .. deprecated:: 2019.2.0 + :param config: minion configuration to use when seeding. + See :mod:`seed module for more details ` + :param boot_dev: String of space-separated devices to boot from (Default: ``'hd'``) +@@ -1342,21 +1631,48 @@ def init(name, + + .. versionadded:: 2019.2.0 + :param boot: +- Specifies kernel for the virtual machine, as well as boot parameters +- for the virtual machine. This is an optionl parameter, and all of the +- keys are optional within the dictionary. If a remote path is provided +- to kernel or initrd, salt will handle the downloading of the specified +- remote fild, and will modify the XML accordingly. ++ Specifies kernel, initial ramdisk and kernel command line parameters for the virtual machine. ++ This is an optional parameter, all of the keys are optional within the dictionary. The structure of ++ the dictionary is documented in :ref:`init-boot-def`. If a remote path is provided to kernel or initrd, ++ salt will handle the downloading of the specified remote file and modify the XML accordingly. ++ To boot VM with UEFI, specify loader and nvram path. ++ ++ .. versionadded:: 3000 + + .. code-block:: python + + { + 'kernel': '/root/f8-i386-vmlinuz', + 'initrd': '/root/f8-i386-initrd', +- 'cmdline': 'console=ttyS0 ks=http://example.com/f8-i386/os/' ++ 'cmdline': 'console=ttyS0 ks=http://example.com/f8-i386/os/', ++ 'loader': '/usr/share/OVMF/OVMF_CODE.fd', ++ 'nvram': '/usr/share/OVMF/OVMF_VARS.ms.fd' + } + +- .. versionadded:: 3000 ++ .. _init-boot-def: ++ ++ .. rubric:: Boot parameters definition ++ ++ The boot parameters dictionary can contains the following properties: ++ ++ kernel ++ The URL or path to the kernel to run the virtual machine with. ++ ++ initrd ++ The URL or path to the initrd file to run the virtual machine with. ++ ++ cmdline ++ The parameters to pass to the kernel provided in the `kernel` property. ++ ++ loader ++ The path to the UEFI binary loader to use. ++ ++ .. versionadded:: sodium ++ ++ nvram ++ The path to the UEFI data template. The file will be copied when creating the virtual machine. ++ ++ .. versionadded:: sodium + + .. _init-nic-def: + +@@ -1398,7 +1714,12 @@ def init(name, + + pool + Path to the folder or name of the pool where disks should be created. +- (Default: depends on hypervisor) ++ (Default: depends on hypervisor and the virt:storagepool configuration) ++ ++ .. versionchanged:: sodium ++ ++ If the value contains no '/', it is considered a pool name where to create a volume. ++ Using volumes will be mandatory for some pools types like rdb, iscsi, etc. + + model + One of the disk busses allowed by libvirt (Default: depends on hypervisor) +@@ -1409,19 +1730,69 @@ def init(name, + Path to the image to use for the disk. If no image is provided, an empty disk will be created + (Default: ``None``) + ++ Note that some pool types do not support uploading an image. This list can evolve with libvirt ++ versions. ++ + overlay_image + ``True`` to create a QCOW2 disk image with ``image`` as backing file. If ``False`` + the file pointed to by the ``image`` property will simply be copied. (Default: ``False``) + ++ .. versionchanged:: sodium ++ ++ This property is only valid on path-based disks, not on volumes. To create a volume with a ++ backing store, set the ``backing_store_path`` and ``backing_store_format`` properties. ++ ++ backing_store_path ++ Path to the backing store image to use. This can also be the name of a volume to use as ++ backing store within the same pool. ++ ++ .. versionadded:: sodium ++ ++ backing_store_format ++ Image format of the disk or volume to use as backing store. This property is mandatory when ++ using ``backing_store_path`` to avoid `problems `_ ++ ++ .. versionadded:: sodium ++ + source_file + Absolute path to the disk image to use. Not to be confused with ``image`` parameter. This + parameter is useful to use disk images that are created outside of this module. Can also + be ``None`` for devices that have no associated image like cdroms. + ++ .. versionchanged:: sodium ++ ++ For volume disks, this can be the name of a volume already existing in the storage pool. ++ + device + Type of device of the disk. Can be one of 'disk', 'cdrom', 'floppy' or 'lun'. + (Default: ``'disk'``) + ++ hostname_property ++ When using ZFS volumes, setting this value to a ZFS property ID will make Salt store the name of the ++ virtual machine inside this property. (Default: ``None``) ++ ++ sparse_volume ++ Boolean to specify whether to use a thin provisioned ZFS volume. ++ ++ Example profile for a bhyve VM with two ZFS disks. The first is ++ cloned from the specified image. The second disk is a thin ++ provisioned volume. ++ ++ .. code-block:: yaml ++ ++ virt: ++ disk: ++ two_zvols: ++ - system: ++ image: zroot/bhyve/CentOS-7-x86_64-v1@v1.0.5 ++ hostname_property: virt:hostname ++ pool: zroot/bhyve/guests ++ - data: ++ pool: tank/disks ++ size: 20G ++ hostname_property: virt:hostname ++ sparse_volume: True ++ + .. _init-graphics-def: + + .. rubric:: Graphics Definition +@@ -1467,162 +1838,142 @@ def init(name, + + .. _disk element: https://libvirt.org/formatdomain.html#elementsDisks + .. _graphics element: https://libvirt.org/formatdomain.html#elementsGraphics +- ''' +- caps = capabilities(**kwargs) +- os_types = sorted({guest['os_type'] for guest in caps['guests']}) +- arches = sorted({guest['arch']['name'] for guest in caps['guests']}) +- if not hypervisor: +- hypervisor = __salt__['config.get']('libvirt:hypervisor', hypervisor) +- if hypervisor is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'libvirt:hypervisor\' configuration property has been deprecated. ' +- 'Rather use the \'virt:connection:uri\' to properly define the libvirt ' +- 'URI or alias of the host to connect to. \'libvirt:hypervisor\' will ' +- 'stop being used in {version}.' +- ) +- else: ++ """ ++ try: ++ conn = __get_conn(**kwargs) ++ caps = _capabilities(conn) ++ os_types = sorted({guest["os_type"] for guest in caps["guests"]}) ++ arches = sorted({guest["arch"]["name"] for guest in caps["guests"]}) ++ ++ virt_hypervisor = hypervisor ++ if not virt_hypervisor: + # Use the machine types as possible values + # Prefer 'kvm' over the others if available +- hypervisors = sorted({x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y}) +- hypervisor = 'kvm' if 'kvm' in hypervisors else hypervisors[0] +- +- # esxi used to be a possible value for the hypervisor: map it to vmware since it's the same +- hypervisor = 'vmware' if hypervisor == 'esxi' else hypervisor +- +- log.debug('Using hypervisor %s', hypervisor) +- +- # the NICs are computed as follows: +- # 1 - get the default NICs from the profile +- # 2 - Complete the users NICS +- # 3 - Update the default NICS list to the users one, matching key is the name +- dmac = kwargs.get('dmac', None) +- if dmac: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'dmac\' parameter has been deprecated. Rather use the \'interfaces\' parameter ' +- 'to properly define the desired MAC address. \'dmac\' will be removed in {version}.' +- ) +- nicp = _get_merged_nics(hypervisor, nic, interfaces, dmac=dmac) +- +- # the disks are computed as follows: +- # 1 - get the disks defined in the profile +- # 2 - set the image on the first disk (will be removed later) +- # 3 - update the disks from the profile with the ones from the user. The matching key is the name. +- pool = kwargs.get('pool', None) +- if pool: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'pool\' parameter has been deprecated. Rather use the \'disks\' parameter ' +- 'to properly define the vmware datastore of disks. \'pool\' will be removed in {version}.' +- ) ++ hypervisors = sorted( ++ { ++ x ++ for y in [ ++ guest["arch"]["domains"].keys() for guest in caps["guests"] ++ ] ++ for x in y ++ } ++ ) ++ virt_hypervisor = "kvm" if "kvm" in hypervisors else hypervisors[0] + +- if image: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'image\' parameter has been deprecated. Rather use the \'disks\' parameter ' +- 'to override or define the image. \'image\' will be removed in {version}.' +- ) ++ # esxi used to be a possible value for the hypervisor: map it to vmware since it's the same ++ virt_hypervisor = "vmware" if virt_hypervisor == "esxi" else virt_hypervisor + +- diskp = _disk_profile(disk, hypervisor, disks, name, image=image, pool=pool, **kwargs) ++ log.debug("Using hypervisor %s", virt_hypervisor) + +- # Create multiple disks, empty or from specified images. +- for _disk in diskp: +- log.debug("Creating disk for VM [ %s ]: %s", name, _disk) ++ nicp = _get_merged_nics(virt_hypervisor, nic, interfaces) + +- if hypervisor == 'vmware': +- if 'image' in _disk: +- # TODO: we should be copying the image file onto the ESX host +- raise SaltInvocationError( +- 'virt.init does not support image ' +- 'template in conjunction with esxi hypervisor' +- ) +- else: +- # assume libvirt manages disks for us +- log.debug('Generating libvirt XML for %s', _disk) +- vol_xml = _gen_vol_xml( +- name, +- _disk['name'], +- _disk['format'], +- _disk['size'], +- _disk['pool'] +- ) +- define_vol_xml_str(vol_xml) ++ # the disks are computed as follows: ++ # 1 - get the disks defined in the profile ++ # 3 - update the disks from the profile with the ones from the user. The matching key is the name. ++ diskp = _disk_profile(conn, disk, virt_hypervisor, disks, name) + +- elif hypervisor in ['qemu', 'kvm', 'xen']: ++ # Create multiple disks, empty or from specified images. ++ for _disk in diskp: ++ # No need to create an image for cdrom devices ++ if _disk.get("device", "disk") == "cdrom": ++ continue + +- create_overlay = enable_qcow +- if create_overlay: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'enable_qcow\' parameter has been deprecated. Rather use the \'disks\' ' +- 'parameter to override or define the image. \'enable_qcow\' will be removed ' +- 'in {version}.' +- ) +- else: +- create_overlay = _disk.get('overlay_image', False) ++ log.debug("Creating disk for VM [ %s ]: %s", name, _disk) + +- if _disk['source_file']: +- if os.path.exists(_disk['source_file']): +- img_dest = _disk['source_file'] ++ if virt_hypervisor == "vmware": ++ if "image" in _disk: ++ # TODO: we should be copying the image file onto the ESX host ++ raise SaltInvocationError( ++ "virt.init does not support image " ++ "template in conjunction with esxi hypervisor" ++ ) + else: +- img_dest = _qemu_image_create(_disk, create_overlay, saltenv) +- else: +- img_dest = None +- +- # Seed only if there is an image specified +- if seed and img_dest and _disk.get('image', None): +- log.debug('Seed command is %s', seed_cmd) +- __salt__[seed_cmd]( +- img_dest, +- id_=name, +- config=kwargs.get('config'), +- install=install, +- pub_key=pub_key, +- priv_key=priv_key, ++ # assume libvirt manages disks for us ++ log.debug("Generating libvirt XML for %s", _disk) ++ volume_name = "{0}/{1}".format(name, _disk["name"]) ++ filename = "{0}.{1}".format(volume_name, _disk["format"]) ++ vol_xml = _gen_vol_xml( ++ filename, _disk["size"], format=_disk["format"] ++ ) ++ _define_vol_xml_str(conn, vol_xml, pool=_disk.get("pool")) ++ ++ elif virt_hypervisor in ["qemu", "kvm", "xen"]: ++ ++ def seeder(path): ++ _seed_image( ++ seed_cmd, ++ path, ++ name, ++ kwargs.get("config"), ++ install, ++ pub_key, ++ priv_key, ++ ) ++ ++ create_overlay = _disk.get("overlay_image", False) ++ format = _disk.get("format") ++ if _disk.get("source_file"): ++ if os.path.exists(_disk["source_file"]): ++ img_dest = _disk["source_file"] ++ else: ++ img_dest = _qemu_image_create(_disk, create_overlay, saltenv) ++ else: ++ _disk_volume_create(conn, _disk, seeder if seed else None, saltenv) ++ img_dest = None ++ ++ # Seed only if there is an image specified ++ if seed and img_dest and _disk.get("image", None): ++ seeder(img_dest) ++ ++ elif hypervisor in ["bhyve"]: ++ img_dest = _zfs_image_create( ++ vm_name=name, ++ pool=_disk.get("pool"), ++ disk_name=_disk.get("name"), ++ disk_size=_disk.get("size"), ++ disk_image_name=_disk.get("image"), ++ hostname_property_name=_disk.get("hostname_property"), ++ sparse_volume=_disk.get("sparse_volume"), + ) + +- else: +- # Unknown hypervisor +- raise SaltInvocationError( +- 'Unsupported hypervisor when handling disk image: {0}' +- .format(hypervisor) +- ) +- +- log.debug('Generating VM XML') +- +- if enable_vnc: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'enable_vnc\' parameter has been deprecated in favor of ' +- '\'graphics\'. Use graphics={\'type\': \'vnc\'} for the same behavior. ' +- '\'enable_vnc\' will be removed in {version}. ') +- graphics = {'type': 'vnc'} ++ else: ++ # Unknown hypervisor ++ raise SaltInvocationError( ++ "Unsupported hypervisor when handling disk image: {0}".format( ++ virt_hypervisor ++ ) ++ ) + +- if os_type is None: +- os_type = 'hvm' if 'hvm' in os_types else os_types[0] +- if arch is None: +- arch = 'x86_64' if 'x86_64' in arches else arches[0] ++ log.debug("Generating VM XML") ++ if os_type is None: ++ os_type = "hvm" if "hvm" in os_types else os_types[0] ++ if arch is None: ++ arch = "x86_64" if "x86_64" in arches else arches[0] + +- if boot is not None: +- boot = _handle_remote_boot_params(boot) ++ if boot is not None: ++ boot = _handle_remote_boot_params(boot) + +- vm_xml = _gen_xml(name, cpu, mem, diskp, nicp, hypervisor, os_type, arch, +- graphics, boot, **kwargs) +- conn = __get_conn(**kwargs) +- try: ++ vm_xml = _gen_xml( ++ conn, ++ name, ++ cpu, ++ mem, ++ diskp, ++ nicp, ++ virt_hypervisor, ++ os_type, ++ arch, ++ graphics, ++ boot, ++ **kwargs ++ ) + conn.defineXML(vm_xml) +- except libvirtError as err: +- # check if failure is due to this domain already existing +- if "domain '{}' already exists".format(name) in six.text_type(err): +- # continue on to seeding +- log.warning(err) +- else: +- conn.close() +- raise err # a real error we should report upwards ++ except libvirt.libvirtError as err: ++ conn.close() ++ raise CommandExecutionError(err.get_error_message()) + + if start: +- log.debug('Starting VM %s', name) ++ log.debug("Starting VM %s", name) + _get_domain(conn, name).create() + conn.close() + +@@ -1630,66 +1981,107 @@ def init(name, + + + def _disks_equal(disk1, disk2): +- ''' ++ """ + Test if two disk elements should be considered like the same device +- ''' +- target1 = disk1.find('target') +- target2 = disk2.find('target') +- source1 = ElementTree.tostring(disk1.find('source')) if disk1.find('source') is not None else None +- source2 = ElementTree.tostring(disk2.find('source')) if disk2.find('source') is not None else None ++ """ ++ target1 = disk1.find("target") ++ target2 = disk2.find("target") ++ source1 = ( ++ disk1.find("source") ++ if disk1.find("source") is not None ++ else ElementTree.Element("source") ++ ) ++ source2 = ( ++ disk2.find("source") ++ if disk2.find("source") is not None ++ else ElementTree.Element("source") ++ ) + +- return source1 == source2 and \ +- target1 is not None and target2 is not None and \ +- target1.get('bus') == target2.get('bus') and \ +- disk1.get('device', 'disk') == disk2.get('device', 'disk') and \ +- target1.get('dev') == target2.get('dev') ++ source1_dict = xmlutil.to_dict(source1, True) ++ source2_dict = xmlutil.to_dict(source2, True) ++ ++ # Remove the index added by libvirt in the source for backing chain ++ if source1_dict: ++ source1_dict.pop("index", None) ++ if source2_dict: ++ source2_dict.pop("index", None) ++ ++ return ( ++ source1_dict == source2_dict ++ and target1 is not None ++ and target2 is not None ++ and target1.get("bus") == target2.get("bus") ++ and disk1.get("device", "disk") == disk2.get("device", "disk") ++ and target1.get("dev") == target2.get("dev") ++ ) + + + def _nics_equal(nic1, nic2): +- ''' ++ """ + Test if two interface elements should be considered like the same device +- ''' ++ """ + + def _filter_nic(nic): +- ''' ++ """ + Filter out elements to ignore when comparing nics +- ''' ++ """ + return { +- 'type': nic.attrib['type'], +- 'source': nic.find('source').attrib[nic.attrib['type']] if nic.find('source') is not None else None, +- 'mac': nic.find('mac').attrib['address'].lower() if nic.find('mac') is not None else None, +- 'model': nic.find('model').attrib['type'] if nic.find('model') is not None else None, ++ "type": nic.attrib["type"], ++ "source": nic.find("source").attrib[nic.attrib["type"]] ++ if nic.find("source") is not None ++ else None, ++ "model": nic.find("model").attrib["type"] ++ if nic.find("model") is not None ++ else None, + } +- return _filter_nic(nic1) == _filter_nic(nic2) ++ ++ def _get_mac(nic): ++ return ( ++ nic.find("mac").attrib["address"].lower() ++ if nic.find("mac") is not None ++ else None ++ ) ++ ++ mac1 = _get_mac(nic1) ++ mac2 = _get_mac(nic2) ++ macs_equal = not mac1 or not mac2 or mac1 == mac2 ++ return _filter_nic(nic1) == _filter_nic(nic2) and macs_equal + + + def _graphics_equal(gfx1, gfx2): +- ''' ++ """ + Test if two graphics devices should be considered the same device +- ''' ++ """ ++ + def _filter_graphics(gfx): +- ''' ++ """ + When the domain is running, the graphics element may contain additional properties + with the default values. This function will strip down the default values. +- ''' ++ """ + gfx_copy = copy.deepcopy(gfx) + +- defaults = [{'node': '.', 'attrib': 'port', 'values': ['5900', '-1']}, +- {'node': '.', 'attrib': 'address', 'values': ['127.0.0.1']}, +- {'node': 'listen', 'attrib': 'address', 'values': ['127.0.0.1']}] ++ defaults = [ ++ {"node": ".", "attrib": "port", "values": ["5900", "-1"]}, ++ {"node": ".", "attrib": "address", "values": ["127.0.0.1"]}, ++ {"node": "listen", "attrib": "address", "values": ["127.0.0.1"]}, ++ ] + + for default in defaults: +- node = gfx_copy.find(default['node']) +- attrib = default['attrib'] +- if node is not None and (attrib not in node.attrib or node.attrib[attrib] in default['values']): +- node.set(attrib, default['values'][0]) ++ node = gfx_copy.find(default["node"]) ++ attrib = default["attrib"] ++ if node is not None and ( ++ attrib in node.attrib and node.attrib[attrib] in default["values"] ++ ): ++ node.attrib.pop(attrib) + return gfx_copy + +- return ElementTree.tostring(_filter_graphics(gfx1)) == ElementTree.tostring(_filter_graphics(gfx2)) ++ return xmlutil.to_dict(_filter_graphics(gfx1), True) == xmlutil.to_dict( ++ _filter_graphics(gfx2), True ++ ) + + + def _diff_lists(old, new, comparator): +- ''' ++ """ + Compare lists to extract the changes + + :param old: old list +@@ -1698,99 +2090,111 @@ def _diff_lists(old, new, comparator): + + The sorted list is the union of unchanged and new lists, but keeping the original + order from the new list. +- ''' ++ """ ++ + def _remove_indent(node): +- ''' ++ """ + Remove the XML indentation to compare XML trees more easily +- ''' ++ """ + node_copy = copy.deepcopy(node) + node_copy.text = None + for item in node_copy.iter(): + item.tail = None + return node_copy + +- diff = {'unchanged': [], 'new': [], 'deleted': [], 'sorted': []} ++ diff = {"unchanged": [], "new": [], "deleted": [], "sorted": []} + # We don't want to alter old since it may be used later by caller + old_devices = copy.deepcopy(old) + for new_item in new: +- found = [item for item in old_devices if comparator(_remove_indent(item), _remove_indent(new_item))] ++ found = [ ++ item ++ for item in old_devices ++ if comparator(_remove_indent(item), _remove_indent(new_item)) ++ ] + if found: + old_devices.remove(found[0]) +- diff['unchanged'].append(found[0]) +- diff['sorted'].append(found[0]) ++ diff["unchanged"].append(found[0]) ++ diff["sorted"].append(found[0]) + else: +- diff['new'].append(new_item) +- diff['sorted'].append(new_item) +- diff['deleted'] = old_devices ++ diff["new"].append(new_item) ++ diff["sorted"].append(new_item) ++ diff["deleted"] = old_devices + return diff + + ++def _get_disk_target(targets, disks_count, prefix): ++ """ ++ Compute the disk target name for a given prefix. ++ ++ :param targets: the list of already computed targets ++ :param disks: the number of disks ++ :param prefix: the prefix of the target name, i.e. "hd" ++ """ ++ return [ ++ "{0}{1}".format(prefix, string.ascii_lowercase[i]) ++ for i in range(disks_count) ++ if "{0}{1}".format(prefix, string.ascii_lowercase[i]) not in targets ++ ][0] ++ ++ + def _diff_disk_lists(old, new): +- ''' ++ """ + Compare disk definitions to extract the changes and fix target devices + + :param old: list of ElementTree nodes representing the old disks + :param new: list of ElementTree nodes representing the new disks +- ''' ++ """ + # Change the target device to avoid duplicates before diffing: this may lead + # to additional changes. Think of unchanged disk 'hda' and another disk listed + # before it becoming 'hda' too... the unchanged need to turn into 'hdb'. + targets = [] +- prefixes = ['fd', 'hd', 'vd', 'sd', 'xvd', 'ubd'] ++ prefixes = ["fd", "hd", "vd", "sd", "xvd", "ubd"] + for disk in new: +- target_node = disk.find('target') +- target = target_node.get('dev') ++ target_node = disk.find("target") ++ target = target_node.get("dev") + prefix = [item for item in prefixes if target.startswith(item)][0] +- new_target = ['{0}{1}'.format(prefix, string.ascii_lowercase[i]) for i in range(len(new)) +- if '{0}{1}'.format(prefix, string.ascii_lowercase[i]) not in targets][0] +- target_node.set('dev', new_target) ++ new_target = _get_disk_target(targets, len(new), prefix) ++ target_node.set("dev", new_target) + targets.append(new_target) + + return _diff_lists(old, new, _disks_equal) + + + def _diff_interface_lists(old, new): +- ''' ++ """ + Compare network interface definitions to extract the changes + + :param old: list of ElementTree nodes representing the old interfaces + :param new: list of ElementTree nodes representing the new interfaces +- ''' +- diff = _diff_lists(old, new, _nics_equal) +- +- # Remove duplicated addresses mac addresses and let libvirt generate them for us +- macs = [nic.find('mac').get('address') for nic in diff['unchanged']] +- for nic in diff['new']: +- mac = nic.find('mac') +- if mac.get('address') in macs: +- nic.remove(mac) +- +- return diff ++ """ ++ return _diff_lists(old, new, _nics_equal) + + + def _diff_graphics_lists(old, new): +- ''' ++ """ + Compare graphic devices definitions to extract the changes + + :param old: list of ElementTree nodes representing the old graphic devices + :param new: list of ElementTree nodes representing the new graphic devices +- ''' ++ """ + return _diff_lists(old, new, _graphics_equal) + + +-def update(name, +- cpu=0, +- mem=0, +- disk_profile=None, +- disks=None, +- nic_profile=None, +- interfaces=None, +- graphics=None, +- live=True, +- boot=None, +- test=False, +- **kwargs): +- ''' ++def update( ++ name, ++ cpu=0, ++ mem=0, ++ disk_profile=None, ++ disks=None, ++ nic_profile=None, ++ interfaces=None, ++ graphics=None, ++ live=True, ++ boot=None, ++ test=False, ++ **kwargs ++): ++ """ + Update the definition of an existing domain. + + :param name: Name of the domain to update +@@ -1825,19 +2229,13 @@ def update(name, + :param password: password to connect with, overriding defaults + + :param boot: +- Specifies kernel for the virtual machine, as well as boot parameters +- for the virtual machine. This is an optionl parameter, and all of the +- keys are optional within the dictionary. If a remote path is provided +- to kernel or initrd, salt will handle the downloading of the specified +- remote fild, and will modify the XML accordingly. ++ Specifies kernel, initial ramdisk and kernel command line parameters for the virtual machine. ++ This is an optional parameter, all of the keys are optional within the dictionary. + +- .. code-block:: python ++ Refer to :ref:`init-boot-def` for the complete boot parameter description. + +- { +- 'kernel': '/root/f8-i386-vmlinuz', +- 'initrd': '/root/f8-i386-initrd', +- 'cmdline': 'console=ttyS0 ks=http://example.com/f8-i386/os/' +- } ++ To update any boot parameters, specify the new path for each. To remove any boot parameters, ++ pass a None object, for instance: 'kernel': ``None``. + + .. versionadded:: 3000 + +@@ -1871,11 +2269,11 @@ def update(name, + + salt '*' virt.update domain cpu=2 mem=1024 + +- ''' ++ """ + status = { +- 'definition': False, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} ++ "definition": False, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, + } + conn = __get_conn(**kwargs) + domain = _get_domain(conn, name) +@@ -1883,56 +2281,68 @@ def update(name, + need_update = False + + # Compute the XML to get the disks, interfaces and graphics +- hypervisor = desc.get('type') +- all_disks = _disk_profile(disk_profile, hypervisor, disks, name, **kwargs) ++ hypervisor = desc.get("type") ++ all_disks = _disk_profile(conn, disk_profile, hypervisor, disks, name) + + if boot is not None: + boot = _handle_remote_boot_params(boot) + +- new_desc = ElementTree.fromstring(_gen_xml(name, +- cpu or 0, +- mem or 0, +- all_disks, +- _get_merged_nics(hypervisor, nic_profile, interfaces), +- hypervisor, +- domain.OSType(), +- desc.find('.//os/type').get('arch'), +- graphics, +- boot, +- **kwargs)) ++ new_desc = ElementTree.fromstring( ++ _gen_xml( ++ conn, ++ name, ++ cpu or 0, ++ mem or 0, ++ all_disks, ++ _get_merged_nics(hypervisor, nic_profile, interfaces), ++ hypervisor, ++ domain.OSType(), ++ desc.find(".//os/type").get("arch"), ++ graphics, ++ boot, ++ **kwargs ++ ) ++ ) + + # Update the cpu +- cpu_node = desc.find('vcpu') ++ cpu_node = desc.find("vcpu") + if cpu and int(cpu_node.text) != cpu: + cpu_node.text = six.text_type(cpu) +- cpu_node.set('current', six.text_type(cpu)) ++ cpu_node.set("current", six.text_type(cpu)) + need_update = True + + # Update the kernel boot parameters +- boot_tags = ['kernel', 'initrd', 'cmdline'] +- parent_tag = desc.find('os') ++ boot_tags = ["kernel", "initrd", "cmdline", "loader", "nvram"] ++ parent_tag = desc.find("os") + + # We need to search for each possible subelement, and update it. + for tag in boot_tags: + # The Existing Tag... +- found_tag = desc.find(tag) ++ found_tag = parent_tag.find(tag) + + # The new value + boot_tag_value = boot.get(tag, None) if boot else None + + # Existing tag is found and values don't match +- if found_tag and found_tag.text != boot_tag_value: ++ if found_tag is not None and found_tag.text != boot_tag_value: + + # If the existing tag is found, but the new value is None + # remove it. If the existing tag is found, and the new value + # doesn't match update it. In either case, mark for update. +- if boot_tag_value is None \ +- and boot is not None \ +- and parent_tag is not None: +- ElementTree.remove(parent_tag, tag) ++ if boot_tag_value is None and boot is not None and parent_tag is not None: ++ parent_tag.remove(found_tag) + else: + found_tag.text = boot_tag_value + ++ # If the existing tag is loader or nvram, we need to update the corresponding attribute ++ if found_tag.tag == "loader" and boot_tag_value is not None: ++ found_tag.set("readonly", "yes") ++ found_tag.set("type", "pflash") ++ ++ if found_tag.tag == "nvram" and boot_tag_value is not None: ++ found_tag.set("template", found_tag.text) ++ found_tag.text = None ++ + need_update = True + + # Existing tag is not found, but value is not None +@@ -1944,52 +2354,79 @@ def update(name, + if parent_tag is not None: + child_tag = ElementTree.SubElement(parent_tag, tag) + else: +- new_parent_tag = ElementTree.Element('os') ++ new_parent_tag = ElementTree.Element("os") + child_tag = ElementTree.SubElement(new_parent_tag, tag) + + child_tag.text = boot_tag_value ++ ++ # If the newly created tag is loader or nvram, we need to update the corresponding attribute ++ if child_tag.tag == "loader": ++ child_tag.set("readonly", "yes") ++ child_tag.set("type", "pflash") ++ ++ if child_tag.tag == "nvram": ++ child_tag.set("template", child_tag.text) ++ child_tag.text = None ++ + need_update = True + + # Update the memory, note that libvirt outputs all memory sizes in KiB +- for mem_node_name in ['memory', 'currentMemory']: ++ for mem_node_name in ["memory", "currentMemory"]: + mem_node = desc.find(mem_node_name) + if mem and int(mem_node.text) != mem * 1024: + mem_node.text = six.text_type(mem) +- mem_node.set('unit', 'MiB') ++ mem_node.set("unit", "MiB") + need_update = True + + # Update the XML definition with the new disks and diff changes +- devices_node = desc.find('devices') +- parameters = {'disk': ['disks', 'disk_profile'], +- 'interface': ['interfaces', 'nic_profile'], +- 'graphics': ['graphics']} ++ devices_node = desc.find("devices") ++ parameters = { ++ "disk": ["disks", "disk_profile"], ++ "interface": ["interfaces", "nic_profile"], ++ "graphics": ["graphics"], ++ } + changes = {} + for dev_type in parameters: + changes[dev_type] = {} + func_locals = locals() +- if [param for param in parameters[dev_type] if func_locals.get(param, None) is not None]: ++ if [ ++ param ++ for param in parameters[dev_type] ++ if func_locals.get(param, None) is not None ++ ]: + old = devices_node.findall(dev_type) +- new = new_desc.findall('devices/{0}'.format(dev_type)) +- changes[dev_type] = globals()['_diff_{0}_lists'.format(dev_type)](old, new) +- if changes[dev_type]['deleted'] or changes[dev_type]['new']: ++ new = new_desc.findall("devices/{0}".format(dev_type)) ++ changes[dev_type] = globals()["_diff_{0}_lists".format(dev_type)](old, new) ++ if changes[dev_type]["deleted"] or changes[dev_type]["new"]: + for item in old: + devices_node.remove(item) +- devices_node.extend(changes[dev_type]['sorted']) ++ devices_node.extend(changes[dev_type]["sorted"]) + need_update = True + + # Set the new definition + if need_update: + # Create missing disks if needed +- if changes['disk']: +- for idx, item in enumerate(changes['disk']['sorted']): +- source_file = all_disks[idx]['source_file'] +- if item in changes['disk']['new'] and source_file and not os.path.isfile(source_file) and not test: +- _qemu_image_create(all_disks[idx]) +- + try: ++ if changes["disk"]: ++ for idx, item in enumerate(changes["disk"]["sorted"]): ++ source_file = all_disks[idx].get("source_file") ++ # We don't want to create image disks for cdrom devices ++ if all_disks[idx].get("device", "disk") == "cdrom": ++ continue ++ if ( ++ item in changes["disk"]["new"] ++ and source_file ++ and not os.path.isfile(source_file) ++ ): ++ _qemu_image_create(all_disks[idx]) ++ elif item in changes["disk"]["new"] and not source_file: ++ _disk_volume_create(conn, all_disks[idx]) ++ + if not test: +- conn.defineXML(salt.utils.stringutils.to_str(ElementTree.tostring(desc))) +- status['definition'] = True ++ conn.defineXML( ++ salt.utils.stringutils.to_str(ElementTree.tostring(desc)) ++ ) ++ status["definition"] = True + except libvirt.libvirtError as err: + conn.close() + raise err +@@ -1998,48 +2435,133 @@ def update(name, + # From that point on, failures are not blocking to try to live update as much + # as possible. + commands = [] ++ removable_changes = [] + if domain.isActive() and live: + if cpu: +- commands.append({'device': 'cpu', +- 'cmd': 'setVcpusFlags', +- 'args': [cpu, libvirt.VIR_DOMAIN_AFFECT_LIVE]}) ++ commands.append( ++ { ++ "device": "cpu", ++ "cmd": "setVcpusFlags", ++ "args": [cpu, libvirt.VIR_DOMAIN_AFFECT_LIVE], ++ } ++ ) + if mem: +- commands.append({'device': 'mem', +- 'cmd': 'setMemoryFlags', +- 'args': [mem * 1024, libvirt.VIR_DOMAIN_AFFECT_LIVE]}) ++ commands.append( ++ { ++ "device": "mem", ++ "cmd": "setMemoryFlags", ++ "args": [mem * 1024, libvirt.VIR_DOMAIN_AFFECT_LIVE], ++ } ++ ) + +- for dev_type in ['disk', 'interface']: +- for added in changes[dev_type].get('new', []): +- commands.append({'device': dev_type, +- 'cmd': 'attachDevice', +- 'args': [salt.utils.stringutils.to_str(ElementTree.tostring(added))]}) ++ # Look for removable device source changes ++ new_disks = [] ++ for new_disk in changes["disk"].get("new", []): ++ device = new_disk.get("device", "disk") ++ if device not in ["cdrom", "floppy"]: ++ new_disks.append(new_disk) ++ continue ++ ++ target_dev = new_disk.find("target").get("dev") ++ matching = [ ++ old_disk ++ for old_disk in changes["disk"].get("deleted", []) ++ if old_disk.get("device", "disk") == device ++ and old_disk.find("target").get("dev") == target_dev ++ ] ++ if not matching: ++ new_disks.append(new_disk) ++ else: ++ # libvirt needs to keep the XML exactly as it was before ++ updated_disk = matching[0] ++ changes["disk"]["deleted"].remove(updated_disk) ++ removable_changes.append(updated_disk) ++ source_node = updated_disk.find("source") ++ new_source_node = new_disk.find("source") ++ source_file = ( ++ new_source_node.get("file") ++ if new_source_node is not None ++ else None ++ ) ++ ++ updated_disk.set("type", "file") ++ # Detaching device ++ if source_node is not None: ++ updated_disk.remove(source_node) ++ ++ # Attaching device ++ if source_file: ++ ElementTree.SubElement( ++ updated_disk, "source", attrib={"file": source_file} ++ ) ++ ++ changes["disk"]["new"] = new_disks ++ ++ for dev_type in ["disk", "interface"]: ++ for added in changes[dev_type].get("new", []): ++ commands.append( ++ { ++ "device": dev_type, ++ "cmd": "attachDevice", ++ "args": [ ++ salt.utils.stringutils.to_str( ++ ElementTree.tostring(added) ++ ) ++ ], ++ } ++ ) ++ ++ for removed in changes[dev_type].get("deleted", []): ++ commands.append( ++ { ++ "device": dev_type, ++ "cmd": "detachDevice", ++ "args": [ ++ salt.utils.stringutils.to_str( ++ ElementTree.tostring(removed) ++ ) ++ ], ++ } ++ ) + +- for removed in changes[dev_type].get('deleted', []): +- commands.append({'device': dev_type, +- 'cmd': 'detachDevice', +- 'args': [salt.utils.stringutils.to_str(ElementTree.tostring(removed))]}) ++ for updated_disk in removable_changes: ++ commands.append( ++ { ++ "device": "disk", ++ "cmd": "updateDeviceFlags", ++ "args": [ ++ salt.utils.stringutils.to_str( ++ ElementTree.tostring(updated_disk) ++ ) ++ ], ++ } ++ ) + + for cmd in commands: + try: +- ret = getattr(domain, cmd['cmd'])(*cmd['args']) if not test else 0 +- device_type = cmd['device'] +- if device_type in ['cpu', 'mem']: ++ ret = getattr(domain, cmd["cmd"])(*cmd["args"]) if not test else 0 ++ device_type = cmd["device"] ++ if device_type in ["cpu", "mem"]: + status[device_type] = not bool(ret) + else: +- actions = {'attachDevice': 'attached', 'detachDevice': 'detached'} +- status[device_type][actions[cmd['cmd']]].append(cmd['args'][0]) ++ actions = { ++ "attachDevice": "attached", ++ "detachDevice": "detached", ++ "updateDeviceFlags": "updated", ++ } ++ status[device_type][actions[cmd["cmd"]]].append(cmd["args"][0]) + + except libvirt.libvirtError as err: +- if 'errors' not in status: +- status['errors'] = [] +- status['errors'].append(six.text_type(err)) ++ if "errors" not in status: ++ status["errors"] = [] ++ status["errors"].append(six.text_type(err)) + + conn.close() + return status + + + def list_domains(**kwargs): +- ''' ++ """ + Return a list of available domains. + + :param connection: libvirt connection URI, overriding defaults +@@ -2057,7 +2579,7 @@ def list_domains(**kwargs): + .. code-block:: bash + + salt '*' virt.list_domains +- ''' ++ """ + vms = [] + conn = __get_conn(**kwargs) + for dom in _get_domain(conn, iterable=True): +@@ -2067,7 +2589,7 @@ def list_domains(**kwargs): + + + def list_active_vms(**kwargs): +- ''' ++ """ + Return a list of names for active virtual machine on the minion + + :param connection: libvirt connection URI, overriding defaults +@@ -2085,7 +2607,7 @@ def list_active_vms(**kwargs): + .. code-block:: bash + + salt '*' virt.list_active_vms +- ''' ++ """ + vms = [] + conn = __get_conn(**kwargs) + for dom in _get_domain(conn, iterable=True, inactive=False): +@@ -2095,7 +2617,7 @@ def list_active_vms(**kwargs): + + + def list_inactive_vms(**kwargs): +- ''' ++ """ + Return a list of names for inactive virtual machine on the minion + + :param connection: libvirt connection URI, overriding defaults +@@ -2113,7 +2635,7 @@ def list_inactive_vms(**kwargs): + .. code-block:: bash + + salt '*' virt.list_inactive_vms +- ''' ++ """ + vms = [] + conn = __get_conn(**kwargs) + for dom in _get_domain(conn, iterable=True, active=False): +@@ -2123,7 +2645,7 @@ def list_inactive_vms(**kwargs): + + + def vm_info(vm_=None, **kwargs): +- ''' ++ """ + Return detailed information about the vms on this hyper in a + list of dicts: + +@@ -2158,38 +2680,43 @@ def vm_info(vm_=None, **kwargs): + + .. code-block:: bash + +- salt '*' virt.vm_info +- ''' +- def _info(dom): +- ''' ++ salt '*' virt.vm_info ++ """ ++ ++ def _info(conn, dom): ++ """ + Compute the infos of a domain +- ''' ++ """ + raw = dom.info() +- return {'cpu': raw[3], +- 'cputime': int(raw[4]), +- 'disks': _get_disks(dom), +- 'graphics': _get_graphics(dom), +- 'nics': _get_nics(dom), +- 'uuid': _get_uuid(dom), +- 'on_crash': _get_on_crash(dom), +- 'on_reboot': _get_on_reboot(dom), +- 'on_poweroff': _get_on_poweroff(dom), +- 'maxMem': int(raw[1]), +- 'mem': int(raw[2]), +- 'state': VIRT_STATE_NAME_MAP.get(raw[0], 'unknown')} ++ return { ++ "cpu": raw[3], ++ "cputime": int(raw[4]), ++ "disks": _get_disks(conn, dom), ++ "graphics": _get_graphics(dom), ++ "nics": _get_nics(dom), ++ "uuid": _get_uuid(dom), ++ "loader": _get_loader(dom), ++ "on_crash": _get_on_crash(dom), ++ "on_reboot": _get_on_reboot(dom), ++ "on_poweroff": _get_on_poweroff(dom), ++ "maxMem": int(raw[1]), ++ "mem": int(raw[2]), ++ "state": VIRT_STATE_NAME_MAP.get(raw[0], "unknown"), ++ } ++ + info = {} + conn = __get_conn(**kwargs) + if vm_: +- info[vm_] = _info(_get_domain(conn, vm_)) ++ info[vm_] = _info(conn, _get_domain(conn, vm_)) + else: + for domain in _get_domain(conn, iterable=True): +- info[domain.name()] = _info(domain) ++ info[domain.name()] = _info(conn, domain) + conn.close() + return info + + + def vm_state(vm_=None, **kwargs): +- ''' ++ """ + Return list of all the vms and their state. + + If you pass a VM name in as an argument then it will return info +@@ -2211,15 +2738,17 @@ def vm_state(vm_=None, **kwargs): + .. code-block:: bash + + salt '*' virt.vm_state +- ''' ++ """ ++ + def _info(dom): +- ''' ++ """ + Compute domain state +- ''' +- state = '' ++ """ ++ state = "" + raw = dom.info() +- state = VIRT_STATE_NAME_MAP.get(raw[0], 'unknown') ++ state = VIRT_STATE_NAME_MAP.get(raw[0], "unknown") + return state ++ + info = {} + conn = __get_conn(**kwargs) + if vm_: +@@ -2232,23 +2761,25 @@ def vm_state(vm_=None, **kwargs): + + + def _node_info(conn): +- ''' ++ """ + Internal variant of node_info taking a libvirt connection as parameter +- ''' ++ """ + raw = conn.getInfo() +- info = {'cpucores': raw[6], +- 'cpumhz': raw[3], +- 'cpumodel': six.text_type(raw[0]), +- 'cpus': raw[2], +- 'cputhreads': raw[7], +- 'numanodes': raw[4], +- 'phymemory': raw[1], +- 'sockets': raw[5]} ++ info = { ++ "cpucores": raw[6], ++ "cpumhz": raw[3], ++ "cpumodel": six.text_type(raw[0]), ++ "cpus": raw[2], ++ "cputhreads": raw[7], ++ "numanodes": raw[4], ++ "phymemory": raw[1], ++ "sockets": raw[5], ++ } + return info + + + def node_info(**kwargs): +- ''' ++ """ + Return a dict with information about this node + + :param connection: libvirt connection URI, overriding defaults +@@ -2266,7 +2797,7 @@ def node_info(**kwargs): + .. code-block:: bash + + salt '*' virt.node_info +- ''' ++ """ + conn = __get_conn(**kwargs) + info = _node_info(conn) + conn.close() +@@ -2274,7 +2805,7 @@ def node_info(**kwargs): + + + def get_nics(vm_, **kwargs): +- ''' ++ """ + Return info about the network interfaces of a named vm + + :param vm_: name of the domain +@@ -2293,7 +2824,7 @@ def get_nics(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.get_nics +- ''' ++ """ + conn = __get_conn(**kwargs) + nics = _get_nics(_get_domain(conn, vm_)) + conn.close() +@@ -2301,7 +2832,7 @@ def get_nics(vm_, **kwargs): + + + def get_macs(vm_, **kwargs): +- ''' ++ """ + Return a list off MAC addresses from the named vm + + :param vm_: name of the domain +@@ -2320,13 +2851,13 @@ def get_macs(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.get_macs +- ''' ++ """ + doc = ElementTree.fromstring(get_xml(vm_, **kwargs)) +- return [node.get('address') for node in doc.findall('devices/interface/mac')] ++ return [node.get("address") for node in doc.findall("devices/interface/mac")] + + + def get_graphics(vm_, **kwargs): +- ''' ++ """ + Returns the information on vnc for a given vm + + :param vm_: name of the domain +@@ -2345,15 +2876,40 @@ def get_graphics(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.get_graphics +- ''' ++ """ + conn = __get_conn(**kwargs) + graphics = _get_graphics(_get_domain(conn, vm_)) + conn.close() + return graphics + + ++def get_loader(vm_, **kwargs): ++ """ ++ Returns the information on the loader for a given vm ++ ++ :param vm_: name of the domain ++ :param connection: libvirt connection URI, overriding defaults ++ :param username: username to connect with, overriding defaults ++ :param password: password to connect with, overriding defaults ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' virt.get_loader ++ ++ .. versionadded:: Fluorine ++ """ ++ conn = __get_conn(**kwargs) ++ try: ++ loader = _get_loader(_get_domain(conn, vm_)) ++ return loader ++ finally: ++ conn.close() ++ ++ + def get_disks(vm_, **kwargs): +- ''' ++ """ + Return the disks of a named vm + + :param vm_: name of the domain +@@ -2372,15 +2928,15 @@ def get_disks(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.get_disks +- ''' ++ """ + conn = __get_conn(**kwargs) +- disks = _get_disks(_get_domain(conn, vm_)) ++ disks = _get_disks(conn, _get_domain(conn, vm_)) + conn.close() + return disks + + + def setmem(vm_, memory, config=False, **kwargs): +- ''' ++ """ + Changes the amount of memory allocated to VM. The VM must be shutdown + for this to work. + +@@ -2403,11 +2959,11 @@ def setmem(vm_, memory, config=False, **kwargs): + + salt '*' virt.setmem + salt '*' virt.setmem my_domain 768 +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + +- if VIRT_STATE_NAME_MAP.get(dom.info()[0], 'unknown') != 'shutdown': ++ if VIRT_STATE_NAME_MAP.get(dom.info()[0], "unknown") != "shutdown": + return False + + # libvirt has a funny bitwise system for the flags in that the flag +@@ -2427,7 +2983,7 @@ def setmem(vm_, memory, config=False, **kwargs): + + + def setvcpus(vm_, vcpus, config=False, **kwargs): +- ''' ++ """ + Changes the amount of vcpus allocated to VM. The VM must be shutdown + for this to work. + +@@ -2452,11 +3008,11 @@ def setvcpus(vm_, vcpus, config=False, **kwargs): + + salt '*' virt.setvcpus + salt '*' virt.setvcpus my_domain 4 +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + +- if VIRT_STATE_NAME_MAP.get(dom.info()[0], 'unknown') != 'shutdown': ++ if VIRT_STATE_NAME_MAP.get(dom.info()[0], "unknown") != "shutdown": + return False + + # see notes in setmem +@@ -2473,9 +3029,9 @@ def setvcpus(vm_, vcpus, config=False, **kwargs): + + + def _freemem(conn): +- ''' ++ """ + Internal variant of freemem taking a libvirt connection as parameter +- ''' ++ """ + mem = conn.getInfo()[1] + # Take off just enough to sustain the hypervisor + mem -= 256 +@@ -2486,7 +3042,7 @@ def _freemem(conn): + + + def freemem(**kwargs): +- ''' ++ """ + Return an int representing the amount of memory (in MB) that has not + been given to virtual machines on this node + +@@ -2505,7 +3061,7 @@ def freemem(**kwargs): + .. code-block:: bash + + salt '*' virt.freemem +- ''' ++ """ + conn = __get_conn(**kwargs) + mem = _freemem(conn) + conn.close() +@@ -2513,9 +3069,9 @@ def freemem(**kwargs): + + + def _freecpu(conn): +- ''' ++ """ + Internal variant of freecpu taking a libvirt connection as parameter +- ''' ++ """ + cpus = conn.getInfo()[2] + for dom in _get_domain(conn, iterable=True): + if dom.ID() > 0: +@@ -2524,7 +3080,7 @@ def _freecpu(conn): + + + def freecpu(**kwargs): +- ''' ++ """ + Return an int representing the number of unallocated cpus on this + hypervisor + +@@ -2543,7 +3099,7 @@ def freecpu(**kwargs): + .. code-block:: bash + + salt '*' virt.freecpu +- ''' ++ """ + conn = __get_conn(**kwargs) + cpus = _freecpu(conn) + conn.close() +@@ -2551,7 +3107,7 @@ def freecpu(**kwargs): + + + def full_info(**kwargs): +- ''' ++ """ + Return the node_info, vm_info and freemem + + :param connection: libvirt connection URI, overriding defaults +@@ -2569,18 +3125,20 @@ def full_info(**kwargs): + .. code-block:: bash + + salt '*' virt.full_info +- ''' ++ """ + conn = __get_conn(**kwargs) +- info = {'freecpu': _freecpu(conn), +- 'freemem': _freemem(conn), +- 'node_info': _node_info(conn), +- 'vm_info': vm_info()} ++ info = { ++ "freecpu": _freecpu(conn), ++ "freemem": _freemem(conn), ++ "node_info": _node_info(conn), ++ "vm_info": vm_info(), ++ } + conn.close() + return info + + + def get_xml(vm_, **kwargs): +- ''' ++ """ + Returns the XML for a given vm + + :param vm_: domain name +@@ -2599,17 +3157,19 @@ def get_xml(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.get_xml +- ''' ++ """ + conn = __get_conn(**kwargs) +- xml_desc = vm_.XMLDesc(0) if isinstance( +- vm_, libvirt.virDomain +- ) else _get_domain(conn, vm_).XMLDesc(0) ++ xml_desc = ( ++ vm_.XMLDesc(0) ++ if isinstance(vm_, libvirt.virDomain) ++ else _get_domain(conn, vm_).XMLDesc(0) ++ ) + conn.close() + return xml_desc + + + def get_profiles(hypervisor=None, **kwargs): +- ''' ++ """ + Return the virt profiles for hypervisor. + + Currently there are profiles for: +@@ -2634,40 +3194,42 @@ def get_profiles(hypervisor=None, **kwargs): + + salt '*' virt.get_profiles + salt '*' virt.get_profiles hypervisor=esxi +- ''' ++ """ + ret = {} + ++ # Use the machine types as possible values ++ # Prefer 'kvm' over the others if available + caps = capabilities(**kwargs) +- hypervisors = sorted({x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y}) +- default_hypervisor = 'kvm' if 'kvm' in hypervisors else hypervisors[0] ++ hypervisors = sorted( ++ { ++ x ++ for y in [guest["arch"]["domains"].keys() for guest in caps["guests"]] ++ for x in y ++ } ++ ) ++ default_hypervisor = "kvm" if "kvm" in hypervisors else hypervisors[0] + + if not hypervisor: +- hypervisor = __salt__['config.get']('libvirt:hypervisor') +- if hypervisor is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'libvirt:hypervisor\' configuration property has been deprecated. ' +- 'Rather use the \'virt:connection:uri\' to properly define the libvirt ' +- 'URI or alias of the host to connect to. \'libvirt:hypervisor\' will ' +- 'stop being used in {version}.' ++ hypervisor = default_hypervisor ++ virtconf = __salt__["config.get"]("virt", {}) ++ for typ in ["disk", "nic"]: ++ _func = getattr(sys.modules[__name__], "_{0}_profile".format(typ)) ++ ret[typ] = { ++ "default": _func( ++ "default", hypervisor if hypervisor else default_hypervisor + ) +- else: +- # Use the machine types as possible values +- # Prefer 'kvm' over the others if available +- hypervisor = default_hypervisor +- virtconf = __salt__['config.get']('virt', {}) +- for typ in ['disk', 'nic']: +- _func = getattr(sys.modules[__name__], '_{0}_profile'.format(typ)) +- ret[typ] = {'default': _func('default', hypervisor)} ++ } + if typ in virtconf: + ret.setdefault(typ, {}) + for prf in virtconf[typ]: +- ret[typ][prf] = _func(prf, hypervisor) ++ ret[typ][prf] = _func( ++ prf, hypervisor if hypervisor else default_hypervisor ++ ) + return ret + + + def shutdown(vm_, **kwargs): +- ''' ++ """ + Send a soft shutdown signal to the named vm + + :param vm_: domain name +@@ -2686,7 +3248,7 @@ def shutdown(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.shutdown +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + ret = dom.shutdown() == 0 +@@ -2695,7 +3257,7 @@ def shutdown(vm_, **kwargs): + + + def pause(vm_, **kwargs): +- ''' ++ """ + Pause the named vm + + :param vm_: domain name +@@ -2714,7 +3276,7 @@ def pause(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.pause +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + ret = dom.suspend() == 0 +@@ -2723,7 +3285,7 @@ def pause(vm_, **kwargs): + + + def resume(vm_, **kwargs): +- ''' ++ """ + Resume the named vm + + :param vm_: domain name +@@ -2742,7 +3304,7 @@ def resume(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.resume +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + ret = dom.resume() == 0 +@@ -2751,7 +3313,7 @@ def resume(vm_, **kwargs): + + + def start(name, **kwargs): +- ''' ++ """ + Start a defined domain + + :param vm_: domain name +@@ -2770,7 +3332,7 @@ def start(name, **kwargs): + .. code-block:: bash + + salt '*' virt.start +- ''' ++ """ + conn = __get_conn(**kwargs) + ret = _get_domain(conn, name).create() == 0 + conn.close() +@@ -2778,7 +3340,7 @@ def start(name, **kwargs): + + + def stop(name, **kwargs): +- ''' ++ """ + Hard power down the virtual machine, this is equivalent to pulling the power. + + :param vm_: domain name +@@ -2797,7 +3359,7 @@ def stop(name, **kwargs): + .. code-block:: bash + + salt '*' virt.stop +- ''' ++ """ + conn = __get_conn(**kwargs) + ret = _get_domain(conn, name).destroy() == 0 + conn.close() +@@ -2805,7 +3367,7 @@ def stop(name, **kwargs): + + + def reboot(name, **kwargs): +- ''' ++ """ + Reboot a domain via ACPI request + + :param vm_: domain name +@@ -2824,7 +3386,7 @@ def reboot(name, **kwargs): + .. code-block:: bash + + salt '*' virt.reboot +- ''' ++ """ + conn = __get_conn(**kwargs) + ret = _get_domain(conn, name).reboot(libvirt.VIR_DOMAIN_REBOOT_DEFAULT) == 0 + conn.close() +@@ -2832,7 +3394,7 @@ def reboot(name, **kwargs): + + + def reset(vm_, **kwargs): +- ''' ++ """ + Reset a VM by emulating the reset button on a physical machine + + :param vm_: domain name +@@ -2851,7 +3413,7 @@ def reset(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.reset +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + +@@ -2864,7 +3426,7 @@ def reset(vm_, **kwargs): + + + def ctrl_alt_del(vm_, **kwargs): +- ''' ++ """ + Sends CTRL+ALT+DEL to a VM + + :param vm_: domain name +@@ -2883,7 +3445,7 @@ def ctrl_alt_del(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.ctrl_alt_del +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + ret = dom.sendKey(0, 0, [29, 56, 111], 3, 0) == 0 +@@ -2892,7 +3454,7 @@ def ctrl_alt_del(vm_, **kwargs): + + + def create_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name +- ''' ++ """ + Start a transient domain based on the XML passed to the function + + :param xml: libvirt XML definition of the domain +@@ -2911,7 +3473,7 @@ def create_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name + .. code-block:: bash + + salt '*' virt.create_xml_str +- ''' ++ """ + conn = __get_conn(**kwargs) + ret = conn.createXML(xml, 0) is not None + conn.close() +@@ -2919,7 +3481,7 @@ def create_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name + + + def create_xml_path(path, **kwargs): +- ''' ++ """ + Start a transient domain based on the XML-file path passed to the function + + :param path: path to a file containing the libvirt XML definition of the domain +@@ -2938,19 +3500,18 @@ def create_xml_path(path, **kwargs): + .. code-block:: bash + + salt '*' virt.create_xml_path +- ''' ++ """ + try: +- with salt.utils.files.fopen(path, 'r') as fp_: ++ with salt.utils.files.fopen(path, "r") as fp_: + return create_xml_str( +- salt.utils.stringutils.to_unicode(fp_.read()), +- **kwargs ++ salt.utils.stringutils.to_unicode(fp_.read()), **kwargs + ) + except (OSError, IOError): + return False + + + def define_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name +- ''' ++ """ + Define a persistent domain based on the XML passed to the function + + :param xml: libvirt XML definition of the domain +@@ -2969,7 +3530,7 @@ def define_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name + .. code-block:: bash + + salt '*' virt.define_xml_str +- ''' ++ """ + conn = __get_conn(**kwargs) + ret = conn.defineXML(xml) is not None + conn.close() +@@ -2977,7 +3538,7 @@ def define_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name + + + def define_xml_path(path, **kwargs): +- ''' ++ """ + Define a persistent domain based on the XML-file path passed to the function + + :param path: path to a file containing the libvirt XML definition of the domain +@@ -2997,22 +3558,41 @@ def define_xml_path(path, **kwargs): + + salt '*' virt.define_xml_path + +- ''' ++ """ + try: +- with salt.utils.files.fopen(path, 'r') as fp_: ++ with salt.utils.files.fopen(path, "r") as fp_: + return define_xml_str( +- salt.utils.stringutils.to_unicode(fp_.read()), +- **kwargs ++ salt.utils.stringutils.to_unicode(fp_.read()), **kwargs + ) + except (OSError, IOError): + return False + + +-def define_vol_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name +- ''' ++def _define_vol_xml_str(conn, xml, pool=None): # pylint: disable=redefined-outer-name ++ """ ++ Same function than define_vml_xml_str but using an already opened libvirt connection ++ """ ++ default_pool = "default" if conn.getType() != "ESX" else "0" ++ poolname = ( ++ pool if pool else __salt__["config.get"]("virt:storagepool", default_pool) ++ ) ++ pool = conn.storagePoolLookupByName(six.text_type(poolname)) ++ ret = pool.createXML(xml, 0) is not None ++ return ret ++ ++ ++def define_vol_xml_str( ++ xml, pool=None, **kwargs ++): # pylint: disable=redefined-outer-name ++ """ + Define a volume based on the XML passed to the function + + :param xml: libvirt XML definition of the storage volume ++ :param pool: ++ storage pool name to define the volume in. ++ If defined, this parameter will override the configuration setting. ++ ++ .. versionadded:: Sodium + :param connection: libvirt connection URI, overriding defaults + + .. versionadded:: 2019.2.0 +@@ -3030,36 +3610,34 @@ def define_vol_xml_str(xml, **kwargs): # pylint: disable=redefined-outer-name + salt '*' virt.define_vol_xml_str + + The storage pool where the disk image will be defined is ``default`` +- unless changed with a configuration like this: ++ unless changed with the pool parameter or a configuration like this: + + .. code-block:: yaml + + virt: + storagepool: mine +- ''' +- poolname = __salt__['config.get']('libvirt:storagepool', None) +- if poolname is not None: +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'libvirt:storagepool\' has been deprecated in favor of ' +- '\'virt:storagepool\'. \'libvirt:storagepool\' will stop ' +- 'being used in {version}.' +- ) +- else: +- poolname = __salt__['config.get']('virt:storagepool', 'default') +- ++ """ + conn = __get_conn(**kwargs) +- pool = conn.storagePoolLookupByName(six.text_type(poolname)) +- ret = pool.createXML(xml, 0) is not None +- conn.close() ++ ret = False ++ try: ++ ret = _define_vol_xml_str(conn, xml, pool=pool) ++ except libvirtError as err: ++ raise CommandExecutionError(err.get_error_message()) ++ finally: ++ conn.close() + return ret + + +-def define_vol_xml_path(path, **kwargs): +- ''' ++def define_vol_xml_path(path, pool=None, **kwargs): ++ """ + Define a volume based on the XML-file path passed to the function + + :param path: path to a file containing the libvirt XML definition of the volume ++ :param pool: ++ storage pool name to define the volume in. ++ If defined, this parameter will override the configuration setting. ++ ++ .. versionadded:: Sodium + :param connection: libvirt connection URI, overriding defaults + + .. versionadded:: 2019.2.0 +@@ -3076,19 +3654,18 @@ def define_vol_xml_path(path, **kwargs): + + salt '*' virt.define_vol_xml_path + +- ''' ++ """ + try: +- with salt.utils.files.fopen(path, 'r') as fp_: ++ with salt.utils.files.fopen(path, "r") as fp_: + return define_vol_xml_str( +- salt.utils.stringutils.to_unicode(fp_.read()), +- **kwargs ++ salt.utils.stringutils.to_unicode(fp_.read()), pool=pool, **kwargs + ) + except (OSError, IOError): + return False + + + def migrate_non_shared(vm_, target, ssh=False): +- ''' ++ """ + Attempt to execute non-shared storage "all" migration + + :param vm_: domain name +@@ -3111,18 +3688,17 @@ def migrate_non_shared(vm_, target, ssh=False): + + For more details on tunnelled data migrations, report to + https://libvirt.org/migration.html#transporttunnel +- ''' +- cmd = _get_migrate_command() + ' --copy-storage-all ' + vm_\ +- + _get_target(target, ssh) ++ """ ++ cmd = ( ++ _get_migrate_command() + " --copy-storage-all " + vm_ + _get_target(target, ssh) ++ ) + +- stdout = subprocess.Popen(cmd, +- shell=True, +- stdout=subprocess.PIPE).communicate()[0] ++ stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] + return salt.utils.stringutils.to_str(stdout) + + + def migrate_non_shared_inc(vm_, target, ssh=False): +- ''' ++ """ + Attempt to execute non-shared storage "all" migration + + :param vm_: domain name +@@ -3145,18 +3721,17 @@ def migrate_non_shared_inc(vm_, target, ssh=False): + + For more details on tunnelled data migrations, report to + https://libvirt.org/migration.html#transporttunnel +- ''' +- cmd = _get_migrate_command() + ' --copy-storage-inc ' + vm_\ +- + _get_target(target, ssh) ++ """ ++ cmd = ( ++ _get_migrate_command() + " --copy-storage-inc " + vm_ + _get_target(target, ssh) ++ ) + +- stdout = subprocess.Popen(cmd, +- shell=True, +- stdout=subprocess.PIPE).communicate()[0] ++ stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] + return salt.utils.stringutils.to_str(stdout) + + + def migrate(vm_, target, ssh=False): +- ''' ++ """ + Shared storage migration + + :param vm_: domain name +@@ -3179,18 +3754,15 @@ def migrate(vm_, target, ssh=False): + + For more details on tunnelled data migrations, report to + https://libvirt.org/migration.html#transporttunnel +- ''' +- cmd = _get_migrate_command() + ' ' + vm_\ +- + _get_target(target, ssh) ++ """ ++ cmd = _get_migrate_command() + " " + vm_ + _get_target(target, ssh) + +- stdout = subprocess.Popen(cmd, +- shell=True, +- stdout=subprocess.PIPE).communicate()[0] ++ stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] + return salt.utils.stringutils.to_str(stdout) + + + def seed_non_shared_migrate(disks, force=False): +- ''' ++ """ + Non shared migration requires that the disks be present on the migration + destination, pass the disks information via this function, to the + migration destination before executing the migration. +@@ -3204,33 +3776,37 @@ def seed_non_shared_migrate(disks, force=False): + .. code-block:: bash + + salt '*' virt.seed_non_shared_migrate +- ''' ++ """ + for _, data in six.iteritems(disks): +- fn_ = data['file'] +- form = data['file format'] +- size = data['virtual size'].split()[1][1:] ++ fn_ = data["file"] ++ form = data["file format"] ++ size = data["virtual size"].split()[1][1:] + if os.path.isfile(fn_) and not force: + # the target exists, check to see if it is compatible +- pre = salt.utils.yaml.safe_load(subprocess.Popen('qemu-img info arch', +- shell=True, +- stdout=subprocess.PIPE).communicate()[0]) +- if pre['file format'] != data['file format']\ +- and pre['virtual size'] != data['virtual size']: ++ pre = salt.utils.yaml.safe_load( ++ subprocess.Popen( ++ "qemu-img info arch", shell=True, stdout=subprocess.PIPE ++ ).communicate()[0] ++ ) ++ if ( ++ pre["file format"] != data["file format"] ++ and pre["virtual size"] != data["virtual size"] ++ ): + return False + if not os.path.isdir(os.path.dirname(fn_)): + os.makedirs(os.path.dirname(fn_)) + if os.path.isfile(fn_): + os.remove(fn_) +- cmd = 'qemu-img create -f ' + form + ' ' + fn_ + ' ' + size ++ cmd = "qemu-img create -f " + form + " " + fn_ + " " + size + subprocess.call(cmd, shell=True) + creds = _libvirt_creds() +- cmd = 'chown ' + creds['user'] + ':' + creds['group'] + ' ' + fn_ ++ cmd = "chown " + creds["user"] + ":" + creds["group"] + " " + fn_ + subprocess.call(cmd, shell=True) + return True + + +-def set_autostart(vm_, state='on', **kwargs): +- ''' ++def set_autostart(vm_, state="on", **kwargs): ++ """ + Set the autostart flag on a VM so that the VM will start with the host + system on reboot. + +@@ -3252,17 +3828,17 @@ def set_autostart(vm_, state='on', **kwargs): + .. code-block:: bash + + salt "*" virt.set_autostart +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) + + # return False if state is set to something other then on or off + ret = False + +- if state == 'on': ++ if state == "on": + ret = dom.setAutostart(1) == 0 + +- elif state == 'off': ++ elif state == "off": + ret = dom.setAutostart(0) == 0 + + conn.close() +@@ -3270,7 +3846,7 @@ def set_autostart(vm_, state='on', **kwargs): + + + def undefine(vm_, **kwargs): +- ''' ++ """ + Remove a defined vm, this does not purge the virtual machine image, and + this only works if the vm is powered down + +@@ -3290,10 +3866,10 @@ def undefine(vm_, **kwargs): + .. code-block:: bash + + salt '*' virt.undefine +- ''' ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) +- if getattr(libvirt, 'VIR_DOMAIN_UNDEFINE_NVRAM', False): ++ if getattr(libvirt, "VIR_DOMAIN_UNDEFINE_NVRAM", False): + # This one is only in 1.2.8+ + ret = dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) == 0 + else: +@@ -3302,16 +3878,12 @@ def undefine(vm_, **kwargs): + return ret + + +-def purge(vm_, dirs=False, removables=None, **kwargs): +- ''' ++def purge(vm_, dirs=False, removables=False, **kwargs): ++ """ + Recursively destroy and delete a persistent virtual machine, pass True for + dir's to also delete the directories containing the virtual machine disk + images - USE WITH EXTREME CAUTION! + +- Pass removables=False to avoid deleting cdrom and floppy images. To avoid +- disruption, the default but dangerous value is True. This will be changed +- to the safer False default value in Sodium. +- + :param vm_: domain name + :param dirs: pass True to remove containing directories + :param removables: pass True to remove removable devices +@@ -3331,31 +3903,46 @@ def purge(vm_, dirs=False, removables=None, **kwargs): + + .. code-block:: bash + +- salt '*' virt.purge removables=False +- ''' ++ salt '*' virt.purge ++ """ + conn = __get_conn(**kwargs) + dom = _get_domain(conn, vm_) +- disks = _get_disks(dom) +- if removables is None: +- salt.utils.versions.warn_until( +- 'Sodium', +- 'removables argument default value is True, but will be changed ' +- 'to False by default in {version}. Please set to True to maintain ' +- 'the current behavior in the future.' +- ) +- removables = True +- if VIRT_STATE_NAME_MAP.get(dom.info()[0], 'unknown') != 'shutdown' and dom.destroy() != 0: ++ disks = _get_disks(conn, dom) ++ if ( ++ VIRT_STATE_NAME_MAP.get(dom.info()[0], "unknown") != "shutdown" ++ and dom.destroy() != 0 ++ ): + return False + directories = set() + for disk in disks: +- if not removables and disks[disk]['type'] in ['cdrom', 'floppy']: ++ if not removables and disks[disk]["type"] in ["cdrom", "floppy"]: + continue +- os.remove(disks[disk]['file']) +- directories.add(os.path.dirname(disks[disk]['file'])) ++ if disks[disk].get("zfs", False): ++ # TODO create solution for 'dataset is busy' ++ time.sleep(3) ++ fs_name = disks[disk]["file"][len("/dev/zvol/") :] ++ log.info("Destroying VM ZFS volume {0}".format(fs_name)) ++ __salt__["zfs.destroy"](name=fs_name, force=True) ++ elif os.path.exists(disks[disk]["file"]): ++ os.remove(disks[disk]["file"]) ++ directories.add(os.path.dirname(disks[disk]["file"])) ++ else: ++ # We may have a volume to delete here ++ matcher = re.match("^(?P[^/]+)/(?P.*)$", disks[disk]["file"],) ++ if matcher: ++ pool_name = matcher.group("pool") ++ pool = None ++ if pool_name in conn.listStoragePools(): ++ pool = conn.storagePoolLookupByName(pool_name) ++ ++ if pool and matcher.group("volume") in pool.listVolumes(): ++ volume = pool.storageVolLookupByName(matcher.group("volume")) ++ volume.delete() ++ + if dirs: + for dir_ in directories: + shutil.rmtree(dir_) +- if getattr(libvirt, 'VIR_DOMAIN_UNDEFINE_NVRAM', False): ++ if getattr(libvirt, "VIR_DOMAIN_UNDEFINE_NVRAM", False): + # This one is only in 1.2.8+ + try: + dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) +@@ -3368,7 +3955,7 @@ def purge(vm_, dirs=False, removables=None, **kwargs): + + + def virt_type(): +- ''' ++ """ + Returns the virtual machine type as a string + + CLI Example: +@@ -3376,92 +3963,53 @@ def virt_type(): + .. code-block:: bash + + salt '*' virt.virt_type +- ''' +- return __grains__['virtual'] ++ """ ++ return __grains__["virtual"] + + + def _is_kvm_hyper(): +- ''' ++ """ + Returns a bool whether or not this node is a KVM hypervisor +- ''' ++ """ + try: +- with salt.utils.files.fopen('/proc/modules') as fp_: +- if 'kvm_' not in salt.utils.stringutils.to_unicode(fp_.read()): ++ with salt.utils.files.fopen("/proc/modules") as fp_: ++ if "kvm_" not in salt.utils.stringutils.to_unicode(fp_.read()): + return False + except IOError: + # No /proc/modules? Are we on Windows? Or Solaris? + return False +- return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) +- +- +-def is_kvm_hyper(): +- ''' +- Returns a bool whether or not this node is a KVM hypervisor +- +- CLI Example: +- +- .. code-block:: bash +- +- salt '*' virt.is_kvm_hyper +- +- .. deprecated:: 2019.2.0 +- ''' +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'is_kvm_hyper\' function has been deprecated. Use the \'get_hypervisor\' == "kvm" instead. ' +- '\'is_kvm_hyper\' will be removed in {version}.' +- ) +- return _is_kvm_hyper() ++ return "libvirtd" in __salt__["cmd.run"](__grains__["ps"]) + + + def _is_xen_hyper(): +- ''' ++ """ + Returns a bool whether or not this node is a XEN hypervisor +- ''' ++ """ + try: +- if __grains__['virtual_subtype'] != 'Xen Dom0': ++ if __grains__["virtual_subtype"] != "Xen Dom0": + return False + except KeyError: + # virtual_subtype isn't set everywhere. + return False + try: +- with salt.utils.files.fopen('/proc/modules') as fp_: +- if 'xen_' not in salt.utils.stringutils.to_unicode(fp_.read()): ++ with salt.utils.files.fopen("/proc/modules") as fp_: ++ if "xen_" not in salt.utils.stringutils.to_unicode(fp_.read()): + return False + except (OSError, IOError): + # No /proc/modules? Are we on Windows? Or Solaris? + return False +- return 'libvirtd' in __salt__['cmd.run'](__grains__['ps']) +- +- +-def is_xen_hyper(): +- ''' +- Returns a bool whether or not this node is a XEN hypervisor +- +- CLI Example: +- +- .. code-block:: bash +- +- salt '*' virt.is_xen_hyper +- +- .. deprecated:: 2019.2.0 +- ''' +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'is_xen_hyper\' function has been deprecated. Use the \'get_hypervisor\' == "xen" instead. ' +- '\'is_xen_hyper\' will be removed in {version}.' +- ) +- return _is_xen_hyper() ++ return "libvirtd" in __salt__["cmd.run"](__grains__["ps"]) + + + def get_hypervisor(): +- ''' ++ """ + Returns the name of the hypervisor running on this node or ``None``. + + Detected hypervisors: + + - kvm + - xen ++ - bhyve + + CLI Example: + +@@ -3470,17 +4018,34 @@ def get_hypervisor(): + salt '*' virt.get_hypervisor + + .. versionadded:: 2019.2.0 +- the function and the ``kvm`` and ``xen`` hypervisors support +- ''' ++ the function and the ``kvm``, ``xen`` and ``bhyve`` hypervisors support ++ """ + # To add a new 'foo' hypervisor, add the _is_foo_hyper function, + # add 'foo' to the list below and add it to the docstring with a .. versionadded:: +- hypervisors = ['kvm', 'xen'] +- result = [hyper for hyper in hypervisors if getattr(sys.modules[__name__], '_is_{}_hyper'.format(hyper))()] ++ hypervisors = ["kvm", "xen", "bhyve"] ++ result = [ ++ hyper ++ for hyper in hypervisors ++ if getattr(sys.modules[__name__], "_is_{}_hyper".format(hyper))() ++ ] + return result[0] if result else None + + ++def _is_bhyve_hyper(): ++ sysctl_cmd = "sysctl hw.vmm.create" ++ vmm_enabled = False ++ try: ++ stdout = subprocess.Popen( ++ sysctl_cmd, shell=True, stdout=subprocess.PIPE ++ ).communicate()[0] ++ vmm_enabled = len(salt.utils.stringutils.to_str(stdout).split('"')[1]) != 0 ++ except IndexError: ++ pass ++ return vmm_enabled ++ ++ + def is_hyper(): +- ''' ++ """ + Returns a bool whether or not this node is a hypervisor of any kind + + CLI Example: +@@ -3488,14 +4053,14 @@ def is_hyper(): + .. code-block:: bash + + salt '*' virt.is_hyper +- ''' ++ """ + if HAS_LIBVIRT: +- return is_xen_hyper() or is_kvm_hyper() ++ return _is_xen_hyper() or _is_kvm_hyper() or _is_bhyve_hyper() + return False + + + def vm_cputime(vm_=None, **kwargs): +- ''' ++ """ + Return cputime used by the vms on this hyper in a + list of dicts: + +@@ -3528,14 +4093,14 @@ def vm_cputime(vm_=None, **kwargs): + .. code-block:: bash + + salt '*' virt.vm_cputime +- ''' ++ """ + conn = __get_conn(**kwargs) + host_cpus = conn.getInfo()[2] + + def _info(dom): +- ''' ++ """ + Compute cputime info of a domain +- ''' ++ """ + raw = dom.info() + vcpus = int(raw[3]) + cputime = int(raw[4]) +@@ -3544,9 +4109,10 @@ def vm_cputime(vm_=None, **kwargs): + # Divide by vcpus to always return a number between 0 and 100 + cputime_percent = (1.0e-7 * cputime / host_cpus) / vcpus + return { +- 'cputime': int(raw[4]), +- 'cputime_percent': int('{0:.0f}'.format(cputime_percent)) +- } ++ "cputime": int(raw[4]), ++ "cputime_percent": int("{0:.0f}".format(cputime_percent)), ++ } ++ + info = {} + if vm_: + info[vm_] = _info(_get_domain(conn, vm_)) +@@ -3558,7 +4124,7 @@ def vm_cputime(vm_=None, **kwargs): + + + def vm_netstats(vm_=None, **kwargs): +- ''' ++ """ + Return combined network counters used by the vms on this hyper in a + list of dicts: + +@@ -3597,36 +4163,38 @@ def vm_netstats(vm_=None, **kwargs): + .. code-block:: bash + + salt '*' virt.vm_netstats +- ''' ++ """ ++ + def _info(dom): +- ''' ++ """ + Compute network stats of a domain +- ''' ++ """ + nics = _get_nics(dom) + ret = { +- 'rx_bytes': 0, +- 'rx_packets': 0, +- 'rx_errs': 0, +- 'rx_drop': 0, +- 'tx_bytes': 0, +- 'tx_packets': 0, +- 'tx_errs': 0, +- 'tx_drop': 0 +- } ++ "rx_bytes": 0, ++ "rx_packets": 0, ++ "rx_errs": 0, ++ "rx_drop": 0, ++ "tx_bytes": 0, ++ "tx_packets": 0, ++ "tx_errs": 0, ++ "tx_drop": 0, ++ } + for attrs in six.itervalues(nics): +- if 'target' in attrs: +- dev = attrs['target'] ++ if "target" in attrs: ++ dev = attrs["target"] + stats = dom.interfaceStats(dev) +- ret['rx_bytes'] += stats[0] +- ret['rx_packets'] += stats[1] +- ret['rx_errs'] += stats[2] +- ret['rx_drop'] += stats[3] +- ret['tx_bytes'] += stats[4] +- ret['tx_packets'] += stats[5] +- ret['tx_errs'] += stats[6] +- ret['tx_drop'] += stats[7] ++ ret["rx_bytes"] += stats[0] ++ ret["rx_packets"] += stats[1] ++ ret["rx_errs"] += stats[2] ++ ret["rx_drop"] += stats[3] ++ ret["tx_bytes"] += stats[4] ++ ret["tx_packets"] += stats[5] ++ ret["tx_errs"] += stats[6] ++ ret["tx_drop"] += stats[7] + + return ret ++ + info = {} + conn = __get_conn(**kwargs) + if vm_: +@@ -3639,7 +4207,7 @@ def vm_netstats(vm_=None, **kwargs): + + + def vm_diskstats(vm_=None, **kwargs): +- ''' ++ """ + Return disk usage counters used by the vms on this hyper in a + list of dicts: + +@@ -3675,36 +4243,33 @@ def vm_diskstats(vm_=None, **kwargs): + .. code-block:: bash + + salt '*' virt.vm_blockstats +- ''' ++ """ ++ + def get_disk_devs(dom): +- ''' ++ """ + Extract the disk devices names from the domain XML definition +- ''' ++ """ + doc = ElementTree.fromstring(get_xml(dom, **kwargs)) +- return [target.get('dev') for target in doc.findall('devices/disk/target')] ++ return [target.get("dev") for target in doc.findall("devices/disk/target")] + + def _info(dom): +- ''' ++ """ + Compute the disk stats of a domain +- ''' ++ """ + # Do not use get_disks, since it uses qemu-img and is very slow + # and unsuitable for any sort of real time statistics + disks = get_disk_devs(dom) +- ret = {'rd_req': 0, +- 'rd_bytes': 0, +- 'wr_req': 0, +- 'wr_bytes': 0, +- 'errs': 0 +- } ++ ret = {"rd_req": 0, "rd_bytes": 0, "wr_req": 0, "wr_bytes": 0, "errs": 0} + for disk in disks: + stats = dom.blockStats(disk) +- ret['rd_req'] += stats[0] +- ret['rd_bytes'] += stats[1] +- ret['wr_req'] += stats[2] +- ret['wr_bytes'] += stats[3] +- ret['errs'] += stats[4] ++ ret["rd_req"] += stats[0] ++ ret["rd_bytes"] += stats[1] ++ ret["wr_req"] += stats[2] ++ ret["wr_bytes"] += stats[3] ++ ret["errs"] += stats[4] + + return ret ++ + info = {} + conn = __get_conn(**kwargs) + if vm_: +@@ -3718,30 +4283,33 @@ def vm_diskstats(vm_=None, **kwargs): + + + def _parse_snapshot_description(vm_snapshot, unix_time=False): +- ''' ++ """ + Parse XML doc and return a dict with the status values. + + :param xmldoc: + :return: +- ''' ++ """ + ret = dict() + tree = ElementTree.fromstring(vm_snapshot.getXMLDesc()) + for node in tree: +- if node.tag == 'name': +- ret['name'] = node.text +- elif node.tag == 'creationTime': +- ret['created'] = datetime.datetime.fromtimestamp(float(node.text)).isoformat(' ') \ +- if not unix_time else float(node.text) +- elif node.tag == 'state': +- ret['running'] = node.text == 'running' ++ if node.tag == "name": ++ ret["name"] = node.text ++ elif node.tag == "creationTime": ++ ret["created"] = ( ++ datetime.datetime.fromtimestamp(float(node.text)).isoformat(" ") ++ if not unix_time ++ else float(node.text) ++ ) ++ elif node.tag == "state": ++ ret["running"] = node.text == "running" + +- ret['current'] = vm_snapshot.isCurrent() == 1 ++ ret["current"] = vm_snapshot.isCurrent() == 1 + + return ret + + + def list_snapshots(domain=None, **kwargs): +- ''' ++ """ + List available snapshots for certain vm or for all. + + :param domain: domain name +@@ -3763,18 +4331,20 @@ def list_snapshots(domain=None, **kwargs): + + salt '*' virt.list_snapshots + salt '*' virt.list_snapshots +- ''' ++ """ + ret = dict() + conn = __get_conn(**kwargs) + for vm_domain in _get_domain(conn, *(domain and [domain] or list()), iterable=True): +- ret[vm_domain.name()] = [_parse_snapshot_description(snap) for snap in vm_domain.listAllSnapshots()] or 'N/A' ++ ret[vm_domain.name()] = [ ++ _parse_snapshot_description(snap) for snap in vm_domain.listAllSnapshots() ++ ] or "N/A" + + conn.close() + return ret + + + def snapshot(domain, name=None, suffix=None, **kwargs): +- ''' ++ """ + Create a snapshot of a VM. + + :param domain: domain name +@@ -3800,18 +4370,22 @@ def snapshot(domain, name=None, suffix=None, **kwargs): + .. code-block:: bash + + salt '*' virt.snapshot +- ''' ++ """ + if name and name.lower() == domain.lower(): +- raise CommandExecutionError('Virtual Machine {name} is already defined. ' +- 'Please choose another name for the snapshot'.format(name=name)) ++ raise CommandExecutionError( ++ "Virtual Machine {name} is already defined. " ++ "Please choose another name for the snapshot".format(name=name) ++ ) + if not name: +- name = "{domain}-{tsnap}".format(domain=domain, tsnap=time.strftime('%Y%m%d-%H%M%S', time.localtime())) ++ name = "{domain}-{tsnap}".format( ++ domain=domain, tsnap=time.strftime("%Y%m%d-%H%M%S", time.localtime()) ++ ) + + if suffix: + name = "{name}-{suffix}".format(name=name, suffix=suffix) + +- doc = ElementTree.Element('domainsnapshot') +- n_name = ElementTree.SubElement(doc, 'name') ++ doc = ElementTree.Element("domainsnapshot") ++ n_name = ElementTree.SubElement(doc, "name") + n_name.text = name + + conn = __get_conn(**kwargs) +@@ -3820,11 +4394,11 @@ def snapshot(domain, name=None, suffix=None, **kwargs): + ) + conn.close() + +- return {'name': name} ++ return {"name": name} + + + def delete_snapshots(name, *names, **kwargs): +- ''' ++ """ + Delete one or more snapshots of the given VM. + + :param name: domain name +@@ -3848,7 +4422,7 @@ def delete_snapshots(name, *names, **kwargs): + salt '*' virt.delete_snapshots all=True + salt '*' virt.delete_snapshots + salt '*' virt.delete_snapshots ... +- ''' ++ """ + deleted = dict() + conn = __get_conn(**kwargs) + domain = _get_domain(conn, name) +@@ -3858,13 +4432,16 @@ def delete_snapshots(name, *names, **kwargs): + snap.delete() + conn.close() + +- available = {name: [_parse_snapshot_description(snap) for snap in domain.listAllSnapshots()] or 'N/A'} ++ available = { ++ name: [_parse_snapshot_description(snap) for snap in domain.listAllSnapshots()] ++ or "N/A" ++ } + +- return {'available': available, 'deleted': deleted} ++ return {"available": available, "deleted": deleted} + + + def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): +- ''' ++ """ + Revert snapshot to the previous from current (if available) or to the specific. + + :param name: domain name +@@ -3888,7 +4465,7 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + + salt '*' virt.revert + salt '*' virt.revert +- ''' ++ """ + ret = dict() + conn = __get_conn(**kwargs) + domain = _get_domain(conn, name) +@@ -3896,22 +4473,32 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + + _snapshots = list() + for snap_obj in snapshots: +- _snapshots.append({'idx': _parse_snapshot_description(snap_obj, unix_time=True)['created'], 'ptr': snap_obj}) +- snapshots = [w_ptr['ptr'] for w_ptr in sorted(_snapshots, key=lambda item: item['idx'], reverse=True)] ++ _snapshots.append( ++ { ++ "idx": _parse_snapshot_description(snap_obj, unix_time=True)["created"], ++ "ptr": snap_obj, ++ } ++ ) ++ snapshots = [ ++ w_ptr["ptr"] ++ for w_ptr in sorted(_snapshots, key=lambda item: item["idx"], reverse=True) ++ ] + del _snapshots + + if not snapshots: + conn.close() +- raise CommandExecutionError('No snapshots found') ++ raise CommandExecutionError("No snapshots found") + elif len(snapshots) == 1: + conn.close() +- raise CommandExecutionError('Cannot revert to itself: only one snapshot is available.') ++ raise CommandExecutionError( ++ "Cannot revert to itself: only one snapshot is available." ++ ) + + snap = None + for p_snap in snapshots: + if not vm_snapshot: +- if p_snap.isCurrent() and snapshots[snapshots.index(p_snap) + 1:]: +- snap = snapshots[snapshots.index(p_snap) + 1:][0] ++ if p_snap.isCurrent() and snapshots[snapshots.index(p_snap) + 1 :]: ++ snap = snapshots[snapshots.index(p_snap) + 1 :][0] + break + elif p_snap.getName() == vm_snapshot: + snap = p_snap +@@ -3920,13 +4507,16 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + if not snap: + conn.close() + raise CommandExecutionError( +- snapshot and 'Snapshot "{0}" not found'.format(vm_snapshot) or 'No more previous snapshots available') ++ snapshot ++ and 'Snapshot "{0}" not found'.format(vm_snapshot) ++ or "No more previous snapshots available" ++ ) + elif snap.isCurrent(): + conn.close() +- raise CommandExecutionError('Cannot revert to the currently running snapshot.') ++ raise CommandExecutionError("Cannot revert to the currently running snapshot.") + + domain.revertToSnapshot(snap) +- ret['reverted'] = snap.getName() ++ ret["reverted"] = snap.getName() + + if cleanup: + delete = list() +@@ -3936,9 +4526,9 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + p_snap.delete() + else: + break +- ret['deleted'] = delete ++ ret["deleted"] = delete + else: +- ret['deleted'] = 'N/A' ++ ret["deleted"] = "N/A" + + conn.close() + +@@ -3946,12 +4536,12 @@ def revert_snapshot(name, vm_snapshot=None, cleanup=False, **kwargs): + + + def _caps_add_machine(machines, node): +- ''' ++ """ + Parse the element of the host capabilities and add it + to the machines list. +- ''' +- maxcpus = node.get('maxCpus') +- canonical = node.get('canonical') ++ """ ++ maxcpus = node.get("maxCpus") ++ canonical = node.get("canonical") + name = node.text + + alternate_name = "" +@@ -3961,202 +4551,237 @@ def _caps_add_machine(machines, node): + + machine = machines.get(name) + if not machine: +- machine = {'alternate_names': []} ++ machine = {"alternate_names": []} + if maxcpus: +- machine['maxcpus'] = int(maxcpus) ++ machine["maxcpus"] = int(maxcpus) + machines[name] = machine + if alternate_name: +- machine['alternate_names'].append(alternate_name) ++ machine["alternate_names"].append(alternate_name) + + + def _parse_caps_guest(guest): +- ''' ++ """ + Parse the element of the connection capabilities XML +- ''' +- arch_node = guest.find('arch') ++ """ ++ arch_node = guest.find("arch") + result = { +- 'os_type': guest.find('os_type').text, +- 'arch': { +- 'name': arch_node.get('name'), +- 'machines': {}, +- 'domains': {} +- }, ++ "os_type": guest.find("os_type").text, ++ "arch": {"name": arch_node.get("name"), "machines": {}, "domains": {}}, + } + + for child in arch_node: +- if child.tag == 'wordsize': +- result['arch']['wordsize'] = int(child.text) +- elif child.tag == 'emulator': +- result['arch']['emulator'] = child.text +- elif child.tag == 'machine': +- _caps_add_machine(result['arch']['machines'], child) +- elif child.tag == 'domain': +- domain_type = child.get('type') +- domain = { +- 'emulator': None, +- 'machines': {} +- } +- emulator_node = child.find('emulator') ++ if child.tag == "wordsize": ++ result["arch"]["wordsize"] = int(child.text) ++ elif child.tag == "emulator": ++ result["arch"]["emulator"] = child.text ++ elif child.tag == "machine": ++ _caps_add_machine(result["arch"]["machines"], child) ++ elif child.tag == "domain": ++ domain_type = child.get("type") ++ domain = {"emulator": None, "machines": {}} ++ emulator_node = child.find("emulator") + if emulator_node is not None: +- domain['emulator'] = emulator_node.text +- for machine in child.findall('machine'): +- _caps_add_machine(domain['machines'], machine) +- result['arch']['domains'][domain_type] = domain ++ domain["emulator"] = emulator_node.text ++ for machine in child.findall("machine"): ++ _caps_add_machine(domain["machines"], machine) ++ result["arch"]["domains"][domain_type] = domain + + # Note that some features have no default and toggle attributes. + # This may not be a perfect match, but represent them as enabled by default + # without possibility to toggle them. + # Some guests may also have no feature at all (xen pv for instance) +- features_nodes = guest.find('features') ++ features_nodes = guest.find("features") + if features_nodes is not None: +- result['features'] = {child.tag: {'toggle': True if child.get('toggle') == 'yes' else False, +- 'default': True if child.get('default') == 'no' else True} +- for child in features_nodes} ++ result["features"] = { ++ child.tag: { ++ "toggle": True if child.get("toggle") == "yes" else False, ++ "default": True if child.get("default") == "no" else True, ++ } ++ for child in features_nodes ++ } + + return result + + + def _parse_caps_cell(cell): +- ''' ++ """ + Parse the nodes of the connection capabilities XML output. +- ''' +- result = { +- 'id': int(cell.get('id')) +- } ++ """ ++ result = {"id": int(cell.get("id"))} + +- mem_node = cell.find('memory') ++ mem_node = cell.find("memory") + if mem_node is not None: +- unit = mem_node.get('unit', 'KiB') ++ unit = mem_node.get("unit", "KiB") + memory = mem_node.text +- result['memory'] = "{} {}".format(memory, unit) ++ result["memory"] = "{} {}".format(memory, unit) + +- pages = [{'size': "{} {}".format(page.get('size'), page.get('unit', 'KiB')), +- 'available': int(page.text)} +- for page in cell.findall('pages')] ++ pages = [ ++ { ++ "size": "{} {}".format(page.get("size"), page.get("unit", "KiB")), ++ "available": int(page.text), ++ } ++ for page in cell.findall("pages") ++ ] + if pages: +- result['pages'] = pages ++ result["pages"] = pages + +- distances = {int(distance.get('id')): int(distance.get('value')) +- for distance in cell.findall('distances/sibling')} ++ distances = { ++ int(distance.get("id")): int(distance.get("value")) ++ for distance in cell.findall("distances/sibling") ++ } + if distances: +- result['distances'] = distances ++ result["distances"] = distances + + cpus = [] +- for cpu_node in cell.findall('cpus/cpu'): +- cpu = { +- 'id': int(cpu_node.get('id')) +- } +- socket_id = cpu_node.get('socket_id') ++ for cpu_node in cell.findall("cpus/cpu"): ++ cpu = {"id": int(cpu_node.get("id"))} ++ socket_id = cpu_node.get("socket_id") + if socket_id: +- cpu['socket_id'] = int(socket_id) ++ cpu["socket_id"] = int(socket_id) + +- core_id = cpu_node.get('core_id') ++ core_id = cpu_node.get("core_id") + if core_id: +- cpu['core_id'] = int(core_id) +- siblings = cpu_node.get('siblings') ++ cpu["core_id"] = int(core_id) ++ siblings = cpu_node.get("siblings") + if siblings: +- cpu['siblings'] = siblings ++ cpu["siblings"] = siblings + cpus.append(cpu) + if cpus: +- result['cpus'] = cpus ++ result["cpus"] = cpus + + return result + + + def _parse_caps_bank(bank): +- ''' ++ """ + Parse the element of the connection capabilities XML. +- ''' ++ """ + result = { +- 'id': int(bank.get('id')), +- 'level': int(bank.get('level')), +- 'type': bank.get('type'), +- 'size': "{} {}".format(bank.get('size'), bank.get('unit')), +- 'cpus': bank.get('cpus') ++ "id": int(bank.get("id")), ++ "level": int(bank.get("level")), ++ "type": bank.get("type"), ++ "size": "{} {}".format(bank.get("size"), bank.get("unit")), ++ "cpus": bank.get("cpus"), + } + + controls = [] +- for control in bank.findall('control'): +- unit = control.get('unit') ++ for control in bank.findall("control"): ++ unit = control.get("unit") + result_control = { +- 'granularity': "{} {}".format(control.get('granularity'), unit), +- 'type': control.get('type'), +- 'maxAllocs': int(control.get('maxAllocs')) ++ "granularity": "{} {}".format(control.get("granularity"), unit), ++ "type": control.get("type"), ++ "maxAllocs": int(control.get("maxAllocs")), + } + +- minimum = control.get('min') ++ minimum = control.get("min") + if minimum: +- result_control['min'] = "{} {}".format(minimum, unit) ++ result_control["min"] = "{} {}".format(minimum, unit) + controls.append(result_control) + if controls: +- result['controls'] = controls ++ result["controls"] = controls + + return result + + + def _parse_caps_host(host): +- ''' ++ """ + Parse the element of the connection capabilities XML. +- ''' ++ """ + result = {} + for child in host: + +- if child.tag == 'uuid': +- result['uuid'] = child.text ++ if child.tag == "uuid": ++ result["uuid"] = child.text + +- elif child.tag == 'cpu': ++ elif child.tag == "cpu": + cpu = { +- 'arch': child.find('arch').text if child.find('arch') is not None else None, +- 'model': child.find('model').text if child.find('model') is not None else None, +- 'vendor': child.find('vendor').text if child.find('vendor') is not None else None, +- 'features': [feature.get('name') for feature in child.findall('feature')], +- 'pages': [{'size': '{} {}'.format(page.get('size'), page.get('unit', 'KiB'))} +- for page in child.findall('pages')] ++ "arch": child.find("arch").text ++ if child.find("arch") is not None ++ else None, ++ "model": child.find("model").text ++ if child.find("model") is not None ++ else None, ++ "vendor": child.find("vendor").text ++ if child.find("vendor") is not None ++ else None, ++ "features": [ ++ feature.get("name") for feature in child.findall("feature") ++ ], ++ "pages": [ ++ {"size": "{} {}".format(page.get("size"), page.get("unit", "KiB"))} ++ for page in child.findall("pages") ++ ], + } + # Parse the cpu tag +- microcode = child.find('microcode') ++ microcode = child.find("microcode") + if microcode is not None: +- cpu['microcode'] = microcode.get('version') ++ cpu["microcode"] = microcode.get("version") + +- topology = child.find('topology') ++ topology = child.find("topology") + if topology is not None: +- cpu['sockets'] = int(topology.get('sockets')) +- cpu['cores'] = int(topology.get('cores')) +- cpu['threads'] = int(topology.get('threads')) +- result['cpu'] = cpu ++ cpu["sockets"] = int(topology.get("sockets")) ++ cpu["cores"] = int(topology.get("cores")) ++ cpu["threads"] = int(topology.get("threads")) ++ result["cpu"] = cpu + + elif child.tag == "power_management": +- result['power_management'] = [node.tag for node in child] ++ result["power_management"] = [node.tag for node in child] + + elif child.tag == "migration_features": +- result['migration'] = { +- 'live': child.find('live') is not None, +- 'transports': [node.text for node in child.findall('uri_transports/uri_transport')] ++ result["migration"] = { ++ "live": child.find("live") is not None, ++ "transports": [ ++ node.text for node in child.findall("uri_transports/uri_transport") ++ ], + } + + elif child.tag == "topology": +- result['topology'] = { +- 'cells': [_parse_caps_cell(cell) for cell in child.findall('cells/cell')] ++ result["topology"] = { ++ "cells": [ ++ _parse_caps_cell(cell) for cell in child.findall("cells/cell") ++ ] + } + +- elif child.tag == 'cache': +- result['cache'] = { +- 'banks': [_parse_caps_bank(bank) for bank in child.findall('bank')] ++ elif child.tag == "cache": ++ result["cache"] = { ++ "banks": [_parse_caps_bank(bank) for bank in child.findall("bank")] + } + +- result['security'] = [{ +- 'model': secmodel.find('model').text if secmodel.find('model') is not None else None, +- 'doi': secmodel.find('doi').text if secmodel.find('doi') is not None else None, +- 'baselabels': [{'type': label.get('type'), 'label': label.text} +- for label in secmodel.findall('baselabel')] ++ result["security"] = [ ++ { ++ "model": secmodel.find("model").text ++ if secmodel.find("model") is not None ++ else None, ++ "doi": secmodel.find("doi").text ++ if secmodel.find("doi") is not None ++ else None, ++ "baselabels": [ ++ {"type": label.get("type"), "label": label.text} ++ for label in secmodel.findall("baselabel") ++ ], + } +- for secmodel in host.findall('secmodel')] ++ for secmodel in host.findall("secmodel") ++ ] + + return result + + ++def _capabilities(conn): ++ """ ++ Return the hypervisor connection capabilities. ++ ++ :param conn: opened libvirt connection to use ++ """ ++ caps = ElementTree.fromstring(conn.getCapabilities()) ++ ++ return { ++ "host": _parse_caps_host(caps.find("host")), ++ "guests": [_parse_caps_guest(guest) for guest in caps.findall("guest")], ++ } ++ ++ + def capabilities(**kwargs): +- ''' ++ """ + Return the hypervisor connection capabilities. + + :param connection: libvirt connection URI, overriding defaults +@@ -4170,149 +4795,156 @@ def capabilities(**kwargs): + .. code-block:: bash + + salt '*' virt.capabilities +- ''' ++ """ + conn = __get_conn(**kwargs) +- caps = ElementTree.fromstring(conn.getCapabilities()) +- conn.close() +- +- return { +- 'host': _parse_caps_host(caps.find('host')), +- 'guests': [_parse_caps_guest(guest) for guest in caps.findall('guest')] +- } ++ try: ++ caps = _capabilities(conn) ++ except libvirt.libvirtError as err: ++ raise CommandExecutionError(str(err)) ++ finally: ++ conn.close() ++ return caps + + + def _parse_caps_enum(node): +- ''' ++ """ + Return a tuple containing the name of the enum and the possible values +- ''' +- return (node.get('name'), [value.text for value in node.findall('value')]) ++ """ ++ return (node.get("name"), [value.text for value in node.findall("value")]) + + + def _parse_caps_cpu(node): +- ''' ++ """ + Parse the element of the domain capabilities +- ''' ++ """ + result = {} +- for mode in node.findall('mode'): +- if not mode.get('supported') == 'yes': ++ for mode in node.findall("mode"): ++ if not mode.get("supported") == "yes": + continue + +- name = mode.get('name') +- if name == 'host-passthrough': ++ name = mode.get("name") ++ if name == "host-passthrough": + result[name] = True + +- elif name == 'host-model': ++ elif name == "host-model": + host_model = {} +- model_node = mode.find('model') ++ model_node = mode.find("model") + if model_node is not None: +- model = { +- 'name': model_node.text +- } ++ model = {"name": model_node.text} + +- vendor_id = model_node.get('vendor_id') ++ vendor_id = model_node.get("vendor_id") + if vendor_id: +- model['vendor_id'] = vendor_id ++ model["vendor_id"] = vendor_id + +- fallback = model_node.get('fallback') ++ fallback = model_node.get("fallback") + if fallback: +- model['fallback'] = fallback +- host_model['model'] = model ++ model["fallback"] = fallback ++ host_model["model"] = model + +- vendor = mode.find('vendor').text if mode.find('vendor') is not None else None ++ vendor = ( ++ mode.find("vendor").text if mode.find("vendor") is not None else None ++ ) + if vendor: +- host_model['vendor'] = vendor ++ host_model["vendor"] = vendor + +- features = {feature.get('name'): feature.get('policy') for feature in mode.findall('feature')} ++ features = { ++ feature.get("name"): feature.get("policy") ++ for feature in mode.findall("feature") ++ } + if features: +- host_model['features'] = features ++ host_model["features"] = features + + result[name] = host_model + +- elif name == 'custom': ++ elif name == "custom": + custom_model = {} +- models = {model.text: model.get('usable') for model in mode.findall('model')} ++ models = { ++ model.text: model.get("usable") for model in mode.findall("model") ++ } + if models: +- custom_model['models'] = models ++ custom_model["models"] = models + result[name] = custom_model + + return result + + + def _parse_caps_devices_features(node): +- ''' ++ """ + Parse the devices or features list of the domain capatilities +- ''' ++ """ + result = {} + for child in node: +- if child.get('supported') == 'yes': +- enums = [_parse_caps_enum(node) for node in child.findall('enum')] ++ if child.get("supported") == "yes": ++ enums = [_parse_caps_enum(node) for node in child.findall("enum")] + result[child.tag] = {item[0]: item[1] for item in enums if item[0]} + return result + + + def _parse_caps_loader(node): +- ''' ++ """ + Parse the element of the domain capabilities. +- ''' +- enums = [_parse_caps_enum(enum) for enum in node.findall('enum')] ++ """ ++ enums = [_parse_caps_enum(enum) for enum in node.findall("enum")] + result = {item[0]: item[1] for item in enums if item[0]} + +- values = [child.text for child in node.findall('value')] ++ values = [child.text for child in node.findall("value")] + + if values: +- result['values'] = values ++ result["values"] = values + + return result + + + def _parse_domain_caps(caps): +- ''' ++ """ + Parse the XML document of domain capabilities into a structure. +- ''' ++ """ + result = { +- 'emulator': caps.find('path').text if caps.find('path') is not None else None, +- 'domain': caps.find('domain').text if caps.find('domain') is not None else None, +- 'machine': caps.find('machine').text if caps.find('machine') is not None else None, +- 'arch': caps.find('arch').text if caps.find('arch') is not None else None ++ "emulator": caps.find("path").text if caps.find("path") is not None else None, ++ "domain": caps.find("domain").text if caps.find("domain") is not None else None, ++ "machine": caps.find("machine").text ++ if caps.find("machine") is not None ++ else None, ++ "arch": caps.find("arch").text if caps.find("arch") is not None else None, + } + + for child in caps: +- if child.tag == 'vcpu' and child.get('max'): +- result['max_vcpus'] = int(child.get('max')) ++ if child.tag == "vcpu" and child.get("max"): ++ result["max_vcpus"] = int(child.get("max")) + +- elif child.tag == 'iothreads': +- result['iothreads'] = child.get('supported') == 'yes' ++ elif child.tag == "iothreads": ++ result["iothreads"] = child.get("supported") == "yes" + +- elif child.tag == 'os': +- result['os'] = {} +- loader_node = child.find('loader') +- if loader_node is not None and loader_node.get('supported') == 'yes': ++ elif child.tag == "os": ++ result["os"] = {} ++ loader_node = child.find("loader") ++ if loader_node is not None and loader_node.get("supported") == "yes": + loader = _parse_caps_loader(loader_node) +- result['os']['loader'] = loader ++ result["os"]["loader"] = loader + +- elif child.tag == 'cpu': ++ elif child.tag == "cpu": + cpu = _parse_caps_cpu(child) + if cpu: +- result['cpu'] = cpu ++ result["cpu"] = cpu + +- elif child.tag == 'devices': ++ elif child.tag == "devices": + devices = _parse_caps_devices_features(child) + if devices: +- result['devices'] = devices ++ result["devices"] = devices + +- elif child.tag == 'features': ++ elif child.tag == "features": + features = _parse_caps_devices_features(child) + if features: +- result['features'] = features ++ result["features"] = features + + return result + + + def domain_capabilities(emulator=None, arch=None, machine=None, domain=None, **kwargs): +- ''' ++ """ + Return the domain capabilities given an emulator, architecture, machine or virtualization type. + +- .. versionadded:: Fluorine ++ .. versionadded:: 2019.2.0 + + :param emulator: return the capabilities for the given emulator binary + :param arch: return the capabilities for the given CPU architecture +@@ -4333,11 +4965,13 @@ def domain_capabilities(emulator=None, arch=None, machine=None, domain=None, **k + + salt '*' virt.domain_capabilities arch='x86_64' domain='kvm' + +- ''' ++ """ + conn = __get_conn(**kwargs) + result = [] + try: +- caps = ElementTree.fromstring(conn.getDomainCapabilities(emulator, arch, machine, domain, 0)) ++ caps = ElementTree.fromstring( ++ conn.getDomainCapabilities(emulator, arch, machine, domain, 0) ++ ) + result = _parse_domain_caps(caps) + finally: + conn.close() +@@ -4346,10 +4980,10 @@ def domain_capabilities(emulator=None, arch=None, machine=None, domain=None, **k + + + def all_capabilities(**kwargs): +- ''' ++ """ + Return the host and domain capabilities in a single call. + +- .. versionadded:: Neon ++ .. versionadded:: Sodium + + :param connection: libvirt connection URI, overriding defaults + :param username: username to connect with, overriding defaults +@@ -4361,31 +4995,45 @@ def all_capabilities(**kwargs): + + salt '*' virt.all_capabilities + +- ''' ++ """ + conn = __get_conn(**kwargs) + result = {} + try: + host_caps = ElementTree.fromstring(conn.getCapabilities()) +- domains = [[(guest.get('arch', {}).get('name', None), key) +- for key in guest.get('arch', {}).get('domains', {}).keys()] +- for guest in [_parse_caps_guest(guest) for guest in host_caps.findall('guest')]] ++ domains = [ ++ [ ++ (guest.get("arch", {}).get("name", None), key) ++ for key in guest.get("arch", {}).get("domains", {}).keys() ++ ] ++ for guest in [ ++ _parse_caps_guest(guest) for guest in host_caps.findall("guest") ++ ] ++ ] + flattened = [pair for item in (x for x in domains) for pair in item] + result = { +- 'host': { +- 'host': _parse_caps_host(host_caps.find('host')), +- 'guests': [_parse_caps_guest(guest) for guest in host_caps.findall('guest')] +- }, +- 'domains': [_parse_domain_caps(ElementTree.fromstring( +- conn.getDomainCapabilities(None, arch, None, domain))) +- for (arch, domain) in flattened]} ++ "host": { ++ "host": _parse_caps_host(host_caps.find("host")), ++ "guests": [ ++ _parse_caps_guest(guest) for guest in host_caps.findall("guest") ++ ], ++ }, ++ "domains": [ ++ _parse_domain_caps( ++ ElementTree.fromstring( ++ conn.getDomainCapabilities(None, arch, None, domain) ++ ) ++ ) ++ for (arch, domain) in flattened ++ ], ++ } + finally: + conn.close() + + return result + + +-def cpu_baseline(full=False, migratable=False, out='libvirt', **kwargs): +- ''' ++def cpu_baseline(full=False, migratable=False, out="libvirt", **kwargs): ++ """ + Return the optimal 'custom' CPU baseline config for VM's on this minion + + .. versionadded:: 2016.3.0 +@@ -4409,72 +5057,80 @@ def cpu_baseline(full=False, migratable=False, out='libvirt', **kwargs): + + salt '*' virt.cpu_baseline + +- ''' ++ """ + conn = __get_conn(**kwargs) + caps = ElementTree.fromstring(conn.getCapabilities()) +- cpu = caps.find('host/cpu') +- log.debug('Host CPU model definition: %s', salt.utils.stringutils.to_str(ElementTree.tostring(cpu))) ++ cpu = caps.find("host/cpu") ++ log.debug( ++ "Host CPU model definition: %s", ++ salt.utils.stringutils.to_str(ElementTree.tostring(cpu)), ++ ) + + flags = 0 + if migratable: + # This one is only in 1.2.14+ +- if getattr(libvirt, 'VIR_CONNECT_BASELINE_CPU_MIGRATABLE', False): ++ if getattr(libvirt, "VIR_CONNECT_BASELINE_CPU_MIGRATABLE", False): + flags += libvirt.VIR_CONNECT_BASELINE_CPU_MIGRATABLE + else: + conn.close() + raise ValueError + +- if full and getattr(libvirt, 'VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES', False): ++ if full and getattr(libvirt, "VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES", False): + # This one is only in 1.1.3+ + flags += libvirt.VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES + +- cpu = ElementTree.fromstring(conn.baselineCPU([salt.utils.stringutils.to_str(ElementTree.tostring(cpu))], flags)) ++ cpu = ElementTree.fromstring( ++ conn.baselineCPU( ++ [salt.utils.stringutils.to_str(ElementTree.tostring(cpu))], flags ++ ) ++ ) + conn.close() + +- if full and not getattr(libvirt, 'VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES', False): ++ if full and not getattr(libvirt, "VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES", False): + # Try do it by ourselves + # Find the models in cpu_map.xml and iterate over them for as long as entries have submodels +- with salt.utils.files.fopen('/usr/share/libvirt/cpu_map.xml', 'r') as cpu_map: ++ with salt.utils.files.fopen("/usr/share/libvirt/cpu_map.xml", "r") as cpu_map: + cpu_map = ElementTree.parse(cpu_map) + +- cpu_model = cpu.find('model').text ++ cpu_model = cpu.find("model").text + while cpu_model: +- cpu_map_models = cpu_map.findall('arch/model') +- cpu_specs = [el for el in cpu_map_models if el.get('name') == cpu_model and bool(len(el))] ++ cpu_map_models = cpu_map.findall("arch/model") ++ cpu_specs = [ ++ el ++ for el in cpu_map_models ++ if el.get("name") == cpu_model and bool(len(el)) ++ ] + + if not cpu_specs: +- raise ValueError('Model {0} not found in CPU map'.format(cpu_model)) ++ raise ValueError("Model {0} not found in CPU map".format(cpu_model)) + elif len(cpu_specs) > 1: +- raise ValueError('Multiple models {0} found in CPU map'.format(cpu_model)) ++ raise ValueError( ++ "Multiple models {0} found in CPU map".format(cpu_model) ++ ) + + cpu_specs = cpu_specs[0] + + # libvirt's cpu map used to nest model elements, to point the parent model. + # keep this code for compatibility with old libvirt versions +- model_node = cpu_specs.find('model') ++ model_node = cpu_specs.find("model") + if model_node is None: + cpu_model = None + else: +- cpu_model = model_node.get('name') ++ cpu_model = model_node.get("name") + +- cpu.extend([feature for feature in cpu_specs.findall('feature')]) ++ cpu.extend([feature for feature in cpu_specs.findall("feature")]) + +- if out == 'salt': ++ if out == "salt": + return { +- 'model': cpu.find('model').text, +- 'vendor': cpu.find('vendor').text, +- 'features': [feature.get('name') for feature in cpu.findall('feature')] ++ "model": cpu.find("model").text, ++ "vendor": cpu.find("vendor").text, ++ "features": [feature.get("name") for feature in cpu.findall("feature")], + } + return cpu.toxml() + + +-def network_define(name, +- bridge, +- forward, +- ipv4_config=None, +- ipv6_config=None, +- **kwargs): +- ''' ++def network_define(name, bridge, forward, ipv4_config=None, ipv6_config=None, **kwargs): ++ """ + Create libvirt network. + + :param name: Network name +@@ -4523,12 +5179,12 @@ def network_define(name, + salt '*' virt.network_define network main bridge openvswitch + + .. versionadded:: 2019.2.0 +- ''' ++ """ + conn = __get_conn(**kwargs) +- vport = kwargs.get('vport', None) +- tag = kwargs.get('tag', None) +- autostart = kwargs.get('autostart', True) +- starting = kwargs.get('start', True) ++ vport = kwargs.get("vport", None) ++ tag = kwargs.get("tag", None) ++ autostart = kwargs.get("autostart", True) ++ starting = kwargs.get("start", True) + + net_xml = _gen_net_xml( + name, +@@ -4540,14 +5196,14 @@ def network_define(name, + ) + try: + conn.networkDefineXML(net_xml) +- except libvirtError as err: ++ except libvirt.libvirtError as err: + log.warning(err) + conn.close() + raise err # a real error we should report upwards + + try: + network = conn.networkLookupByName(name) +- except libvirtError as err: ++ except libvirt.libvirtError as err: + log.warning(err) + conn.close() + raise err # a real error we should report upwards +@@ -4570,7 +5226,7 @@ def network_define(name, + + + def list_networks(**kwargs): +- ''' ++ """ + List all virtual networks. + + :param connection: libvirt connection URI, overriding defaults +@@ -4584,7 +5240,7 @@ def list_networks(**kwargs): + .. code-block:: bash + + salt '*' virt.list_networks +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + return [net.name() for net in conn.listAllNetworks()] +@@ -4593,7 +5249,7 @@ def list_networks(**kwargs): + + + def network_info(name=None, **kwargs): +- ''' ++ """ + Return informations on a virtual network provided its name. + + :param name: virtual network name +@@ -4610,42 +5266,48 @@ def network_info(name=None, **kwargs): + .. code-block:: bash + + salt '*' virt.network_info default +- ''' ++ """ + result = {} + conn = __get_conn(**kwargs) + + def _net_get_leases(net): +- ''' ++ """ + Get all DHCP leases for a network +- ''' ++ """ + leases = net.DHCPLeases() + for lease in leases: +- if lease['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV4: +- lease['type'] = 'ipv4' +- elif lease['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV6: +- lease['type'] = 'ipv6' ++ if lease["type"] == libvirt.VIR_IP_ADDR_TYPE_IPV4: ++ lease["type"] = "ipv4" ++ elif lease["type"] == libvirt.VIR_IP_ADDR_TYPE_IPV6: ++ lease["type"] = "ipv6" + else: +- lease['type'] = 'unknown' ++ lease["type"] = "unknown" + return leases + + try: +- nets = [net for net in conn.listAllNetworks() if name is None or net.name() == name] +- result = {net.name(): { +- 'uuid': net.UUIDString(), +- 'bridge': net.bridgeName(), +- 'autostart': net.autostart(), +- 'active': net.isActive(), +- 'persistent': net.isPersistent(), +- 'leases': _net_get_leases(net)} for net in nets} ++ nets = [ ++ net for net in conn.listAllNetworks() if name is None or net.name() == name ++ ] ++ result = { ++ net.name(): { ++ "uuid": net.UUIDString(), ++ "bridge": net.bridgeName(), ++ "autostart": net.autostart(), ++ "active": net.isActive(), ++ "persistent": net.isPersistent(), ++ "leases": _net_get_leases(net), ++ } ++ for net in nets ++ } + except libvirt.libvirtError as err: +- log.debug('Silenced libvirt error: %s', str(err)) ++ log.debug("Silenced libvirt error: %s", str(err)) + finally: + conn.close() + return result + + + def network_get_xml(name, **kwargs): +- ''' ++ """ + Return the XML definition of a virtual network + + :param name: libvirt network name +@@ -4660,7 +5322,7 @@ def network_get_xml(name, **kwargs): + .. code-block:: bash + + salt '*' virt.network_get_xml default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + return conn.networkLookupByName(name).XMLDesc() +@@ -4669,7 +5331,7 @@ def network_get_xml(name, **kwargs): + + + def network_start(name, **kwargs): +- ''' ++ """ + Start a defined virtual network. + + :param name: virtual network name +@@ -4684,7 +5346,7 @@ def network_start(name, **kwargs): + .. code-block:: bash + + salt '*' virt.network_start default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + net = conn.networkLookupByName(name) +@@ -4694,7 +5356,7 @@ def network_start(name, **kwargs): + + + def network_stop(name, **kwargs): +- ''' ++ """ + Stop a defined virtual network. + + :param name: virtual network name +@@ -4709,7 +5371,7 @@ def network_stop(name, **kwargs): + .. code-block:: bash + + salt '*' virt.network_stop default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + net = conn.networkLookupByName(name) +@@ -4719,7 +5381,7 @@ def network_stop(name, **kwargs): + + + def network_undefine(name, **kwargs): +- ''' ++ """ + Remove a defined virtual network. This does not stop the virtual network. + + :param name: virtual network name +@@ -4734,7 +5396,7 @@ def network_undefine(name, **kwargs): + .. code-block:: bash + + salt '*' virt.network_undefine default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + net = conn.networkLookupByName(name) +@@ -4743,8 +5405,8 @@ def network_undefine(name, **kwargs): + conn.close() + + +-def network_set_autostart(name, state='on', **kwargs): +- ''' ++def network_set_autostart(name, state="on", **kwargs): ++ """ + Set the autostart flag on a virtual network so that the network + will start with the host system on reboot. + +@@ -4762,45 +5424,216 @@ def network_set_autostart(name, state='on', **kwargs): + .. code-block:: bash + + salt "*" virt.network_set_autostart +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + net = conn.networkLookupByName(name) +- return not bool(net.setAutostart(1 if state == 'on' else 0)) ++ return not bool(net.setAutostart(1 if state == "on" else 0)) + finally: + conn.close() + + + def _parse_pools_caps(doc): +- ''' ++ """ + Parse libvirt pool capabilities XML +- ''' ++ """ ++ + def _parse_pool_caps(pool): + pool_caps = { +- 'name': pool.get('type'), +- 'supported': pool.get('supported', 'no') == 'yes' ++ "name": pool.get("type"), ++ "supported": pool.get("supported", "no") == "yes", + } +- for option_kind in ['pool', 'vol']: ++ for option_kind in ["pool", "vol"]: + options = {} +- default_format_node = pool.find('{0}Options/defaultFormat'.format(option_kind)) ++ default_format_node = pool.find( ++ "{0}Options/defaultFormat".format(option_kind) ++ ) + if default_format_node is not None: +- options['default_format'] = default_format_node.get('type') +- options_enums = {enum.get('name'): [value.text for value in enum.findall('value')] +- for enum in pool.findall('{0}Options/enum'.format(option_kind))} ++ options["default_format"] = default_format_node.get("type") ++ options_enums = { ++ enum.get("name"): [value.text for value in enum.findall("value")] ++ for enum in pool.findall("{0}Options/enum".format(option_kind)) ++ } + if options_enums: + options.update(options_enums) + if options: +- if 'options' not in pool_caps: +- pool_caps['options'] = {} +- kind = option_kind if option_kind is not 'vol' else 'volume' +- pool_caps['options'][kind] = options ++ if "options" not in pool_caps: ++ pool_caps["options"] = {} ++ kind = option_kind if option_kind is not "vol" else "volume" ++ pool_caps["options"][kind] = options + return pool_caps + +- return [_parse_pool_caps(pool) for pool in doc.findall('pool')] ++ return [_parse_pool_caps(pool) for pool in doc.findall("pool")] ++ ++ ++def _pool_capabilities(conn): ++ """ ++ Return the hypervisor connection storage pool capabilities. ++ ++ :param conn: opened libvirt connection to use ++ """ ++ has_pool_capabilities = bool(getattr(conn, "getStoragePoolCapabilities", None)) ++ if has_pool_capabilities: ++ caps = ElementTree.fromstring(conn.getStoragePoolCapabilities()) ++ pool_types = _parse_pools_caps(caps) ++ else: ++ # Compute reasonable values ++ all_hypervisors = ["xen", "kvm", "bhyve"] ++ images_formats = [ ++ "none", ++ "raw", ++ "dir", ++ "bochs", ++ "cloop", ++ "dmg", ++ "iso", ++ "vpc", ++ "vdi", ++ "fat", ++ "vhd", ++ "ploop", ++ "cow", ++ "qcow", ++ "qcow2", ++ "qed", ++ "vmdk", ++ ] ++ common_drivers = [ ++ { ++ "name": "fs", ++ "default_source_format": "auto", ++ "source_formats": [ ++ "auto", ++ "ext2", ++ "ext3", ++ "ext4", ++ "ufs", ++ "iso9660", ++ "udf", ++ "gfs", ++ "gfs2", ++ "vfat", ++ "hfs+", ++ "xfs", ++ "ocfs2", ++ ], ++ "default_target_format": "raw", ++ "target_formats": images_formats, ++ }, ++ { ++ "name": "dir", ++ "default_target_format": "raw", ++ "target_formats": images_formats, ++ }, ++ {"name": "iscsi"}, ++ {"name": "scsi"}, ++ { ++ "name": "logical", ++ "default_source_format": "lvm2", ++ "source_formats": ["unknown", "lvm2"], ++ }, ++ { ++ "name": "netfs", ++ "default_source_format": "auto", ++ "source_formats": ["auto", "nfs", "glusterfs", "cifs"], ++ "default_target_format": "raw", ++ "target_formats": images_formats, ++ }, ++ { ++ "name": "disk", ++ "default_source_format": "unknown", ++ "source_formats": [ ++ "unknown", ++ "dos", ++ "dvh", ++ "gpt", ++ "mac", ++ "bsd", ++ "pc98", ++ "sun", ++ "lvm2", ++ ], ++ "default_target_format": "none", ++ "target_formats": [ ++ "none", ++ "linux", ++ "fat16", ++ "fat32", ++ "linux-swap", ++ "linux-lvm", ++ "linux-raid", ++ "extended", ++ ], ++ }, ++ {"name": "mpath"}, ++ {"name": "rbd", "default_target_format": "raw", "target_formats": []}, ++ { ++ "name": "sheepdog", ++ "version": 10000, ++ "hypervisors": ["kvm"], ++ "default_target_format": "raw", ++ "target_formats": images_formats, ++ }, ++ { ++ "name": "gluster", ++ "version": 1002000, ++ "hypervisors": ["kvm"], ++ "default_target_format": "raw", ++ "target_formats": images_formats, ++ }, ++ {"name": "zfs", "version": 1002008, "hypervisors": ["bhyve"]}, ++ { ++ "name": "iscsi-direct", ++ "version": 4007000, ++ "hypervisors": ["kvm", "xen"], ++ }, ++ ] ++ ++ libvirt_version = conn.getLibVersion() ++ hypervisor = get_hypervisor() ++ ++ def _get_backend_output(backend): ++ output = { ++ "name": backend["name"], ++ "supported": ( ++ not backend.get("version") or libvirt_version >= backend["version"] ++ ) ++ and hypervisor in backend.get("hypervisors", all_hypervisors), ++ "options": { ++ "pool": { ++ "default_format": backend.get("default_source_format"), ++ "sourceFormatType": backend.get("source_formats"), ++ }, ++ "volume": { ++ "default_format": backend.get("default_target_format"), ++ "targetFormatType": backend.get("target_formats"), ++ }, ++ }, ++ } ++ ++ # Cleanup the empty members to match the libvirt output ++ for option_kind in ["pool", "volume"]: ++ if not [ ++ value ++ for value in output["options"][option_kind].values() ++ if value is not None ++ ]: ++ del output["options"][option_kind] ++ if not output["options"]: ++ del output["options"] ++ ++ return output ++ ++ pool_types = [_get_backend_output(backend) for backend in common_drivers] ++ ++ return { ++ "computed": not has_pool_capabilities, ++ "pool_types": pool_types, ++ } + + + def pool_capabilities(**kwargs): +- ''' ++ """ + Return the hypervisor connection storage pool capabilities. + + The returned data are either directly extracted from libvirt or computed. +@@ -4819,132 +5652,32 @@ def pool_capabilities(**kwargs): + + salt '*' virt.pool_capabilities + +- ''' ++ """ + try: + conn = __get_conn(**kwargs) +- has_pool_capabilities = bool(getattr(conn, 'getStoragePoolCapabilities', None)) +- if has_pool_capabilities: +- caps = ElementTree.fromstring(conn.getStoragePoolCapabilities()) +- pool_types = _parse_pools_caps(caps) +- else: +- # Compute reasonable values +- all_hypervisors = ['xen', 'kvm', 'bhyve'] +- images_formats = ['none', 'raw', 'dir', 'bochs', 'cloop', 'dmg', 'iso', 'vpc', 'vdi', +- 'fat', 'vhd', 'ploop', 'cow', 'qcow', 'qcow2', 'qed', 'vmdk'] +- common_drivers = [ +- { +- 'name': 'fs', +- 'default_source_format': 'auto', +- 'source_formats': ['auto', 'ext2', 'ext3', 'ext4', 'ufs', 'iso9660', 'udf', 'gfs', 'gfs2', +- 'vfat', 'hfs+', 'xfs', 'ocfs2'], +- 'default_target_format': 'raw', +- 'target_formats': images_formats +- }, +- { +- 'name': 'dir', +- 'default_target_format': 'raw', +- 'target_formats': images_formats +- }, +- {'name': 'iscsi'}, +- {'name': 'scsi'}, +- { +- 'name': 'logical', +- 'default_source_format': 'lvm2', +- 'source_formats': ['unknown', 'lvm2'], +- }, +- { +- 'name': 'netfs', +- 'default_source_format': 'auto', +- 'source_formats': ['auto', 'nfs', 'glusterfs', 'cifs'], +- 'default_target_format': 'raw', +- 'target_formats': images_formats +- }, +- { +- 'name': 'disk', +- 'default_source_format': 'unknown', +- 'source_formats': ['unknown', 'dos', 'dvh', 'gpt', 'mac', 'bsd', 'pc98', 'sun', 'lvm2'], +- 'default_target_format': 'none', +- 'target_formats': ['none', 'linux', 'fat16', 'fat32', 'linux-swap', 'linux-lvm', +- 'linux-raid', 'extended'] +- }, +- {'name': 'mpath'}, +- { +- 'name': 'rbd', +- 'default_target_format': 'raw', +- 'target_formats': [] +- }, +- { +- 'name': 'sheepdog', +- 'version': 10000, +- 'hypervisors': ['kvm'], +- 'default_target_format': 'raw', +- 'target_formats': images_formats +- }, +- { +- 'name': 'gluster', +- 'version': 1002000, +- 'hypervisors': ['kvm'], +- 'default_target_format': 'raw', +- 'target_formats': images_formats +- }, +- {'name': 'zfs', 'version': 1002008, 'hypervisors': ['bhyve']}, +- {'name': 'iscsi-direct', 'version': 4007000, 'hypervisors': ['kvm', 'xen']} +- ] +- +- libvirt_version = conn.getLibVersion() +- hypervisor = get_hypervisor() +- +- def _get_backend_output(backend): +- output = { +- 'name': backend['name'], +- 'supported': (not backend.get('version') or libvirt_version >= backend['version']) and +- hypervisor in backend.get('hypervisors', all_hypervisors), +- 'options': { +- 'pool': { +- 'default_format': backend.get('default_source_format'), +- 'sourceFormatType': backend.get('source_formats') +- }, +- 'volume': { +- 'default_format': backend.get('default_target_format'), +- 'targetFormatType': backend.get('target_formats') +- } +- } +- } +- +- # Cleanup the empty members to match the libvirt output +- for option_kind in ['pool', 'volume']: +- if not [value for value in output['options'][option_kind].values() if value is not None]: +- del output['options'][option_kind] +- if not output['options']: +- del output['options'] +- +- return output +- pool_types = [_get_backend_output(backend) for backend in common_drivers] ++ return _pool_capabilities(conn) + finally: + conn.close() + +- return { +- 'computed': not has_pool_capabilities, +- 'pool_types': pool_types, +- } +- + +-def pool_define(name, +- ptype, +- target=None, +- permissions=None, +- source_devices=None, +- source_dir=None, +- source_initiator=None, +- source_adapter=None, +- source_hosts=None, +- source_auth=None, +- source_name=None, +- source_format=None, +- transient=False, +- start=True, # pylint: disable=redefined-outer-name +- **kwargs): +- ''' ++def pool_define( ++ name, ++ ptype, ++ target=None, ++ permissions=None, ++ source_devices=None, ++ source_dir=None, ++ source_initiator=None, ++ source_adapter=None, ++ source_hosts=None, ++ source_auth=None, ++ source_name=None, ++ source_format=None, ++ transient=False, ++ start=True, # pylint: disable=redefined-outer-name ++ **kwargs ++): ++ """ + Create libvirt pool. + + :param name: Pool name +@@ -5070,7 +5803,7 @@ def pool_define(name, + source_dir=samba_share source_hosts="['example.com']" target=/mnt/cifs + + .. versionadded:: 2019.2.0 +- ''' ++ """ + conn = __get_conn(**kwargs) + auth = _pool_set_secret(conn, ptype, name, source_auth) + +@@ -5086,7 +5819,7 @@ def pool_define(name, + source_auth=auth, + source_name=source_name, + source_format=source_format, +- source_initiator=source_initiator ++ source_initiator=source_initiator, + ) + try: + if transient: +@@ -5095,7 +5828,7 @@ def pool_define(name, + pool = conn.storagePoolDefineXML(pool_xml) + if start: + pool.create() +- except libvirtError as err: ++ except libvirt.libvirtError as err: + raise err # a real error we should report upwards + finally: + conn.close() +@@ -5104,66 +5837,69 @@ def pool_define(name, + return True + + +-def _pool_set_secret(conn, pool_type, pool_name, source_auth, uuid=None, usage=None, test=False): +- secret_types = { +- 'rbd': 'ceph', +- 'iscsi': 'chap', +- 'iscsi-direct': 'chap' +- } ++def _pool_set_secret( ++ conn, pool_type, pool_name, source_auth, uuid=None, usage=None, test=False ++): ++ secret_types = {"rbd": "ceph", "iscsi": "chap", "iscsi-direct": "chap"} + secret_type = secret_types.get(pool_type) + auth = source_auth +- if source_auth and 'username' in source_auth and 'password' in source_auth: ++ if source_auth and "username" in source_auth and "password" in source_auth: + if secret_type: + # Get the previously defined secret if any + secret = None + if usage: +- usage_type = libvirt.VIR_SECRET_USAGE_TYPE_CEPH if secret_type == 'ceph' \ +- else libvirt.VIR_SECRET_USAGE_TYPE_ISCSI ++ usage_type = ( ++ libvirt.VIR_SECRET_USAGE_TYPE_CEPH ++ if secret_type == "ceph" ++ else libvirt.VIR_SECRET_USAGE_TYPE_ISCSI ++ ) + secret = conn.secretLookupByUsage(usage_type, usage) + elif uuid: + secret = conn.secretLookupByUUIDString(uuid) + + # Create secret if needed + if not secret: +- description = 'Passphrase for {} pool created by Salt'.format(pool_name) ++ description = "Passphrase for {} pool created by Salt".format(pool_name) + if not usage: +- usage = 'pool_{}'.format(pool_name) ++ usage = "pool_{}".format(pool_name) + secret_xml = _gen_secret_xml(secret_type, usage, description) + if not test: + secret = conn.secretDefineXML(secret_xml) + + # Assign the password to it +- password = auth['password'] +- if pool_type == 'rbd': ++ password = auth["password"] ++ if pool_type == "rbd": + # RBD password are already base64-encoded, but libvirt will base64-encode them later + password = base64.b64decode(salt.utils.stringutils.to_bytes(password)) + if not test: + secret.setValue(password) + + # update auth with secret reference +- auth['type'] = secret_type +- auth['secret'] = { +- 'type': 'uuid' if uuid else 'usage', +- 'value': uuid if uuid else usage, ++ auth["type"] = secret_type ++ auth["secret"] = { ++ "type": "uuid" if uuid else "usage", ++ "value": uuid if uuid else usage, + } + return auth + + +-def pool_update(name, +- ptype, +- target=None, +- permissions=None, +- source_devices=None, +- source_dir=None, +- source_initiator=None, +- source_adapter=None, +- source_hosts=None, +- source_auth=None, +- source_name=None, +- source_format=None, +- test=False, +- **kwargs): +- ''' ++def pool_update( ++ name, ++ ptype, ++ target=None, ++ permissions=None, ++ source_devices=None, ++ source_dir=None, ++ source_initiator=None, ++ source_adapter=None, ++ source_hosts=None, ++ source_auth=None, ++ source_name=None, ++ source_format=None, ++ test=False, ++ **kwargs ++): ++ """ + Update a libvirt storage pool if needed. + If called with test=True, this is also reporting whether an update would be performed. + +@@ -5268,7 +6004,7 @@ def pool_update(name, + source_dir=samba_share source_hosts="['example.com']" target=/mnt/cifs + + .. versionadded:: 3000 +- ''' ++ """ + # Get the current definition to compare the two + conn = __get_conn(**kwargs) + needs_update = False +@@ -5278,29 +6014,33 @@ def pool_update(name, + + # If we have username and password in source_auth generate a new secret + # Or change the value of the existing one +- secret_node = old_xml.find('source/auth/secret') +- usage = secret_node.get('usage') if secret_node is not None else None +- uuid = secret_node.get('uuid') if secret_node is not None else None +- auth = _pool_set_secret(conn, ptype, name, source_auth, uuid=uuid, usage=usage, test=test) ++ secret_node = old_xml.find("source/auth/secret") ++ usage = secret_node.get("usage") if secret_node is not None else None ++ uuid = secret_node.get("uuid") if secret_node is not None else None ++ auth = _pool_set_secret( ++ conn, ptype, name, source_auth, uuid=uuid, usage=usage, test=test ++ ) + + # Compute new definition +- new_xml = ElementTree.fromstring(_gen_pool_xml( +- name, +- ptype, +- target, +- permissions=permissions, +- source_devices=source_devices, +- source_dir=source_dir, +- source_initiator=source_initiator, +- source_adapter=source_adapter, +- source_hosts=source_hosts, +- source_auth=auth, +- source_name=source_name, +- source_format=source_format +- )) ++ new_xml = ElementTree.fromstring( ++ _gen_pool_xml( ++ name, ++ ptype, ++ target, ++ permissions=permissions, ++ source_devices=source_devices, ++ source_dir=source_dir, ++ source_initiator=source_initiator, ++ source_adapter=source_adapter, ++ source_hosts=source_hosts, ++ source_auth=auth, ++ source_name=source_name, ++ source_format=source_format, ++ ) ++ ) + + # Copy over the uuid, capacity, allocation, available elements +- elements_to_copy = ['available', 'allocation', 'capacity', 'uuid'] ++ elements_to_copy = ["available", "allocation", "capacity", "uuid"] + for to_copy in elements_to_copy: + element = old_xml.find(to_copy) + new_xml.insert(1, element) +@@ -5313,29 +6053,37 @@ def pool_update(name, + + def space_stripper(node): + if node.tail is not None: +- node.tail = node.tail.strip(' \t\n') ++ node.tail = node.tail.strip(" \t\n") + if node.text is not None: +- node.text = node.text.strip(' \t\n') ++ node.text = node.text.strip(" \t\n") + + visit_xml(old_xml, space_stripper) + visit_xml(new_xml, space_stripper) + + def empty_node_remover(node): + for child in node: +- if not child.tail and not child.text and not child.items() and not child: ++ if ( ++ not child.tail ++ and not child.text ++ and not child.items() ++ and not child ++ ): + node.remove(child) ++ + visit_xml(old_xml, empty_node_remover) + +- needs_update = ElementTree.tostring(old_xml) != ElementTree.tostring(new_xml) ++ needs_update = xmlutil.to_dict(old_xml, True) != xmlutil.to_dict(new_xml, True) + if needs_update and not test: +- conn.storagePoolDefineXML(salt.utils.stringutils.to_str(ElementTree.tostring(new_xml))) ++ conn.storagePoolDefineXML( ++ salt.utils.stringutils.to_str(ElementTree.tostring(new_xml)) ++ ) + finally: + conn.close() + return needs_update + + + def list_pools(**kwargs): +- ''' ++ """ + List all storage pools. + + :param connection: libvirt connection URI, overriding defaults +@@ -5349,7 +6097,7 @@ def list_pools(**kwargs): + .. code-block:: bash + + salt '*' virt.list_pools +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + return [pool.name() for pool in conn.listAllStoragePools()] +@@ -5358,7 +6106,7 @@ def list_pools(**kwargs): + + + def pool_info(name=None, **kwargs): +- ''' ++ """ + Return informations on a storage pool provided its name. + + :param name: libvirt storage pool name +@@ -5375,45 +6123,49 @@ def pool_info(name=None, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_info default +- ''' ++ """ + result = {} + conn = __get_conn(**kwargs) + + def _pool_extract_infos(pool): +- ''' ++ """ + Format the pool info dictionary + + :param pool: the libvirt pool object +- ''' +- states = ['inactive', 'building', 'running', 'degraded', 'inaccessible'] ++ """ ++ states = ["inactive", "building", "running", "degraded", "inaccessible"] + infos = pool.info() +- state = states[infos[0]] if infos[0] < len(states) else 'unknown' ++ state = states[infos[0]] if infos[0] < len(states) else "unknown" + desc = ElementTree.fromstring(pool.XMLDesc()) +- path_node = desc.find('target/path') ++ path_node = desc.find("target/path") + return { +- 'uuid': pool.UUIDString(), +- 'state': state, +- 'capacity': infos[1], +- 'allocation': infos[2], +- 'free': infos[3], +- 'autostart': pool.autostart(), +- 'persistent': pool.isPersistent(), +- 'target_path': path_node.text if path_node is not None else None, +- 'type': desc.get('type') ++ "uuid": pool.UUIDString(), ++ "state": state, ++ "capacity": infos[1], ++ "allocation": infos[2], ++ "free": infos[3], ++ "autostart": pool.autostart(), ++ "persistent": pool.isPersistent(), ++ "target_path": path_node.text if path_node is not None else None, ++ "type": desc.get("type"), + } + + try: +- pools = [pool for pool in conn.listAllStoragePools() if name is None or pool.name() == name] ++ pools = [ ++ pool ++ for pool in conn.listAllStoragePools() ++ if name is None or pool.name() == name ++ ] + result = {pool.name(): _pool_extract_infos(pool) for pool in pools} + except libvirt.libvirtError as err: +- log.debug('Silenced libvirt error: %s', str(err)) ++ log.debug("Silenced libvirt error: %s", str(err)) + finally: + conn.close() + return result + + + def pool_get_xml(name, **kwargs): +- ''' ++ """ + Return the XML definition of a virtual storage pool + + :param name: libvirt storage pool name +@@ -5428,7 +6180,7 @@ def pool_get_xml(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_get_xml default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + return conn.storagePoolLookupByName(name).XMLDesc() +@@ -5437,7 +6189,7 @@ def pool_get_xml(name, **kwargs): + + + def pool_start(name, **kwargs): +- ''' ++ """ + Start a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5452,7 +6204,7 @@ def pool_start(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_start default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5462,7 +6214,7 @@ def pool_start(name, **kwargs): + + + def pool_build(name, **kwargs): +- ''' ++ """ + Build a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5477,7 +6229,7 @@ def pool_build(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_build default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5487,7 +6239,7 @@ def pool_build(name, **kwargs): + + + def pool_stop(name, **kwargs): +- ''' ++ """ + Stop a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5502,7 +6254,7 @@ def pool_stop(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_stop default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5512,7 +6264,7 @@ def pool_stop(name, **kwargs): + + + def pool_undefine(name, **kwargs): +- ''' ++ """ + Remove a defined libvirt storage pool. The pool needs to be stopped before calling. + + :param name: libvirt storage pool name +@@ -5527,7 +6279,7 @@ def pool_undefine(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_undefine default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5537,7 +6289,7 @@ def pool_undefine(name, **kwargs): + + + def pool_delete(name, **kwargs): +- ''' ++ """ + Delete the resources of a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5552,17 +6304,33 @@ def pool_delete(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_delete default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) ++ desc = ElementTree.fromstring(pool.XMLDesc()) ++ ++ # Is there a secret that we generated and would need to be removed? ++ # Don't remove the other secrets ++ auth_node = desc.find("source/auth") ++ if auth_node is not None: ++ auth_types = { ++ "ceph": libvirt.VIR_SECRET_USAGE_TYPE_CEPH, ++ "iscsi": libvirt.VIR_SECRET_USAGE_TYPE_ISCSI, ++ } ++ secret_type = auth_types[auth_node.get("type")] ++ secret_usage = auth_node.find("secret").get("usage") ++ if secret_type and "pool_{}".format(name) == secret_usage: ++ secret = conn.secretLookupByUsage(secret_type, secret_usage) ++ secret.undefine() ++ + return not bool(pool.delete(libvirt.VIR_STORAGE_POOL_DELETE_NORMAL)) + finally: + conn.close() + + + def pool_refresh(name, **kwargs): +- ''' ++ """ + Refresh a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5577,7 +6345,7 @@ def pool_refresh(name, **kwargs): + .. code-block:: bash + + salt '*' virt.pool_refresh default +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5586,8 +6354,8 @@ def pool_refresh(name, **kwargs): + conn.close() + + +-def pool_set_autostart(name, state='on', **kwargs): +- ''' ++def pool_set_autostart(name, state="on", **kwargs): ++ """ + Set the autostart flag on a libvirt storage pool so that the storage pool + will start with the host system on reboot. + +@@ -5605,17 +6373,17 @@ def pool_set_autostart(name, state='on', **kwargs): + .. code-block:: bash + + salt "*" virt.pool_set_autostart +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +- return not bool(pool.setAutostart(1 if state == 'on' else 0)) ++ return not bool(pool.setAutostart(1 if state == "on" else 0)) + finally: + conn.close() + + + def pool_list_volumes(name, **kwargs): +- ''' ++ """ + List the volumes contained in a defined libvirt storage pool. + + :param name: libvirt storage pool name +@@ -5630,7 +6398,7 @@ def pool_list_volumes(name, **kwargs): + .. code-block:: bash + + salt "*" virt.pool_list_volumes +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + pool = conn.storagePoolLookupByName(name) +@@ -5640,27 +6408,28 @@ def pool_list_volumes(name, **kwargs): + + + def _get_storage_vol(conn, pool, vol): +- ''' ++ """ + Helper function getting a storage volume. Will throw a libvirtError + if the pool or the volume couldn't be found. + + :param conn: libvirt connection object to use + :param pool: pool name + :param vol: volume name +- ''' ++ """ + pool_obj = conn.storagePoolLookupByName(pool) + return pool_obj.storageVolLookupByName(vol) + + + def _is_valid_volume(vol): +- ''' ++ """ + Checks whether a volume is valid for further use since those may have disappeared since + the last pool refresh. +- ''' ++ """ + try: + # Getting info on an invalid volume raises error and libvirt logs an error + def discarder(ctxt, error): # pylint: disable=unused-argument + log.debug("Ignore libvirt error: %s", error[2]) ++ + # Disable the libvirt error logging + libvirt.registerErrorHandler(discarder, None) + vol.info() +@@ -5672,20 +6441,34 @@ def _is_valid_volume(vol): + + + def _get_all_volumes_paths(conn): +- ''' ++ """ + Extract the path and backing stores path of all volumes. + + :param conn: libvirt connection to use +- ''' +- volumes = [vol for l in +- [obj.listAllVolumes() for obj in conn.listAllStoragePools() +- if obj.info()[0] == libvirt.VIR_STORAGE_POOL_RUNNING] for vol in l] +- return {vol.path(): [path.text for path in ElementTree.fromstring(vol.XMLDesc()).findall('.//backingStore/path')] +- for vol in volumes if _is_valid_volume(vol)} ++ """ ++ volumes = [ ++ vol ++ for l in [ ++ obj.listAllVolumes() ++ for obj in conn.listAllStoragePools() ++ if obj.info()[0] == libvirt.VIR_STORAGE_POOL_RUNNING ++ ] ++ for vol in l ++ ] ++ return { ++ vol.path(): [ ++ path.text ++ for path in ElementTree.fromstring(vol.XMLDesc()).findall( ++ ".//backingStore/path" ++ ) ++ ] ++ for vol in volumes ++ if _is_valid_volume(vol) ++ } + + + def volume_infos(pool=None, volume=None, **kwargs): +- ''' ++ """ + Provide details on a storage volume. If no volume name is provided, the infos + all the volumes contained in the pool are provided. If no pool is provided, + the infos of the volumes of all pools are output. +@@ -5703,7 +6486,7 @@ def volume_infos(pool=None, volume=None, **kwargs): + .. code-block:: bash + + salt "*" virt.volume_infos +- ''' ++ """ + result = {} + conn = __get_conn(**kwargs) + try: +@@ -5714,52 +6497,88 @@ def volume_infos(pool=None, volume=None, **kwargs): + except CommandExecutionError: + # Having no VM is not an error here. + domains_list = [] +- disks = {domain.name(): +- {node.get('file') for node +- in ElementTree.fromstring(domain.XMLDesc(0)).findall('.//disk/source/[@file]')} +- for domain in domains_list} ++ disks = { ++ domain.name(): { ++ node.get("file") ++ for node in ElementTree.fromstring(domain.XMLDesc(0)).findall( ++ ".//disk/source/[@file]" ++ ) ++ } ++ for domain in domains_list ++ } + + def _volume_extract_infos(vol): +- ''' ++ """ + Format the volume info dictionary + + :param vol: the libvirt storage volume object. +- ''' +- types = ['file', 'block', 'dir', 'network', 'netdir', 'ploop'] ++ """ ++ types = ["file", "block", "dir", "network", "netdir", "ploop"] + infos = vol.info() + ++ vol_xml = ElementTree.fromstring(vol.XMLDesc()) ++ backing_store_path = vol_xml.find("./backingStore/path") ++ backing_store_format = vol_xml.find("./backingStore/format") ++ backing_store = None ++ if backing_store_path is not None: ++ backing_store = { ++ "path": backing_store_path.text, ++ "format": backing_store_format.get("type") ++ if backing_store_format is not None ++ else None, ++ } ++ ++ format_node = vol_xml.find("./target/format") ++ + # If we have a path, check its use. + used_by = [] + if vol.path(): +- as_backing_store = {path for (path, all_paths) in backing_stores.items() if vol.path() in all_paths} +- used_by = [vm_name for (vm_name, vm_disks) in disks.items() +- if vm_disks & as_backing_store or vol.path() in vm_disks] ++ as_backing_store = { ++ path ++ for (path, all_paths) in backing_stores.items() ++ if vol.path() in all_paths ++ } ++ used_by = [ ++ vm_name ++ for (vm_name, vm_disks) in disks.items() ++ if vm_disks & as_backing_store or vol.path() in vm_disks ++ ] + + return { +- 'type': types[infos[0]] if infos[0] < len(types) else 'unknown', +- 'key': vol.key(), +- 'path': vol.path(), +- 'capacity': infos[1], +- 'allocation': infos[2], +- 'used_by': used_by, ++ "type": types[infos[0]] if infos[0] < len(types) else "unknown", ++ "key": vol.key(), ++ "path": vol.path(), ++ "capacity": infos[1], ++ "allocation": infos[2], ++ "used_by": used_by, ++ "backing_store": backing_store, ++ "format": format_node.get("type") if format_node is not None else None, + } + +- pools = [obj for obj in conn.listAllStoragePools() +- if (pool is None or obj.name() == pool) and obj.info()[0] == libvirt.VIR_STORAGE_POOL_RUNNING] +- vols = {pool_obj.name(): {vol.name(): _volume_extract_infos(vol) +- for vol in pool_obj.listAllVolumes() +- if (volume is None or vol.name() == volume) and _is_valid_volume(vol)} +- for pool_obj in pools} ++ pools = [ ++ obj ++ for obj in conn.listAllStoragePools() ++ if (pool is None or obj.name() == pool) ++ and obj.info()[0] == libvirt.VIR_STORAGE_POOL_RUNNING ++ ] ++ vols = { ++ pool_obj.name(): { ++ vol.name(): _volume_extract_infos(vol) ++ for vol in pool_obj.listAllVolumes() ++ if (volume is None or vol.name() == volume) and _is_valid_volume(vol) ++ } ++ for pool_obj in pools ++ } + return {pool_name: volumes for (pool_name, volumes) in vols.items() if volumes} + except libvirt.libvirtError as err: +- log.debug('Silenced libvirt error: %s', str(err)) ++ log.debug("Silenced libvirt error: %s", str(err)) + finally: + conn.close() + return result + + + def volume_delete(pool, volume, **kwargs): +- ''' ++ """ + Delete a libvirt managed volume. + + :param pool: libvirt storage pool name +@@ -5775,10 +6594,222 @@ def volume_delete(pool, volume, **kwargs): + .. code-block:: bash + + salt "*" virt.volume_delete +- ''' ++ """ + conn = __get_conn(**kwargs) + try: + vol = _get_storage_vol(conn, pool, volume) + return not bool(vol.delete()) + finally: + conn.close() ++ ++ ++def volume_define( ++ pool, ++ name, ++ size, ++ allocation=0, ++ format=None, ++ type=None, ++ permissions=None, ++ backing_store=None, ++ nocow=False, ++ **kwargs ++): ++ """ ++ Create libvirt volume. ++ ++ :param pool: name of the pool to create the volume in ++ :param name: name of the volume to define ++ :param size: capacity of the volume to define in MiB ++ :param allocation: allocated size of the volume in MiB. Defaults to 0. ++ :param format: ++ volume format. The allowed values are depending on the pool type. ++ Check the virt.pool_capabilities output for the possible values and the default. ++ :param type: ++ type of the volume. One of file, block, dir, network, netdiri, ploop or None. ++ By default, the type is guessed by libvirt from the pool type. ++ :param permissions: ++ Permissions to set on the target folder. This is mostly used for filesystem-based ++ pool types. See :ref:`pool-define-permissions` for more details on this structure. ++ :param backing_store: ++ dictionary describing a backing file for the volume. It must contain a ``path`` ++ property pointing to the base volume and a ``format`` property defining the format ++ of the base volume. ++ ++ The base volume format will not be guessed for security reasons and is thus mandatory. ++ :param nocow: disable COW for the volume. ++ :param connection: libvirt connection URI, overriding defaults ++ :param username: username to connect with, overriding defaults ++ :param password: password to connect with, overriding defaults ++ ++ .. rubric:: CLI Example: ++ ++ Volume on ESX: ++ ++ .. code-block:: bash ++ ++ salt '*' virt.volume_define "[local-storage]" myvm/myvm.vmdk vmdk 8192 ++ ++ QCow2 volume with backing file: ++ ++ .. code-block:: bash ++ ++ salt '*' virt.volume_define default myvm.qcow2 qcow2 8192 \ ++ permissions="{'mode': '0775', 'owner': '123', 'group': '345'"}" \ ++ backing_store="{'path': '/path/to/base.img', 'format': 'raw'}" \ ++ nocow=True ++ ++ .. versionadded:: Sodium ++ """ ++ ret = False ++ try: ++ conn = __get_conn(**kwargs) ++ pool_obj = conn.storagePoolLookupByName(pool) ++ pool_type = ElementTree.fromstring(pool_obj.XMLDesc()).get("type") ++ new_allocation = allocation ++ if pool_type == "logical" and size != allocation: ++ new_allocation = size ++ xml = _gen_vol_xml( ++ name, ++ size, ++ format=format, ++ allocation=new_allocation, ++ type=type, ++ permissions=permissions, ++ backing_store=backing_store, ++ nocow=nocow, ++ ) ++ ret = _define_vol_xml_str(conn, xml, pool=pool) ++ except libvirt.libvirtError as err: ++ raise CommandExecutionError(err.get_error_message()) ++ finally: ++ conn.close() ++ return ret ++ ++ ++def _volume_upload(conn, pool, volume, file, offset=0, length=0, sparse=False): ++ """ ++ Function performing the heavy duty for volume_upload but using an already ++ opened libvirt connection. ++ """ ++ ++ def handler(stream, nbytes, opaque): ++ return os.read(opaque, nbytes) ++ ++ def holeHandler(stream, opaque): ++ """ ++ Taken from the sparsestream.py libvirt-python example. ++ """ ++ fd = opaque ++ cur = os.lseek(fd, 0, os.SEEK_CUR) ++ ++ try: ++ data = os.lseek(fd, cur, os.SEEK_DATA) ++ except OSError as e: ++ if e.errno != 6: ++ raise e ++ else: ++ data = -1 ++ if data < 0: ++ inData = False ++ eof = os.lseek(fd, 0, os.SEEK_END) ++ if eof < cur: ++ raise RuntimeError("Current position in file after EOF: {}".format(cur)) ++ sectionLen = eof - cur ++ else: ++ if data > cur: ++ inData = False ++ sectionLen = data - cur ++ else: ++ inData = True ++ ++ hole = os.lseek(fd, data, os.SEEK_HOLE) ++ if hole < 0: ++ raise RuntimeError("No trailing hole") ++ ++ if hole == data: ++ raise RuntimeError("Impossible happened") ++ else: ++ sectionLen = hole - data ++ os.lseek(fd, cur, os.SEEK_SET) ++ return [inData, sectionLen] ++ ++ def skipHandler(stream, length, opaque): ++ return os.lseek(opaque, length, os.SEEK_CUR) ++ ++ stream = None ++ fd = None ++ ret = False ++ try: ++ pool_obj = conn.storagePoolLookupByName(pool) ++ vol_obj = pool_obj.storageVolLookupByName(volume) ++ ++ stream = conn.newStream() ++ fd = os.open(file, os.O_RDONLY) ++ vol_obj.upload( ++ stream, ++ offset, ++ length, ++ libvirt.VIR_STORAGE_VOL_UPLOAD_SPARSE_STREAM if sparse else 0, ++ ) ++ if sparse: ++ stream.sparseSendAll(handler, holeHandler, skipHandler, fd) ++ else: ++ stream.sendAll(handler, fd) ++ ret = True ++ except libvirt.libvirtError as err: ++ raise CommandExecutionError(err.get_error_message()) ++ finally: ++ if fd: ++ try: ++ os.close(fd) ++ except OSError as err: ++ if stream: ++ stream.abort() ++ if ret: ++ raise CommandExecutionError( ++ "Failed to close file: {0}".format(err.strerror) ++ ) ++ if stream: ++ try: ++ stream.finish() ++ except libvirt.libvirtError as err: ++ if ret: ++ raise CommandExecutionError( ++ "Failed to finish stream: {0}".format(err.get_error_message()) ++ ) ++ return ret ++ ++ ++def volume_upload(pool, volume, file, offset=0, length=0, sparse=False, **kwargs): ++ """ ++ Create libvirt volume. ++ ++ :param pool: name of the pool to create the volume in ++ :param name: name of the volume to define ++ :param file: the file to upload to the volume ++ :param offset: where to start writing the data in the volume ++ :param length: amount of bytes to transfer to the volume ++ :param sparse: set to True to preserve data sparsiness. ++ :param connection: libvirt connection URI, overriding defaults ++ :param username: username to connect with, overriding defaults ++ :param password: password to connect with, overriding defaults ++ ++ .. rubric:: CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' virt.volume_upload default myvm.qcow2 /path/to/disk.qcow2 ++ ++ .. versionadded:: Sodium ++ """ ++ conn = __get_conn(**kwargs) ++ ++ ret = False ++ try: ++ ret = _volume_upload( ++ conn, pool, volume, file, offset=offset, length=length, sparse=sparse ++ ) ++ finally: ++ conn.close() ++ return ret +diff --git a/salt/states/virt.py b/salt/states/virt.py +index 819776d707..fdef002293 100644 +--- a/salt/states/virt.py ++++ b/salt/states/virt.py +@@ -1,5 +1,5 @@ + # -*- coding: utf-8 -*- +-''' ++""" + Manage virt + =========== + +@@ -10,47 +10,49 @@ for the generation and signing of certificates for systems running libvirt: + + libvirt_keys: + virt.keys +-''' ++""" + + # Import Python libs + from __future__ import absolute_import, print_function, unicode_literals +-import copy ++ + import fnmatch + import os + +-try: +- import libvirt # pylint: disable=import-error +- HAS_LIBVIRT = True +-except ImportError: +- HAS_LIBVIRT = False +- + # Import Salt libs + import salt.utils.args + import salt.utils.files + import salt.utils.stringutils + import salt.utils.versions +-from salt.exceptions import CommandExecutionError ++from salt.exceptions import CommandExecutionError, SaltInvocationError + + # Import 3rd-party libs + from salt.ext import six + +-__virtualname__ = 'virt' ++try: ++ import libvirt # pylint: disable=import-error ++ ++ HAS_LIBVIRT = True ++except ImportError: ++ HAS_LIBVIRT = False ++ ++ ++__virtualname__ = "virt" + + + def __virtual__(): +- ''' ++ """ + Only if virt module is available. + + :return: +- ''' ++ """ + +- if 'virt.node_info' in __salt__: ++ if "virt.node_info" in __salt__: + return __virtualname__ + return False + + +-def keys(name, basepath='/etc/pki', **kwargs): +- ''' ++def keys(name, basepath="/etc/pki", **kwargs): ++ """ + Manage libvirt keys. + + name +@@ -90,65 +92,68 @@ def keys(name, basepath='/etc/pki', **kwargs): + + .. versionadded:: 2018.3.0 + +- ''' +- ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''} ++ """ ++ ret = {"name": name, "changes": {}, "result": True, "comment": ""} + + # Grab all kwargs to make them available as pillar values + # rename them to something hopefully unique to avoid + # overriding anything existing + pillar_kwargs = {} + for key, value in six.iteritems(kwargs): +- pillar_kwargs['ext_pillar_virt.{0}'.format(key)] = value ++ pillar_kwargs["ext_pillar_virt.{0}".format(key)] = value + +- pillar = __salt__['pillar.ext']({'libvirt': '_'}, pillar_kwargs) ++ pillar = __salt__["pillar.ext"]({"libvirt": "_"}, pillar_kwargs) + paths = { +- 'serverkey': os.path.join(basepath, 'libvirt', +- 'private', 'serverkey.pem'), +- 'servercert': os.path.join(basepath, 'libvirt', +- 'servercert.pem'), +- 'clientkey': os.path.join(basepath, 'libvirt', +- 'private', 'clientkey.pem'), +- 'clientcert': os.path.join(basepath, 'libvirt', +- 'clientcert.pem'), +- 'cacert': os.path.join(basepath, 'CA', 'cacert.pem') ++ "serverkey": os.path.join(basepath, "libvirt", "private", "serverkey.pem"), ++ "servercert": os.path.join(basepath, "libvirt", "servercert.pem"), ++ "clientkey": os.path.join(basepath, "libvirt", "private", "clientkey.pem"), ++ "clientcert": os.path.join(basepath, "libvirt", "clientcert.pem"), ++ "cacert": os.path.join(basepath, "CA", "cacert.pem"), + } + + for key in paths: +- p_key = 'libvirt.{0}.pem'.format(key) ++ p_key = "libvirt.{0}.pem".format(key) + if p_key not in pillar: + continue + if not os.path.exists(os.path.dirname(paths[key])): + os.makedirs(os.path.dirname(paths[key])) + if os.path.isfile(paths[key]): +- with salt.utils.files.fopen(paths[key], 'r') as fp_: ++ with salt.utils.files.fopen(paths[key], "r") as fp_: + if salt.utils.stringutils.to_unicode(fp_.read()) != pillar[p_key]: +- ret['changes'][key] = 'update' ++ ret["changes"][key] = "update" + else: +- ret['changes'][key] = 'new' +- +- if not ret['changes']: +- ret['comment'] = 'All keys are correct' +- elif __opts__['test']: +- ret['result'] = None +- ret['comment'] = 'Libvirt keys are set to be updated' +- ret['changes'] = {} ++ ret["changes"][key] = "new" ++ ++ if not ret["changes"]: ++ ret["comment"] = "All keys are correct" ++ elif __opts__["test"]: ++ ret["result"] = None ++ ret["comment"] = "Libvirt keys are set to be updated" ++ ret["changes"] = {} + else: +- for key in ret['changes']: +- with salt.utils.files.fopen(paths[key], 'w+') as fp_: ++ for key in ret["changes"]: ++ with salt.utils.files.fopen(paths[key], "w+") as fp_: + fp_.write( +- salt.utils.stringutils.to_str( +- pillar['libvirt.{0}.pem'.format(key)] +- ) ++ salt.utils.stringutils.to_str(pillar["libvirt.{0}.pem".format(key)]) + ) + +- ret['comment'] = 'Updated libvirt certs and keys' ++ ret["comment"] = "Updated libvirt certs and keys" + + return ret + + +-def _virt_call(domain, function, section, comment, state=None, +- connection=None, username=None, password=None, **kwargs): +- ''' ++def _virt_call( ++ domain, ++ function, ++ section, ++ comment, ++ state=None, ++ connection=None, ++ username=None, ++ password=None, ++ **kwargs ++): ++ """ + Helper to call the virt functions. Wildcards supported. + + :param domain: the domain to apply the function on. Can contain wildcards. +@@ -157,9 +162,9 @@ def _virt_call(domain, function, section, comment, state=None, + :param comment: comment to return + :param state: the expected final state of the VM. If None the VM state won't be checked. + :return: the salt state return +- ''' +- ret = {'name': domain, 'changes': {}, 'result': True, 'comment': ''} +- targeted_domains = fnmatch.filter(__salt__['virt.list_domains'](), domain) ++ """ ++ ret = {"name": domain, "changes": {}, "result": True, "comment": ""} ++ targeted_domains = fnmatch.filter(__salt__["virt.list_domains"](), domain) + changed_domains = list() + ignored_domains = list() + noaction_domains = list() +@@ -168,35 +173,39 @@ def _virt_call(domain, function, section, comment, state=None, + action_needed = True + # If a state has been provided, use it to see if we have something to do + if state is not None: +- domain_state = __salt__['virt.vm_state'](targeted_domain) ++ domain_state = __salt__["virt.vm_state"](targeted_domain) + action_needed = domain_state.get(targeted_domain) != state + if action_needed: +- response = __salt__['virt.{0}'.format(function)](targeted_domain, +- connection=connection, +- username=username, +- password=password, +- **kwargs) ++ response = __salt__["virt.{0}".format(function)]( ++ targeted_domain, ++ connection=connection, ++ username=username, ++ password=password, ++ **kwargs ++ ) + if isinstance(response, dict): +- response = response['name'] +- changed_domains.append({'domain': targeted_domain, function: response}) ++ response = response["name"] ++ changed_domains.append({"domain": targeted_domain, function: response}) + else: + noaction_domains.append(targeted_domain) + except libvirt.libvirtError as err: +- ignored_domains.append({'domain': targeted_domain, 'issue': six.text_type(err)}) ++ ignored_domains.append( ++ {"domain": targeted_domain, "issue": six.text_type(err)} ++ ) + if not changed_domains: +- ret['result'] = not ignored_domains and bool(targeted_domains) +- ret['comment'] = 'No changes had happened' ++ ret["result"] = not ignored_domains and bool(targeted_domains) ++ ret["comment"] = "No changes had happened" + if ignored_domains: +- ret['changes'] = {'ignored': ignored_domains} ++ ret["changes"] = {"ignored": ignored_domains} + else: +- ret['changes'] = {section: changed_domains} +- ret['comment'] = comment ++ ret["changes"] = {section: changed_domains} ++ ret["comment"] = comment + + return ret + + + def stopped(name, connection=None, username=None, password=None): +- ''' ++ """ + Stops a VM by shutting it down nicely. + + .. versionadded:: 2016.3.0 +@@ -215,14 +224,22 @@ def stopped(name, connection=None, username=None, password=None): + + domain_name: + virt.stopped +- ''' ++ """ + +- return _virt_call(name, 'shutdown', 'stopped', 'Machine has been shut down', state='shutdown', +- connection=connection, username=username, password=password) ++ return _virt_call( ++ name, ++ "shutdown", ++ "stopped", ++ "Machine has been shut down", ++ state="shutdown", ++ connection=connection, ++ username=username, ++ password=password, ++ ) + + + def powered_off(name, connection=None, username=None, password=None): +- ''' ++ """ + Stops a VM by power off. + + .. versionadded:: 2016.3.0 +@@ -241,32 +258,42 @@ def powered_off(name, connection=None, username=None, password=None): + + domain_name: + virt.stopped +- ''' +- return _virt_call(name, 'stop', 'unpowered', 'Machine has been powered off', state='shutdown', +- connection=connection, username=username, password=password) +- +- +-def defined(name, +- cpu=None, +- mem=None, +- vm_type=None, +- disk_profile=None, +- disks=None, +- nic_profile=None, +- interfaces=None, +- graphics=None, +- seed=True, +- install=True, +- pub_key=None, +- priv_key=None, +- connection=None, +- username=None, +- password=None, +- os_type=None, +- arch=None, +- boot=None, +- update=True): +- ''' ++ """ ++ return _virt_call( ++ name, ++ "stop", ++ "unpowered", ++ "Machine has been powered off", ++ state="shutdown", ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ++ ++def defined( ++ name, ++ cpu=None, ++ mem=None, ++ vm_type=None, ++ disk_profile=None, ++ disks=None, ++ nic_profile=None, ++ interfaces=None, ++ graphics=None, ++ seed=True, ++ install=True, ++ pub_key=None, ++ priv_key=None, ++ connection=None, ++ username=None, ++ password=None, ++ os_type=None, ++ arch=None, ++ boot=None, ++ update=True, ++): ++ """ + Starts an existing guest, or defines and starts a new VM with specified arguments. + + .. versionadded:: sodium +@@ -311,19 +338,15 @@ def defined(name, + but ``x86_64`` is prefed over ``i686``. Only used when creating a new virtual machine. + + :param boot: +- Specifies kernel for the virtual machine, as well as boot parameters +- for the virtual machine. This is an optionl parameter, and all of the +- keys are optional within the dictionary. If a remote path is provided +- to kernel or initrd, salt will handle the downloading of the specified +- remote fild, and will modify the XML accordingly. ++ Specifies kernel, initial ramdisk and kernel command line parameters for the virtual machine. ++ This is an optional parameter, all of the keys are optional within the dictionary. + +- .. code-block:: python ++ Refer to :ref:`init-boot-def` for the complete boot parameters description. + +- { +- 'kernel': '/root/f8-i386-vmlinuz', +- 'initrd': '/root/f8-i386-initrd', +- 'cmdline': 'console=ttyS0 ks=http://example.com/f8-i386/os/' +- } ++ To update any boot parameters, specify the new path for each. To remove any boot parameters, ++ pass a None object, for instance: 'kernel': ``None``. ++ ++ .. versionadded:: 3000 + + :param update: set to ``False`` to prevent updating a defined domain. (Default: ``True``) + +@@ -361,94 +384,104 @@ def defined(name, + type: address + address: 192.168.0.125 + +- ''' ++ """ + +- ret = {'name': name, +- 'changes': {}, +- 'result': True if not __opts__['test'] else None, +- 'comment': '' +- } ++ ret = { ++ "name": name, ++ "changes": {}, ++ "result": True if not __opts__["test"] else None, ++ "comment": "", ++ } + + try: +- if name in __salt__['virt.list_domains'](connection=connection, username=username, password=password): ++ if name in __salt__["virt.list_domains"]( ++ connection=connection, username=username, password=password ++ ): + status = {} + if update: +- status = __salt__['virt.update'](name, +- cpu=cpu, +- mem=mem, +- disk_profile=disk_profile, +- disks=disks, +- nic_profile=nic_profile, +- interfaces=interfaces, +- graphics=graphics, +- live=True, +- connection=connection, +- username=username, +- password=password, +- boot=boot, +- test=__opts__['test']) +- ret['changes'][name] = status +- if not status.get('definition'): +- ret['comment'] = 'Domain {0} unchanged'.format(name) +- ret['result'] = True +- elif status.get('errors'): +- ret['comment'] = 'Domain {0} updated with live update(s) failures'.format(name) ++ status = __salt__["virt.update"]( ++ name, ++ cpu=cpu, ++ mem=mem, ++ disk_profile=disk_profile, ++ disks=disks, ++ nic_profile=nic_profile, ++ interfaces=interfaces, ++ graphics=graphics, ++ live=True, ++ connection=connection, ++ username=username, ++ password=password, ++ boot=boot, ++ test=__opts__["test"], ++ ) ++ ret["changes"][name] = status ++ if not status.get("definition"): ++ ret["comment"] = "Domain {0} unchanged".format(name) ++ ret["result"] = True ++ elif status.get("errors"): ++ ret[ ++ "comment" ++ ] = "Domain {0} updated with live update(s) failures".format(name) + else: +- ret['comment'] = 'Domain {0} updated'.format(name) ++ ret["comment"] = "Domain {0} updated".format(name) + else: +- if not __opts__['test']: +- __salt__['virt.init'](name, +- cpu=cpu, +- mem=mem, +- os_type=os_type, +- arch=arch, +- hypervisor=vm_type, +- disk=disk_profile, +- disks=disks, +- nic=nic_profile, +- interfaces=interfaces, +- graphics=graphics, +- seed=seed, +- install=install, +- pub_key=pub_key, +- priv_key=priv_key, +- connection=connection, +- username=username, +- password=password, +- boot=boot, +- start=False) +- ret['changes'][name] = {'definition': True} +- ret['comment'] = 'Domain {0} defined'.format(name) ++ if not __opts__["test"]: ++ __salt__["virt.init"]( ++ name, ++ cpu=cpu, ++ mem=mem, ++ os_type=os_type, ++ arch=arch, ++ hypervisor=vm_type, ++ disk=disk_profile, ++ disks=disks, ++ nic=nic_profile, ++ interfaces=interfaces, ++ graphics=graphics, ++ seed=seed, ++ install=install, ++ pub_key=pub_key, ++ priv_key=priv_key, ++ connection=connection, ++ username=username, ++ password=password, ++ boot=boot, ++ start=False, ++ ) ++ ret["changes"][name] = {"definition": True} ++ ret["comment"] = "Domain {0} defined".format(name) + except libvirt.libvirtError as err: + # Something bad happened when defining / updating the VM, report it +- ret['comment'] = six.text_type(err) +- ret['result'] = False ++ ret["comment"] = six.text_type(err) ++ ret["result"] = False + + return ret + + +-def running(name, +- cpu=None, +- mem=None, +- image=None, +- vm_type=None, +- disk_profile=None, +- disks=None, +- nic_profile=None, +- interfaces=None, +- graphics=None, +- seed=True, +- install=True, +- pub_key=None, +- priv_key=None, +- update=False, +- connection=None, +- username=None, +- password=None, +- os_type=None, +- arch=None, +- boot=None): +- ''' ++def running( ++ name, ++ cpu=None, ++ mem=None, ++ vm_type=None, ++ disk_profile=None, ++ disks=None, ++ nic_profile=None, ++ interfaces=None, ++ graphics=None, ++ seed=True, ++ install=True, ++ pub_key=None, ++ priv_key=None, ++ update=False, ++ connection=None, ++ username=None, ++ password=None, ++ os_type=None, ++ arch=None, ++ boot=None, ++): ++ """ + Starts an existing guest, or defines and starts a new VM with specified arguments. + + .. versionadded:: 2016.3.0 +@@ -456,9 +489,6 @@ def running(name, + :param name: name of the virtual machine to run + :param cpu: number of CPUs for the virtual machine to create + :param mem: amount of memory in MiB for the new virtual machine +- :param image: disk image to use for the first disk of the new VM +- +- .. deprecated:: 2019.2.0 + :param vm_type: force virtual machine type for the new VM. The default value is taken from + the host capabilities. This could be useful for example to use ``'qemu'`` type instead + of the ``'kvm'`` one. +@@ -534,19 +564,13 @@ def running(name, + .. versionadded:: 3000 + + :param boot: +- Specifies kernel for the virtual machine, as well as boot parameters +- for the virtual machine. This is an optionl parameter, and all of the +- keys are optional within the dictionary. If a remote path is provided +- to kernel or initrd, salt will handle the downloading of the specified +- remote fild, and will modify the XML accordingly. ++ Specifies kernel, initial ramdisk and kernel command line parameters for the virtual machine. ++ This is an optional parameter, all of the keys are optional within the dictionary. + +- .. code-block:: python ++ Refer to :ref:`init-boot-def` for the complete boot parameters description. + +- { +- 'kernel': '/root/f8-i386-vmlinuz', +- 'initrd': '/root/f8-i386-initrd', +- 'cmdline': 'console=ttyS0 ks=http://example.com/f8-i386/os/' +- } ++ To update any boot parameters, specify the new path for each. To remove any boot parameters, ++ pass a None object, for instance: 'kernel': ``None``. + + .. versionadded:: 3000 + +@@ -606,81 +630,69 @@ def running(name, + type: address + address: 192.168.0.125 + +- ''' ++ """ + merged_disks = disks +- if image: +- default_disks = [{'system': {}}] +- disknames = ['system'] +- if disk_profile: +- disklist = copy.deepcopy( +- __salt__['config.get']('virt:disk', {}).get(disk_profile, default_disks)) +- disknames = disklist.keys() +- disk = {'name': disknames[0], 'image': image} +- if merged_disks: +- first_disk = [d for d in merged_disks if d.get('name') == disknames[0]] +- if first_disk and 'image' not in first_disk[0]: +- first_disk[0]['image'] = image +- else: +- merged_disks.append(disk) +- else: +- merged_disks = [disk] +- salt.utils.versions.warn_until( +- 'Sodium', +- '\'image\' parameter has been deprecated. Rather use the \'disks\' parameter ' +- 'to override or define the image. \'image\' will be removed in {version}.' +- ) + + if not update: +- salt.utils.versions.warn_until('Magnesium', +- '\'update\' parameter has been deprecated. Future behavior will be the one of update=True' +- 'It will be removed in {version}.') +- ret = defined(name, +- cpu=cpu, +- mem=mem, +- vm_type=vm_type, +- disk_profile=disk_profile, +- disks=merged_disks, +- nic_profile=nic_profile, +- interfaces=interfaces, +- graphics=graphics, +- seed=seed, +- install=install, +- pub_key=pub_key, +- priv_key=priv_key, +- os_type=os_type, +- arch=arch, +- boot=boot, +- update=update, +- connection=connection, +- username=username, +- password=password) +- +- result = True if not __opts__['test'] else None +- if ret['result'] is None or ret['result']: +- changed = ret['changes'][name].get('definition', False) ++ salt.utils.versions.warn_until( ++ "Aluminium", ++ "'update' parameter has been deprecated. Future behavior will be the one of update=True" ++ "It will be removed in {version}.", ++ ) ++ ret = defined( ++ name, ++ cpu=cpu, ++ mem=mem, ++ vm_type=vm_type, ++ disk_profile=disk_profile, ++ disks=merged_disks, ++ nic_profile=nic_profile, ++ interfaces=interfaces, ++ graphics=graphics, ++ seed=seed, ++ install=install, ++ pub_key=pub_key, ++ priv_key=priv_key, ++ os_type=os_type, ++ arch=arch, ++ boot=boot, ++ update=update, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ++ result = True if not __opts__["test"] else None ++ if ret["result"] is None or ret["result"]: ++ changed = ret["changes"][name].get("definition", False) + try: +- domain_state = __salt__['virt.vm_state'](name) +- if domain_state.get(name) != 'running': +- if not __opts__['test']: +- __salt__['virt.start'](name, connection=connection, username=username, password=password) +- comment = 'Domain {} started'.format(name) +- if not ret['comment'].endswith('unchanged'): +- comment = '{} and started'.format(ret['comment']) +- ret['comment'] = comment +- ret['changes'][name]['started'] = True ++ domain_state = __salt__["virt.vm_state"](name) ++ if domain_state.get(name) != "running": ++ if not __opts__["test"]: ++ __salt__["virt.start"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ comment = "Domain {} started".format(name) ++ if not ret["comment"].endswith("unchanged"): ++ comment = "{} and started".format(ret["comment"]) ++ ret["comment"] = comment ++ ret["changes"][name]["started"] = True + elif not changed: +- ret['comment'] = 'Domain {0} exists and is running'.format(name) ++ ret["comment"] = "Domain {0} exists and is running".format(name) + + except libvirt.libvirtError as err: + # Something bad happened when starting / updating the VM, report it +- ret['comment'] = six.text_type(err) +- ret['result'] = False ++ ret["comment"] = six.text_type(err) ++ ret["result"] = False + + return ret + + + def snapshot(name, suffix=None, connection=None, username=None, password=None): +- ''' ++ """ + Takes a snapshot of a particular VM or by a UNIX-style wildcard. + + .. versionadded:: 2016.3.0 +@@ -704,15 +716,23 @@ def snapshot(name, suffix=None, connection=None, username=None, password=None): + domain*: + virt.snapshot: + - suffix: periodic +- ''' ++ """ + +- return _virt_call(name, 'snapshot', 'saved', 'Snapshot has been taken', suffix=suffix, +- connection=connection, username=username, password=password) ++ return _virt_call( ++ name, ++ "snapshot", ++ "saved", ++ "Snapshot has been taken", ++ suffix=suffix, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + + + # Deprecated states + def rebooted(name, connection=None, username=None, password=None): +- ''' ++ """ + Reboots VMs + + .. versionadded:: 2016.3.0 +@@ -728,14 +748,21 @@ def rebooted(name, connection=None, username=None, password=None): + :param password: password to connect with, overriding defaults + + .. versionadded:: 2019.2.0 +- ''' ++ """ + +- return _virt_call(name, 'reboot', 'rebooted', "Machine has been rebooted", +- connection=connection, username=username, password=password) ++ return _virt_call( ++ name, ++ "reboot", ++ "rebooted", ++ "Machine has been rebooted", ++ connection=connection, ++ username=username, ++ password=password, ++ ) + + + def unpowered(name): +- ''' ++ """ + .. deprecated:: 2016.3.0 + Use :py:func:`~salt.modules.virt.powered_off` instead. + +@@ -747,13 +774,13 @@ def unpowered(name): + + domain_name: + virt.stopped +- ''' ++ """ + +- return _virt_call(name, 'stop', 'unpowered', 'Machine has been powered off') ++ return _virt_call(name, "stop", "unpowered", "Machine has been powered off") + + + def saved(name, suffix=None): +- ''' ++ """ + .. deprecated:: 2016.3.0 + Use :py:func:`~salt.modules.virt.snapshot` instead. + +@@ -770,13 +797,17 @@ def saved(name, suffix=None): + domain*: + virt.saved: + - suffix: periodic +- ''' ++ """ + +- return _virt_call(name, 'snapshot', 'saved', 'Snapshots has been taken', suffix=suffix) ++ return _virt_call( ++ name, "snapshot", "saved", "Snapshots has been taken", suffix=suffix ++ ) + + +-def reverted(name, snapshot=None, cleanup=False): # pylint: disable=redefined-outer-name +- ''' ++def reverted( ++ name, snapshot=None, cleanup=False ++): # pylint: disable=redefined-outer-name ++ """ + .. deprecated:: 2016.3.0 + + Reverts to the particular snapshot. +@@ -793,59 +824,71 @@ def reverted(name, snapshot=None, cleanup=False): # pylint: disable=redefined-o + virt.reverted: + - snapshot: snapshot_name + - cleanup: False +- ''' +- ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''} ++ """ ++ ret = {"name": name, "changes": {}, "result": False, "comment": ""} + + try: +- domains = fnmatch.filter(__salt__['virt.list_domains'](), name) ++ domains = fnmatch.filter(__salt__["virt.list_domains"](), name) + if not domains: +- ret['comment'] = 'No domains found for criteria "{0}"'.format(name) ++ ret["comment"] = 'No domains found for criteria "{0}"'.format(name) + else: + ignored_domains = list() + if len(domains) > 1: +- ret['changes'] = {'reverted': list()} ++ ret["changes"] = {"reverted": list()} + for domain in domains: + result = {} + try: +- result = __salt__['virt.revert_snapshot'](domain, snapshot=snapshot, cleanup=cleanup) +- result = {'domain': domain, 'current': result['reverted'], 'deleted': result['deleted']} ++ result = __salt__["virt.revert_snapshot"]( ++ domain, snapshot=snapshot, cleanup=cleanup ++ ) ++ result = { ++ "domain": domain, ++ "current": result["reverted"], ++ "deleted": result["deleted"], ++ } + except CommandExecutionError as err: + if len(domains) > 1: +- ignored_domains.append({'domain': domain, 'issue': six.text_type(err)}) ++ ignored_domains.append( ++ {"domain": domain, "issue": six.text_type(err)} ++ ) + if len(domains) > 1: + if result: +- ret['changes']['reverted'].append(result) ++ ret["changes"]["reverted"].append(result) + else: +- ret['changes'] = result ++ ret["changes"] = result + break + +- ret['result'] = len(domains) != len(ignored_domains) +- if ret['result']: +- ret['comment'] = 'Domain{0} has been reverted'.format(len(domains) > 1 and "s" or "") ++ ret["result"] = len(domains) != len(ignored_domains) ++ if ret["result"]: ++ ret["comment"] = "Domain{0} has been reverted".format( ++ len(domains) > 1 and "s" or "" ++ ) + if ignored_domains: +- ret['changes']['ignored'] = ignored_domains +- if not ret['changes']['reverted']: +- ret['changes'].pop('reverted') ++ ret["changes"]["ignored"] = ignored_domains ++ if not ret["changes"]["reverted"]: ++ ret["changes"].pop("reverted") + except libvirt.libvirtError as err: +- ret['comment'] = six.text_type(err) ++ ret["comment"] = six.text_type(err) + except CommandExecutionError as err: +- ret['comment'] = six.text_type(err) ++ ret["comment"] = six.text_type(err) + + return ret + + +-def network_defined(name, +- bridge, +- forward, +- vport=None, +- tag=None, +- ipv4_config=None, +- ipv6_config=None, +- autostart=True, +- connection=None, +- username=None, +- password=None): +- ''' ++def network_defined( ++ name, ++ bridge, ++ forward, ++ vport=None, ++ tag=None, ++ ipv4_config=None, ++ ipv6_config=None, ++ autostart=True, ++ connection=None, ++ username=None, ++ password=None, ++): ++ """ + Defines a new network with specified arguments. + + :param bridge: Bridge name +@@ -899,53 +942,60 @@ def network_defined(name, + end: 192.168.42.150 + - autostart: True + +- ''' +- ret = {'name': name, +- 'changes': {}, +- 'result': True if not __opts__['test'] else None, +- 'comment': '' +- } ++ """ ++ ret = { ++ "name": name, ++ "changes": {}, ++ "result": True if not __opts__["test"] else None, ++ "comment": "", ++ } + + try: +- info = __salt__['virt.network_info'](name, connection=connection, username=username, password=password) ++ info = __salt__["virt.network_info"]( ++ name, connection=connection, username=username, password=password ++ ) + if info and info[name]: +- ret['comment'] = 'Network {0} exists'.format(name) +- ret['result'] = True ++ ret["comment"] = "Network {0} exists".format(name) ++ ret["result"] = True + else: +- if not __opts__['test']: +- __salt__['virt.network_define'](name, +- bridge, +- forward, +- vport=vport, +- tag=tag, +- ipv4_config=ipv4_config, +- ipv6_config=ipv6_config, +- autostart=autostart, +- start=False, +- connection=connection, +- username=username, +- password=password) +- ret['changes'][name] = 'Network defined' +- ret['comment'] = 'Network {0} defined'.format(name) ++ if not __opts__["test"]: ++ __salt__["virt.network_define"]( ++ name, ++ bridge, ++ forward, ++ vport=vport, ++ tag=tag, ++ ipv4_config=ipv4_config, ++ ipv6_config=ipv6_config, ++ autostart=autostart, ++ start=False, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ret["changes"][name] = "Network defined" ++ ret["comment"] = "Network {0} defined".format(name) + except libvirt.libvirtError as err: +- ret['result'] = False +- ret['comment'] = err.get_error_message() ++ ret["result"] = False ++ ret["comment"] = err.get_error_message() + + return ret + + +-def network_running(name, +- bridge, +- forward, +- vport=None, +- tag=None, +- ipv4_config=None, +- ipv6_config=None, +- autostart=True, +- connection=None, +- username=None, +- password=None): +- ''' ++def network_running( ++ name, ++ bridge, ++ forward, ++ vport=None, ++ tag=None, ++ ipv4_config=None, ++ ipv6_config=None, ++ autostart=True, ++ connection=None, ++ username=None, ++ password=None, ++): ++ """ + Defines and starts a new network with specified arguments. + + :param bridge: Bridge name +@@ -1007,57 +1057,70 @@ def network_running(name, + end: 192.168.42.150 + - autostart: True + +- ''' +- ret = network_defined(name, +- bridge, +- forward, +- vport=vport, +- tag=tag, +- ipv4_config=ipv4_config, +- ipv6_config=ipv6_config, +- autostart=autostart, +- connection=connection, +- username=username, +- password=password) +- +- defined = name in ret['changes'] and ret['changes'][name].startswith('Network defined') +- +- result = True if not __opts__['test'] else None +- if ret['result'] is None or ret['result']: ++ """ ++ ret = network_defined( ++ name, ++ bridge, ++ forward, ++ vport=vport, ++ tag=tag, ++ ipv4_config=ipv4_config, ++ ipv6_config=ipv6_config, ++ autostart=autostart, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ++ defined = name in ret["changes"] and ret["changes"][name].startswith( ++ "Network defined" ++ ) ++ ++ result = True if not __opts__["test"] else None ++ if ret["result"] is None or ret["result"]: + try: +- info = __salt__['virt.network_info'](name, connection=connection, username=username, password=password) ++ info = __salt__["virt.network_info"]( ++ name, connection=connection, username=username, password=password ++ ) + # In the corner case where test=True and the network wasn't defined + # we may not get the network in the info dict and that is normal. +- if info.get(name, {}).get('active', False): +- ret['comment'] = '{} and is running'.format(ret['comment']) ++ if info.get(name, {}).get("active", False): ++ ret["comment"] = "{} and is running".format(ret["comment"]) + else: +- if not __opts__['test']: +- __salt__['virt.network_start'](name, connection=connection, username=username, password=password) +- change = 'Network started' +- if name in ret['changes']: +- change = '{} and started'.format(ret['changes'][name]) +- ret['changes'][name] = change +- ret['comment'] = '{} and started'.format(ret['comment']) +- ret['result'] = result ++ if not __opts__["test"]: ++ __salt__["virt.network_start"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ change = "Network started" ++ if name in ret["changes"]: ++ change = "{} and started".format(ret["changes"][name]) ++ ret["changes"][name] = change ++ ret["comment"] = "{} and started".format(ret["comment"]) ++ ret["result"] = result + + except libvirt.libvirtError as err: +- ret['result'] = False +- ret['comment'] = err.get_error_message() ++ ret["result"] = False ++ ret["comment"] = err.get_error_message() + + return ret + + +-def pool_defined(name, +- ptype=None, +- target=None, +- permissions=None, +- source=None, +- transient=False, +- autostart=True, +- connection=None, +- username=None, +- password=None): +- ''' ++def pool_defined( ++ name, ++ ptype=None, ++ target=None, ++ permissions=None, ++ source=None, ++ transient=False, ++ autostart=True, ++ connection=None, ++ username=None, ++ password=None, ++): ++ """ + Defines a new pool with specified arguments. + + .. versionadded:: sodium +@@ -1097,124 +1160,150 @@ def pool_defined(name, + format: cifs + - autostart: True + +- ''' +- ret = {'name': name, +- 'changes': {}, +- 'result': True if not __opts__['test'] else None, +- 'comment': '' +- } ++ """ ++ ret = { ++ "name": name, ++ "changes": {}, ++ "result": True if not __opts__["test"] else None, ++ "comment": "", ++ } + + try: +- info = __salt__['virt.pool_info'](name, connection=connection, username=username, password=password) ++ info = __salt__["virt.pool_info"]( ++ name, connection=connection, username=username, password=password ++ ) + needs_autostart = False + if info: +- needs_autostart = info[name]['autostart'] and not autostart or not info[name]['autostart'] and autostart ++ needs_autostart = ( ++ info[name]["autostart"] ++ and not autostart ++ or not info[name]["autostart"] ++ and autostart ++ ) + + # Update can happen for both running and stopped pools +- needs_update = __salt__['virt.pool_update'](name, +- ptype=ptype, +- target=target, +- permissions=permissions, +- source_devices=(source or {}).get('devices'), +- source_dir=(source or {}).get('dir'), +- source_initiator=(source or {}).get('initiator'), +- source_adapter=(source or {}).get('adapter'), +- source_hosts=(source or {}).get('hosts'), +- source_auth=(source or {}).get('auth'), +- source_name=(source or {}).get('name'), +- source_format=(source or {}).get('format'), +- test=True, +- connection=connection, +- username=username, +- password=password) ++ needs_update = __salt__["virt.pool_update"]( ++ name, ++ ptype=ptype, ++ target=target, ++ permissions=permissions, ++ source_devices=(source or {}).get("devices"), ++ source_dir=(source or {}).get("dir"), ++ source_initiator=(source or {}).get("initiator"), ++ source_adapter=(source or {}).get("adapter"), ++ source_hosts=(source or {}).get("hosts"), ++ source_auth=(source or {}).get("auth"), ++ source_name=(source or {}).get("name"), ++ source_format=(source or {}).get("format"), ++ test=True, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + if needs_update: +- if not __opts__['test']: +- __salt__['virt.pool_update'](name, +- ptype=ptype, +- target=target, +- permissions=permissions, +- source_devices=(source or {}).get('devices'), +- source_dir=(source or {}).get('dir'), +- source_initiator=(source or {}).get('initiator'), +- source_adapter=(source or {}).get('adapter'), +- source_hosts=(source or {}).get('hosts'), +- source_auth=(source or {}).get('auth'), +- source_name=(source or {}).get('name'), +- source_format=(source or {}).get('format'), +- connection=connection, +- username=username, +- password=password) +- +- action = '' +- if info[name]['state'] != 'running': +- if not __opts__['test']: +- __salt__['virt.pool_build'](name, connection=connection, username=username, password=password) +- action = ', built' +- +- action = '{}, autostart flag changed'.format(action) if needs_autostart else action +- ret['changes'][name] = 'Pool updated{0}'.format(action) +- ret['comment'] = 'Pool {0} updated{1}'.format(name, action) ++ if not __opts__["test"]: ++ __salt__["virt.pool_update"]( ++ name, ++ ptype=ptype, ++ target=target, ++ permissions=permissions, ++ source_devices=(source or {}).get("devices"), ++ source_dir=(source or {}).get("dir"), ++ source_initiator=(source or {}).get("initiator"), ++ source_adapter=(source or {}).get("adapter"), ++ source_hosts=(source or {}).get("hosts"), ++ source_auth=(source or {}).get("auth"), ++ source_name=(source or {}).get("name"), ++ source_format=(source or {}).get("format"), ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ++ action = "" ++ if info[name]["state"] != "running": ++ if not __opts__["test"]: ++ __salt__["virt.pool_build"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ action = ", built" ++ ++ action = ( ++ "{}, autostart flag changed".format(action) ++ if needs_autostart ++ else action ++ ) ++ ret["changes"][name] = "Pool updated{0}".format(action) ++ ret["comment"] = "Pool {0} updated{1}".format(name, action) + + else: +- ret['comment'] = 'Pool {0} unchanged'.format(name) +- ret['result'] = True ++ ret["comment"] = "Pool {0} unchanged".format(name) ++ ret["result"] = True + else: + needs_autostart = autostart +- if not __opts__['test']: +- __salt__['virt.pool_define'](name, +- ptype=ptype, +- target=target, +- permissions=permissions, +- source_devices=(source or {}).get('devices'), +- source_dir=(source or {}).get('dir'), +- source_initiator=(source or {}).get('initiator'), +- source_adapter=(source or {}).get('adapter'), +- source_hosts=(source or {}).get('hosts'), +- source_auth=(source or {}).get('auth'), +- source_name=(source or {}).get('name'), +- source_format=(source or {}).get('format'), +- transient=transient, +- start=False, +- connection=connection, +- username=username, +- password=password) +- +- __salt__['virt.pool_build'](name, +- connection=connection, +- username=username, +- password=password) ++ if not __opts__["test"]: ++ __salt__["virt.pool_define"]( ++ name, ++ ptype=ptype, ++ target=target, ++ permissions=permissions, ++ source_devices=(source or {}).get("devices"), ++ source_dir=(source or {}).get("dir"), ++ source_initiator=(source or {}).get("initiator"), ++ source_adapter=(source or {}).get("adapter"), ++ source_hosts=(source or {}).get("hosts"), ++ source_auth=(source or {}).get("auth"), ++ source_name=(source or {}).get("name"), ++ source_format=(source or {}).get("format"), ++ transient=transient, ++ start=False, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ ++ __salt__["virt.pool_build"]( ++ name, connection=connection, username=username, password=password ++ ) + if needs_autostart: +- ret['changes'][name] = 'Pool defined, marked for autostart' +- ret['comment'] = 'Pool {0} defined, marked for autostart'.format(name) ++ ret["changes"][name] = "Pool defined, marked for autostart" ++ ret["comment"] = "Pool {0} defined, marked for autostart".format(name) + else: +- ret['changes'][name] = 'Pool defined' +- ret['comment'] = 'Pool {0} defined'.format(name) ++ ret["changes"][name] = "Pool defined" ++ ret["comment"] = "Pool {0} defined".format(name) + + if needs_autostart: +- if not __opts__['test']: +- __salt__['virt.pool_set_autostart'](name, +- state='on' if autostart else 'off', +- connection=connection, +- username=username, +- password=password) ++ if not __opts__["test"]: ++ __salt__["virt.pool_set_autostart"]( ++ name, ++ state="on" if autostart else "off", ++ connection=connection, ++ username=username, ++ password=password, ++ ) + except libvirt.libvirtError as err: +- ret['comment'] = err.get_error_message() +- ret['result'] = False ++ ret["comment"] = err.get_error_message() ++ ret["result"] = False + + return ret + + +-def pool_running(name, +- ptype=None, +- target=None, +- permissions=None, +- source=None, +- transient=False, +- autostart=True, +- connection=None, +- username=None, +- password=None): +- ''' ++def pool_running( ++ name, ++ ptype=None, ++ target=None, ++ permissions=None, ++ source=None, ++ transient=False, ++ autostart=True, ++ connection=None, ++ username=None, ++ password=None, ++): ++ """ + Defines and starts a new pool with specified arguments. + + .. versionadded:: 2019.2.0 +@@ -1230,8 +1319,6 @@ def pool_running(name, + when set to ``True``, the pool will be automatically undefined after being stopped. (Default: ``False``) + :param autostart: + Whether to start the pool when booting the host. (Default: ``True``) +- :param start: +- When ``True``, define and start the pool, otherwise the pool will be left stopped. + :param connection: libvirt connection URI, overriding defaults + :param username: username to connect with, overriding defaults + :param password: password to connect with, overriding defaults +@@ -1259,68 +1346,83 @@ def pool_running(name, + format: cifs + - autostart: True + +- ''' +- ret = pool_defined(name, +- ptype=ptype, +- target=target, +- permissions=permissions, +- source=source, +- transient=transient, +- autostart=autostart, +- connection=connection, +- username=username, +- password=password) +- defined = name in ret['changes'] and ret['changes'][name].startswith('Pool defined') +- updated = name in ret['changes'] and ret['changes'][name].startswith('Pool updated') +- +- result = True if not __opts__['test'] else None +- if ret['result'] is None or ret['result']: ++ """ ++ ret = pool_defined( ++ name, ++ ptype=ptype, ++ target=target, ++ permissions=permissions, ++ source=source, ++ transient=transient, ++ autostart=autostart, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ defined = name in ret["changes"] and ret["changes"][name].startswith("Pool defined") ++ updated = name in ret["changes"] and ret["changes"][name].startswith("Pool updated") ++ ++ result = True if not __opts__["test"] else None ++ if ret["result"] is None or ret["result"]: + try: +- info = __salt__['virt.pool_info'](name, connection=connection, username=username, password=password) +- action = 'started' +- # In the corner case where test=True and the pool wasn't defined ++ info = __salt__["virt.pool_info"]( ++ name, connection=connection, username=username, password=password ++ ) ++ action = "started" ++ # In the corner case where test=True and the pool wasn"t defined + # we may get not get our pool in the info dict and that is normal. +- is_running = info.get(name, {}).get('state', 'stopped') == 'running' ++ is_running = info.get(name, {}).get("state", "stopped") == "running" + if is_running: + if updated: +- action = 'built, restarted' +- if not __opts__['test']: +- __salt__['virt.pool_stop'](name, connection=connection, username=username, password=password) +- if not __opts__['test']: +- __salt__['virt.pool_build'](name, connection=connection, username=username, password=password) ++ action = "built, restarted" ++ if not __opts__["test"]: ++ __salt__["virt.pool_stop"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ if not __opts__["test"]: ++ __salt__["virt.pool_build"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + else: +- action = 'already running' ++ action = "already running" + result = True + + if not is_running or updated or defined: +- if not __opts__['test']: +- __salt__['virt.pool_start'](name, connection=connection, username=username, password=password) ++ if not __opts__["test"]: ++ __salt__["virt.pool_start"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + +- comment = 'Pool {0}'.format(name) +- change = 'Pool' +- if name in ret['changes']: +- comment = '{0},'.format(ret['comment']) +- change = '{0},'.format(ret['changes'][name]) ++ comment = "Pool {0}".format(name) ++ change = "Pool" ++ if name in ret["changes"]: ++ comment = "{0},".format(ret["comment"]) ++ change = "{0},".format(ret["changes"][name]) + +- if action != 'already running': +- ret['changes'][name] = '{0} {1}'.format(change, action) ++ if action != "already running": ++ ret["changes"][name] = "{0} {1}".format(change, action) + +- ret['comment'] = '{0} {1}'.format(comment, action) +- ret['result'] = result ++ ret["comment"] = "{0} {1}".format(comment, action) ++ ret["result"] = result + + except libvirt.libvirtError as err: +- ret['comment'] = err.get_error_message() +- ret['result'] = False ++ ret["comment"] = err.get_error_message() ++ ret["result"] = False + + return ret + + +-def pool_deleted(name, +- purge=False, +- connection=None, +- username=None, +- password=None): +- ''' ++def pool_deleted(name, purge=False, connection=None, username=None, password=None): ++ """ + Deletes a virtual storage pool. + + :param name: the name of the pool to delete. +@@ -1345,81 +1447,253 @@ def pool_deleted(name, + - purge: True + + .. versionadded:: 3000 +- ''' +- ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''} ++ """ ++ ret = {"name": name, "changes": {}, "result": True, "comment": ""} + + try: +- info = __salt__['virt.pool_info'](name, connection=connection, username=username, password=password) ++ info = __salt__["virt.pool_info"]( ++ name, connection=connection, username=username, password=password ++ ) + if info: +- ret['changes']['stopped'] = False +- ret['changes']['deleted'] = False +- ret['changes']['undefined'] = False +- ret['changes']['deleted_volumes'] = [] ++ ret["changes"]["stopped"] = False ++ ret["changes"]["deleted"] = False ++ ret["changes"]["undefined"] = False ++ ret["changes"]["deleted_volumes"] = [] + unsupported = [] + +- if info[name]['state'] == 'running': ++ if info[name]["state"] == "running": + if purge: +- unsupported_volume_delete = ['iscsi', 'iscsi-direct', 'mpath', 'scsi'] +- if info[name]['type'] not in unsupported_volume_delete: +- __salt__['virt.pool_refresh'](name, +- connection=connection, +- username=username, +- password=password) +- volumes = __salt__['virt.pool_list_volumes'](name, +- connection=connection, +- username=username, +- password=password) ++ unsupported_volume_delete = [ ++ "iscsi", ++ "iscsi-direct", ++ "mpath", ++ "scsi", ++ ] ++ if info[name]["type"] not in unsupported_volume_delete: ++ __salt__["virt.pool_refresh"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ volumes = __salt__["virt.pool_list_volumes"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + for volume in volumes: + # Not supported for iSCSI and SCSI drivers +- deleted = __opts__['test'] +- if not __opts__['test']: +- deleted = __salt__['virt.volume_delete'](name, +- volume, +- connection=connection, +- username=username, +- password=password) ++ deleted = __opts__["test"] ++ if not __opts__["test"]: ++ deleted = __salt__["virt.volume_delete"]( ++ name, ++ volume, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + if deleted: +- ret['changes']['deleted_volumes'].append(volume) ++ ret["changes"]["deleted_volumes"].append(volume) + else: +- unsupported.append('deleting volume') +- +- if not __opts__['test']: +- ret['changes']['stopped'] = __salt__['virt.pool_stop'](name, +- connection=connection, +- username=username, +- password=password) ++ unsupported.append("deleting volume") ++ ++ if not __opts__["test"]: ++ ret["changes"]["stopped"] = __salt__["virt.pool_stop"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + else: +- ret['changes']['stopped'] = True ++ ret["changes"]["stopped"] = True + + if purge: +- supported_pool_delete = ['dir', 'fs', 'netfs', 'logical', 'vstorage', 'zfs'] +- if info[name]['type'] in supported_pool_delete: +- if not __opts__['test']: +- ret['changes']['deleted'] = __salt__['virt.pool_delete'](name, +- connection=connection, +- username=username, +- password=password) ++ supported_pool_delete = [ ++ "dir", ++ "fs", ++ "netfs", ++ "logical", ++ "vstorage", ++ "zfs", ++ ] ++ if info[name]["type"] in supported_pool_delete: ++ if not __opts__["test"]: ++ ret["changes"]["deleted"] = __salt__["virt.pool_delete"]( ++ name, ++ connection=connection, ++ username=username, ++ password=password, ++ ) + else: +- ret['changes']['deleted'] = True ++ ret["changes"]["deleted"] = True + else: +- unsupported.append('deleting pool') ++ unsupported.append("deleting pool") + +- if not __opts__['test']: +- ret['changes']['undefined'] = __salt__['virt.pool_undefine'](name, +- connection=connection, +- username=username, +- password=password) ++ if not __opts__["test"]: ++ ret["changes"]["undefined"] = __salt__["virt.pool_undefine"]( ++ name, connection=connection, username=username, password=password ++ ) + else: +- ret['changes']['undefined'] = True +- ret['result'] = None ++ ret["changes"]["undefined"] = True ++ ret["result"] = None + + if unsupported: +- ret['comment'] = 'Unsupported actions for pool of type "{0}": {1}'.format(info[name]['type'], +- ', '.join(unsupported)) ++ ret[ ++ "comment" ++ ] = 'Unsupported actions for pool of type "{0}": {1}'.format( ++ info[name]["type"], ", ".join(unsupported) ++ ) + else: +- ret['comment'] = 'Storage pool could not be found: {0}'.format(name) ++ ret["comment"] = "Storage pool could not be found: {0}".format(name) + except libvirt.libvirtError as err: +- ret['comment'] = 'Failed deleting pool: {0}'.format(err.get_error_message()) +- ret['result'] = False ++ ret["comment"] = "Failed deleting pool: {0}".format(err.get_error_message()) ++ ret["result"] = False ++ ++ return ret ++ ++ ++def volume_defined( ++ pool, ++ name, ++ size, ++ allocation=0, ++ format=None, ++ type=None, ++ permissions=None, ++ backing_store=None, ++ nocow=False, ++ connection=None, ++ username=None, ++ password=None, ++): ++ """ ++ Ensure a disk volume is existing. ++ ++ :param pool: name of the pool containing the volume ++ :param name: name of the volume ++ :param size: capacity of the volume to define in MiB ++ :param allocation: allocated size of the volume in MiB. Defaults to 0. ++ :param format: ++ volume format. The allowed values are depending on the pool type. ++ Check the virt.pool_capabilities output for the possible values and the default. ++ :param type: ++ type of the volume. One of file, block, dir, network, netdiri, ploop or None. ++ By default, the type is guessed by libvirt from the pool type. ++ :param permissions: ++ Permissions to set on the target folder. This is mostly used for filesystem-based ++ pool types. See :ref:`pool-define-permissions` for more details on this structure. ++ :param backing_store: ++ dictionary describing a backing file for the volume. It must contain a ``path`` ++ property pointing to the base volume and a ``format`` property defining the format ++ of the base volume. ++ ++ The base volume format will not be guessed for security reasons and is thus mandatory. ++ :param nocow: disable COW for the volume. ++ :param connection: libvirt connection URI, overriding defaults ++ :param username: username to connect with, overriding defaults ++ :param password: password to connect with, overriding defaults ++ ++ .. rubric:: CLI Example: + ++ Volume on ESX: ++ ++ .. code-block:: yaml ++ ++ esx_volume: ++ virt.volume_defined: ++ - pool: "[local-storage]" ++ - name: myvm/myvm.vmdk ++ - size: 8192 ++ ++ QCow2 volume with backing file: ++ ++ .. code-block:: bash ++ ++ myvolume: ++ virt.volume_defined: ++ - pool: default ++ - name: myvm.qcow2 ++ - format: qcow2 ++ - size: 8192 ++ - permissions: ++ mode: '0775' ++ owner: '123' ++ group: '345' ++ - backing_store: ++ path: /path/to/base.img ++ format: raw ++ - nocow: True ++ ++ .. versionadded:: Sodium ++ """ ++ ret = {"name": name, "changes": {}, "result": True, "comment": ""} ++ ++ pools = __salt__["virt.list_pools"]( ++ connection=connection, username=username, password=password ++ ) ++ if pool not in pools: ++ raise SaltInvocationError("Storage pool {} not existing".format(pool)) ++ ++ vol_infos = ( ++ __salt__["virt.volume_infos"]( ++ pool, name, connection=connection, username=username, password=password ++ ) ++ .get(pool, {}) ++ .get(name) ++ ) ++ ++ if vol_infos: ++ ret["comment"] = "volume is existing" ++ # if backing store or format are different, return an error ++ backing_store_info = vol_infos.get("backing_store") or {} ++ same_backing_store = backing_store_info.get("path") == ( ++ backing_store or {} ++ ).get("path") and backing_store_info.get("format") == (backing_store or {}).get( ++ "format" ++ ) ++ if not same_backing_store or ( ++ vol_infos.get("format") != format and format is not None ++ ): ++ ret["result"] = False ++ ret[ ++ "comment" ++ ] = "A volume with the same name but different backing store or format is existing" ++ return ret ++ ++ # otherwise assume the volume has already been defined ++ # if the sizes don't match, issue a warning comment: too dangerous to do this for now ++ if int(vol_infos.get("capacity")) != int(size) * 1024 * 1024: ++ ret[ ++ "comment" ++ ] = "The capacity of the volume is different, but no resize performed" ++ return ret ++ ++ ret["result"] = None if __opts__["test"] else True ++ test_comment = "would be " ++ try: ++ if not __opts__["test"]: ++ __salt__["virt.volume_define"]( ++ pool, ++ name, ++ size, ++ allocation=allocation, ++ format=format, ++ type=type, ++ permissions=permissions, ++ backing_store=backing_store, ++ nocow=nocow, ++ connection=connection, ++ username=username, ++ password=password, ++ ) ++ test_comment = "" ++ ++ ret["comment"] = "Volume {} {}defined in pool {}".format( ++ name, test_comment, pool ++ ) ++ ret["changes"] = {"{}/{}".format(pool, name): {"old": "", "new": "defined"}} ++ except libvirt.libvirtError as err: ++ ret["comment"] = err.get_error_message() ++ ret["result"] = False + return ret +diff --git a/salt/templates/virt/libvirt_domain.jinja b/salt/templates/virt/libvirt_domain.jinja +index fdaea168f2..aac6283eb0 100644 +--- a/salt/templates/virt/libvirt_domain.jinja ++++ b/salt/templates/virt/libvirt_domain.jinja +@@ -15,6 +15,12 @@ + {% if 'cmdline' in boot %} + {{ boot.cmdline }} + {% endif %} ++ {% if 'loader' in boot %} ++ {{ boot.loader }} ++ {% endif %} ++ {% if 'nvram' in boot %} ++ ++ {% endif %} + {% endif %} + {% for dev in boot_dev %} + +@@ -22,16 +28,31 @@ + + + {% for disk in disks %} +- +- {% if 'source_file' in disk %} ++ ++ {% if disk.type == 'file' and 'source_file' in disk -%} + + {% endif %} ++ {% if disk.type == 'volume' and 'pool' in disk -%} ++ ++ {% endif %} ++ {%- if disk.type == 'network' %} ++ ++ {%- for host in disk.get('hosts') %} ++ ++ {%- endfor %} ++ {%- if disk.get("auth") %} ++ ++ ++ ++ {%- endif %} ++ ++ {%- endif %} + + {% if disk.address -%} +
+ {% endif %} + {% if disk.driver -%} +- ++ + {% endif %} + + {% endfor %} +@@ -43,7 +64,9 @@ + {% for nic in nics %} + + ++ {% if nic.get('mac') -%} + ++ {%- endif %} + {% if nic.model %}{% endif %} + + {% endfor %} +diff --git a/salt/templates/virt/libvirt_volume.jinja b/salt/templates/virt/libvirt_volume.jinja +index 5cbe098826..8e5748179f 100644 +--- a/salt/templates/virt/libvirt_volume.jinja ++++ b/salt/templates/virt/libvirt_volume.jinja +@@ -1,17 +1,39 @@ +- +- {{ name }}/{{ filename }} +- {{ name }}/{{ volname }} ++ ++ {{ name }} + + + {{ size }} +- 0 ++ {{ allocation }} + +- {{ pool }}{{ name }}/{{ filename }} +- ++ {%- if format %} ++ ++ {%- endif %} ++ {%- if target.permissions -%} + +- 00 +- 0 +- 0 ++ {%- if target.permissions.get('mode') %} ++ {{ target.permissions.mode }} ++ {%- endif %} ++ {%- if target.permissions.get('owner') %} ++ {{ target.permissions.owner }} ++ {%- endif %} ++ {%- if target.permissions.get('group') %} ++ {{ target.permissions.group }} ++ {%- endif %} ++ {%- if target.permissions.get('label') %} ++ ++ {%- endif %} + ++ {%- endif %} ++ {%- if target.nocow %} ++ ++ {%- endif %} + ++ {%- if backingStore %} ++ ++ {{ backingStore.path }} ++ {%- if backingStore.format %} ++ ++ {%- endif %} ++ ++ {%- endif %} + +diff --git a/tests/unit/modules/test_virt.py b/tests/unit/modules/test_virt.py +index 3e9bd5ef49..d3988464f6 100644 +--- a/tests/unit/modules/test_virt.py ++++ b/tests/unit/modules/test_virt.py +@@ -1,865 +1,1238 @@ + # -*- coding: utf-8 -*- +-''' ++""" + virt execution module unit tests +-''' ++""" + + # pylint: disable=3rd-party-module-not-gated + + # Import python libs + from __future__ import absolute_import, print_function, unicode_literals +-import os +-import re ++ + import datetime ++import os + import shutil ++import tempfile + +-# Import Salt Testing libs +-from tests.support.mixins import LoaderModuleMockMixin +-from tests.support.unit import TestCase +-from tests.support.mock import MagicMock, patch ++import salt.config ++import salt.modules.config as config ++import salt.modules.virt as virt ++import salt.syspaths + + # Import salt libs + import salt.utils.yaml +-import salt.modules.virt as virt +-import salt.modules.config as config + from salt._compat import ElementTree as ET +-import salt.config +-import salt.syspaths +-import tempfile +-from salt.exceptions import CommandExecutionError ++from salt.exceptions import CommandExecutionError, SaltInvocationError + + # Import third party libs + from salt.ext import six ++ + # pylint: disable=import-error + from salt.ext.six.moves import range # pylint: disable=redefined-builtin + ++# Import Salt Testing libs ++from tests.support.mixins import LoaderModuleMockMixin ++from tests.support.mock import MagicMock, patch ++from tests.support.unit import TestCase ++ + # pylint: disable=invalid-name,protected-access,attribute-defined-outside-init,too-many-public-methods,unused-argument + + + class LibvirtMock(MagicMock): # pylint: disable=too-many-ancestors +- ''' ++ """ + Libvirt library mock +- ''' ++ """ ++ + class virDomain(MagicMock): +- ''' ++ """ + virDomain mock +- ''' ++ """ + + class libvirtError(Exception): +- ''' ++ """ + libvirtError mock +- ''' ++ """ ++ ++ def __init__(self, msg): ++ super().__init__(msg) ++ self.msg = msg ++ ++ def get_error_message(self): ++ return self.msg + + + class VirtTestCase(TestCase, LoaderModuleMockMixin): +- ''' ++ """ + Test cases for salt.module.virt +- ''' ++ """ + + def setup_loader_modules(self): + self.mock_libvirt = LibvirtMock() + self.mock_conn = MagicMock() ++ self.mock_conn.getStoragePoolCapabilities.return_value = ( ++ "" ++ ) + self.mock_libvirt.openAuth.return_value = self.mock_conn + self.mock_popen = MagicMock() +- self.addCleanup(delattr, self, 'mock_libvirt') +- self.addCleanup(delattr, self, 'mock_conn') +- self.addCleanup(delattr, self, 'mock_popen') ++ self.addCleanup(delattr, self, "mock_libvirt") ++ self.addCleanup(delattr, self, "mock_conn") ++ self.addCleanup(delattr, self, "mock_popen") + self.mock_subprocess = MagicMock() +- self.mock_subprocess.return_value = self.mock_subprocess # pylint: disable=no-member +- self.mock_subprocess.Popen.return_value = self.mock_popen # pylint: disable=no-member ++ self.mock_subprocess.return_value = ( ++ self.mock_subprocess ++ ) # pylint: disable=no-member ++ self.mock_subprocess.Popen.return_value = ( ++ self.mock_popen ++ ) # pylint: disable=no-member + loader_globals = { +- '__salt__': { +- 'config.get': config.get, +- 'config.option': config.option, +- }, +- 'libvirt': self.mock_libvirt, +- 'subprocess': self.mock_subprocess ++ "__salt__": {"config.get": config.get, "config.option": config.option}, ++ "libvirt": self.mock_libvirt, ++ "subprocess": self.mock_subprocess, + } + return {virt: loader_globals, config: loader_globals} + + def set_mock_vm(self, name, xml): +- ''' ++ """ + Define VM to use in tests +- ''' +- self.mock_conn.listDefinedDomains.return_value = [name] # pylint: disable=no-member ++ """ ++ self.mock_conn.listDefinedDomains.return_value = [ ++ name ++ ] # pylint: disable=no-member + mock_domain = self.mock_libvirt.virDomain() +- self.mock_conn.lookupByName.return_value = mock_domain # pylint: disable=no-member ++ self.mock_conn.lookupByName.return_value = ( ++ mock_domain # pylint: disable=no-member ++ ) + mock_domain.XMLDesc.return_value = xml # pylint: disable=no-member + + # Return state as shutdown +- mock_domain.info.return_value = [4, 2048 * 1024, 1024 * 1024, 2, 1234] # pylint: disable=no-member ++ mock_domain.info.return_value = [ ++ 4, ++ 2048 * 1024, ++ 1024 * 1024, ++ 2, ++ 1234, ++ ] # pylint: disable=no-member + mock_domain.ID.return_value = 1 + mock_domain.name.return_value = name + return mock_domain + + def test_disk_profile_merge(self): +- ''' ++ """ + Test virt._disk_profile() when merging with user-defined disks +- ''' +- root_dir = os.path.join(salt.syspaths.ROOT_DIR, 'srv', 'salt-images') +- userdisks = [{'name': 'data', 'size': 16384, 'format': 'raw'}] +- +- disks = virt._disk_profile('default', 'kvm', userdisks, 'myvm', image='/path/to/image') +- self.assertEqual( +- [{'name': 'system', +- 'device': 'disk', +- 'size': 8192, +- 'format': 'qcow2', +- 'model': 'virtio', +- 'filename': 'myvm_system.qcow2', +- 'image': '/path/to/image', +- 'source_file': '{0}{1}myvm_system.qcow2'.format(root_dir, os.sep)}, +- {'name': 'data', +- 'device': 'disk', +- 'size': 16384, +- 'format': 'raw', +- 'model': 'virtio', +- 'filename': 'myvm_data.raw', +- 'source_file': '{0}{1}myvm_data.raw'.format(root_dir, os.sep)}], +- disks ++ """ ++ root_dir = os.path.join(salt.syspaths.ROOT_DIR, "srv", "salt-images") ++ userdisks = [ ++ {"name": "system", "image": "/path/to/image"}, ++ {"name": "data", "size": 16384, "format": "raw"}, ++ ] ++ ++ disks = virt._disk_profile(self.mock_conn, "default", "kvm", userdisks, "myvm") ++ self.assertEqual( ++ [ ++ { ++ "name": "system", ++ "device": "disk", ++ "size": 8192, ++ "format": "qcow2", ++ "model": "virtio", ++ "filename": "myvm_system.qcow2", ++ "image": "/path/to/image", ++ "source_file": "{0}{1}myvm_system.qcow2".format(root_dir, os.sep), ++ }, ++ { ++ "name": "data", ++ "device": "disk", ++ "size": 16384, ++ "format": "raw", ++ "model": "virtio", ++ "filename": "myvm_data.raw", ++ "source_file": "{0}{1}myvm_data.raw".format(root_dir, os.sep), ++ }, ++ ], ++ disks, + ) + + def test_boot_default_dev(self): +- ''' ++ """ + Test virt._gen_xml() default boot device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64' +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64" ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('os/boot').attrib['dev'], 'hd') +- self.assertEqual(root.find('os/type').attrib['arch'], 'x86_64') +- self.assertEqual(root.find('os/type').text, 'hvm') ++ self.assertEqual(root.find("os/boot").attrib["dev"], "hd") ++ self.assertEqual(root.find("os/type").attrib["arch"], "x86_64") ++ self.assertEqual(root.find("os/type").text, "hvm") + + def test_boot_custom_dev(self): +- ''' ++ """ + Test virt._gen_xml() custom boot device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- boot_dev='cdrom' +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ boot_dev="cdrom", ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('os/boot').attrib['dev'], 'cdrom') ++ self.assertEqual(root.find("os/boot").attrib["dev"], "cdrom") + + def test_boot_multiple_devs(self): +- ''' ++ """ + Test virt._gen_xml() multiple boot devices +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- boot_dev='cdrom network' +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ boot_dev="cdrom network", ++ ) + root = ET.fromstring(xml_data) +- devs = root.findall('.//boot') ++ devs = root.findall(".//boot") + self.assertTrue(len(devs) == 2) + + def test_gen_xml_no_nic(self): +- ''' ++ """ + Test virt._gen_xml() serial console +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='pty', +- console=True +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="pty", ++ console=True, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'pty') +- self.assertEqual(root.find('devices/console').attrib['type'], 'pty') ++ self.assertEqual(root.find("devices/serial").attrib["type"], "pty") ++ self.assertEqual(root.find("devices/console").attrib["type"], "pty") + + def test_gen_xml_for_serial_console(self): +- ''' ++ """ + Test virt._gen_xml() serial console +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='pty', +- console=True +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="pty", ++ console=True, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'pty') +- self.assertEqual(root.find('devices/console').attrib['type'], 'pty') ++ self.assertEqual(root.find("devices/serial").attrib["type"], "pty") ++ self.assertEqual(root.find("devices/console").attrib["type"], "pty") + + def test_gen_xml_for_telnet_console(self): +- ''' ++ """ + Test virt._gen_xml() telnet console +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='tcp', ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="tcp", + console=True, +- telnet_port=22223 +- ) ++ telnet_port=22223, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'tcp') +- self.assertEqual(root.find('devices/console').attrib['type'], 'tcp') +- self.assertEqual(root.find('devices/console/source').attrib['service'], '22223') ++ self.assertEqual(root.find("devices/serial").attrib["type"], "tcp") ++ self.assertEqual(root.find("devices/console").attrib["type"], "tcp") ++ self.assertEqual(root.find("devices/console/source").attrib["service"], "22223") + + def test_gen_xml_for_telnet_console_unspecified_port(self): +- ''' ++ """ + Test virt._gen_xml() telnet console without any specified port +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='tcp', +- console=True +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="tcp", ++ console=True, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'tcp') +- self.assertEqual(root.find('devices/console').attrib['type'], 'tcp') +- self.assertIsInstance(int(root.find('devices/console/source').attrib['service']), int) ++ self.assertEqual(root.find("devices/serial").attrib["type"], "tcp") ++ self.assertEqual(root.find("devices/console").attrib["type"], "tcp") ++ self.assertIsInstance( ++ int(root.find("devices/console/source").attrib["service"]), int ++ ) + + def test_gen_xml_for_serial_no_console(self): +- ''' ++ """ + Test virt._gen_xml() with no serial console +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='pty', +- console=False +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="pty", ++ console=False, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'pty') +- self.assertEqual(root.find('devices/console'), None) ++ self.assertEqual(root.find("devices/serial").attrib["type"], "pty") ++ self.assertEqual(root.find("devices/console"), None) + + def test_gen_xml_for_telnet_no_console(self): +- ''' ++ """ + Test virt._gen_xml() with no telnet console +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- serial_type='tcp', ++ "kvm", ++ "hvm", ++ "x86_64", ++ serial_type="tcp", + console=False, +- ) ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/serial').attrib['type'], 'tcp') +- self.assertEqual(root.find('devices/console'), None) ++ self.assertEqual(root.find("devices/serial").attrib["type"], "tcp") ++ self.assertEqual(root.find("devices/console"), None) + + def test_gen_xml_nographics_default(self): +- ''' ++ """ + Test virt._gen_xml() with default no graphics device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64' +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64" ++ ) ++ root = ET.fromstring(xml_data) ++ self.assertIsNone(root.find("devices/graphics")) ++ ++ def test_gen_xml_noloader_default(self): ++ """ ++ Test virt._gen_xml() with default no loader ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") ++ xml_data = virt._gen_xml( ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64" ++ ) + root = ET.fromstring(xml_data) +- self.assertIsNone(root.find('devices/graphics')) ++ self.assertIsNone(root.find("os/loader")) + + def test_gen_xml_vnc_default(self): +- ''' ++ """ + Test virt._gen_xml() with default vnc graphics device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- graphics={'type': 'vnc', 'port': 1234, 'tlsPort': 5678, +- 'listen': {'type': 'address', 'address': 'myhost'}}, +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ graphics={ ++ "type": "vnc", ++ "port": 1234, ++ "tlsPort": 5678, ++ "listen": {"type": "address", "address": "myhost"}, ++ }, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/graphics').attrib['type'], 'vnc') +- self.assertEqual(root.find('devices/graphics').attrib['autoport'], 'no') +- self.assertEqual(root.find('devices/graphics').attrib['port'], '1234') +- self.assertFalse('tlsPort' in root.find('devices/graphics').attrib) +- self.assertEqual(root.find('devices/graphics').attrib['listen'], 'myhost') +- self.assertEqual(root.find('devices/graphics/listen').attrib['type'], 'address') +- self.assertEqual(root.find('devices/graphics/listen').attrib['address'], 'myhost') ++ self.assertEqual(root.find("devices/graphics").attrib["type"], "vnc") ++ self.assertEqual(root.find("devices/graphics").attrib["autoport"], "no") ++ self.assertEqual(root.find("devices/graphics").attrib["port"], "1234") ++ self.assertFalse("tlsPort" in root.find("devices/graphics").attrib) ++ self.assertEqual(root.find("devices/graphics").attrib["listen"], "myhost") ++ self.assertEqual(root.find("devices/graphics/listen").attrib["type"], "address") ++ self.assertEqual( ++ root.find("devices/graphics/listen").attrib["address"], "myhost" ++ ) + + def test_gen_xml_spice_default(self): +- ''' ++ """ + Test virt._gen_xml() with default spice graphics device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- graphics={'type': 'spice'}, +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ graphics={"type": "spice"}, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/graphics').attrib['type'], 'spice') +- self.assertEqual(root.find('devices/graphics').attrib['autoport'], 'yes') +- self.assertEqual(root.find('devices/graphics').attrib['listen'], '0.0.0.0') +- self.assertEqual(root.find('devices/graphics/listen').attrib['type'], 'address') +- self.assertEqual(root.find('devices/graphics/listen').attrib['address'], '0.0.0.0') ++ self.assertEqual(root.find("devices/graphics").attrib["type"], "spice") ++ self.assertEqual(root.find("devices/graphics").attrib["autoport"], "yes") ++ self.assertEqual(root.find("devices/graphics").attrib["listen"], "0.0.0.0") ++ self.assertEqual(root.find("devices/graphics/listen").attrib["type"], "address") ++ self.assertEqual( ++ root.find("devices/graphics/listen").attrib["address"], "0.0.0.0" ++ ) + + def test_gen_xml_spice(self): +- ''' ++ """ + Test virt._gen_xml() with spice graphics device +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- graphics={'type': 'spice', 'port': 1234, 'tls_port': 5678, 'listen': {'type': 'none'}}, +- ) ++ "kvm", ++ "hvm", ++ "x86_64", ++ graphics={ ++ "type": "spice", ++ "port": 1234, ++ "tls_port": 5678, ++ "listen": {"type": "none"}, ++ }, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('devices/graphics').attrib['type'], 'spice') +- self.assertEqual(root.find('devices/graphics').attrib['autoport'], 'no') +- self.assertEqual(root.find('devices/graphics').attrib['port'], '1234') +- self.assertEqual(root.find('devices/graphics').attrib['tlsPort'], '5678') +- self.assertFalse('listen' in root.find('devices/graphics').attrib) +- self.assertEqual(root.find('devices/graphics/listen').attrib['type'], 'none') +- self.assertFalse('address' in root.find('devices/graphics/listen').attrib) ++ self.assertEqual(root.find("devices/graphics").attrib["type"], "spice") ++ self.assertEqual(root.find("devices/graphics").attrib["autoport"], "no") ++ self.assertEqual(root.find("devices/graphics").attrib["port"], "1234") ++ self.assertEqual(root.find("devices/graphics").attrib["tlsPort"], "5678") ++ self.assertFalse("listen" in root.find("devices/graphics").attrib) ++ self.assertEqual(root.find("devices/graphics/listen").attrib["type"], "none") ++ self.assertFalse("address" in root.find("devices/graphics/listen").attrib) + + def test_default_disk_profile_hypervisor_esxi(self): +- ''' ++ """ + Test virt._disk_profile() default ESXi profile +- ''' ++ """ + mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._disk_profile('nonexistent', 'vmware') ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._disk_profile( ++ self.mock_conn, "nonexistent", "vmware", None, "test-vm" ++ ) + self.assertTrue(len(ret) == 1) +- found = [disk for disk in ret if disk['name'] == 'system'] ++ found = [disk for disk in ret if disk["name"] == "system"] + self.assertTrue(bool(found)) + system = found[0] +- self.assertEqual(system['format'], 'vmdk') +- self.assertEqual(system['model'], 'scsi') +- self.assertTrue(int(system['size']) >= 1) ++ self.assertEqual(system["format"], "vmdk") ++ self.assertEqual(system["model"], "scsi") ++ self.assertTrue(int(system["size"]) >= 1) + + def test_default_disk_profile_hypervisor_kvm(self): +- ''' ++ """ + Test virt._disk_profile() default KVM profile +- ''' +- mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._disk_profile('nonexistent', 'kvm') ++ """ ++ mock = MagicMock(side_effect=[{}, "/images/dir"]) ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._disk_profile( ++ self.mock_conn, "nonexistent", "kvm", None, "test-vm" ++ ) + self.assertTrue(len(ret) == 1) +- found = [disk for disk in ret if disk['name'] == 'system'] ++ found = [disk for disk in ret if disk["name"] == "system"] + self.assertTrue(bool(found)) + system = found[0] +- self.assertEqual(system['format'], 'qcow2') +- self.assertEqual(system['model'], 'virtio') +- self.assertTrue(int(system['size']) >= 1) ++ self.assertEqual(system["format"], "qcow2") ++ self.assertEqual(system["model"], "virtio") ++ self.assertTrue(int(system["size"]) >= 1) + + def test_default_disk_profile_hypervisor_xen(self): +- ''' ++ """ + Test virt._disk_profile() default XEN profile +- ''' +- mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._disk_profile('nonexistent', 'xen') ++ """ ++ mock = MagicMock(side_effect=[{}, "/images/dir"]) ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._disk_profile( ++ self.mock_conn, "nonexistent", "xen", None, "test-vm" ++ ) + self.assertTrue(len(ret) == 1) +- found = [disk for disk in ret if disk['name'] == 'system'] ++ found = [disk for disk in ret if disk["name"] == "system"] + self.assertTrue(bool(found)) + system = found[0] +- self.assertEqual(system['format'], 'qcow2') +- self.assertEqual(system['model'], 'xen') +- self.assertTrue(int(system['size']) >= 1) ++ self.assertEqual(system["format"], "qcow2") ++ self.assertEqual(system["model"], "xen") ++ self.assertTrue(int(system["size"]) >= 1) + + def test_default_nic_profile_hypervisor_esxi(self): +- ''' ++ """ + Test virt._nic_profile() default ESXi profile +- ''' ++ """ + mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._nic_profile('nonexistent', 'vmware') ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._nic_profile("nonexistent", "vmware") + self.assertTrue(len(ret) == 1) + eth0 = ret[0] +- self.assertEqual(eth0['name'], 'eth0') +- self.assertEqual(eth0['type'], 'bridge') +- self.assertEqual(eth0['source'], 'DEFAULT') +- self.assertEqual(eth0['model'], 'e1000') ++ self.assertEqual(eth0["name"], "eth0") ++ self.assertEqual(eth0["type"], "bridge") ++ self.assertEqual(eth0["source"], "DEFAULT") ++ self.assertEqual(eth0["model"], "e1000") + + def test_default_nic_profile_hypervisor_kvm(self): +- ''' ++ """ + Test virt._nic_profile() default KVM profile +- ''' ++ """ + mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._nic_profile('nonexistent', 'kvm') ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._nic_profile("nonexistent", "kvm") + self.assertTrue(len(ret) == 1) + eth0 = ret[0] +- self.assertEqual(eth0['name'], 'eth0') +- self.assertEqual(eth0['type'], 'bridge') +- self.assertEqual(eth0['source'], 'br0') +- self.assertEqual(eth0['model'], 'virtio') ++ self.assertEqual(eth0["name"], "eth0") ++ self.assertEqual(eth0["type"], "bridge") ++ self.assertEqual(eth0["source"], "br0") ++ self.assertEqual(eth0["model"], "virtio") + + def test_default_nic_profile_hypervisor_xen(self): +- ''' ++ """ + Test virt._nic_profile() default XEN profile +- ''' ++ """ + mock = MagicMock(return_value={}) +- with patch.dict(virt.__salt__, {'config.get': mock}): # pylint: disable=no-member +- ret = virt._nic_profile('nonexistent', 'xen') ++ with patch.dict( ++ virt.__salt__, {"config.get": mock} # pylint: disable=no-member ++ ): ++ ret = virt._nic_profile("nonexistent", "xen") + self.assertTrue(len(ret) == 1) + eth0 = ret[0] +- self.assertEqual(eth0['name'], 'eth0') +- self.assertEqual(eth0['type'], 'bridge') +- self.assertEqual(eth0['source'], 'br0') +- self.assertFalse(eth0['model']) +- +- def test_gen_vol_xml(self): +- ''' +- Test virt._get_vol_xml() +- ''' +- xml_data = virt._gen_vol_xml('vmname', 'system', 'qcow2', 8192, '/path/to/image/') ++ self.assertEqual(eth0["name"], "eth0") ++ self.assertEqual(eth0["type"], "bridge") ++ self.assertEqual(eth0["source"], "br0") ++ self.assertFalse(eth0["model"]) ++ ++ def test_gen_vol_xml_esx(self): ++ """ ++ Test virt._get_vol_xml() for the ESX case ++ """ ++ xml_data = virt._gen_vol_xml("vmname/system.vmdk", 8192, format="vmdk") ++ root = ET.fromstring(xml_data) ++ self.assertIsNone(root.get("type")) ++ self.assertEqual(root.find("name").text, "vmname/system.vmdk") ++ self.assertEqual(root.find("capacity").attrib["unit"], "KiB") ++ self.assertEqual(root.find("capacity").text, six.text_type(8192 * 1024)) ++ self.assertEqual(root.find("allocation").text, six.text_type(0)) ++ self.assertEqual(root.find("target/format").get("type"), "vmdk") ++ self.assertIsNone(root.find("target/permissions")) ++ self.assertIsNone(root.find("target/nocow")) ++ self.assertIsNone(root.find("backingStore")) ++ ++ def test_gen_vol_xml_file(self): ++ """ ++ Test virt._get_vol_xml() for a file volume ++ """ ++ xml_data = virt._gen_vol_xml( ++ "myvm_system.qcow2", ++ 8192, ++ format="qcow2", ++ allocation=4096, ++ type="file", ++ permissions={ ++ "mode": "0775", ++ "owner": "123", ++ "group": "456", ++ "label": "sec_label", ++ }, ++ backing_store={"path": "/backing/image", "format": "raw"}, ++ nocow=True, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.find('name').text, 'vmname/system.qcow2') +- self.assertEqual(root.find('key').text, 'vmname/system') +- self.assertEqual(root.find('capacity').attrib['unit'], 'KiB') +- self.assertEqual(root.find('capacity').text, six.text_type(8192 * 1024)) ++ self.assertEqual(root.get("type"), "file") ++ self.assertEqual(root.find("name").text, "myvm_system.qcow2") ++ self.assertIsNone(root.find("key")) ++ self.assertIsNone(root.find("target/path")) ++ self.assertEqual(root.find("target/format").get("type"), "qcow2") ++ self.assertEqual(root.find("capacity").attrib["unit"], "KiB") ++ self.assertEqual(root.find("capacity").text, six.text_type(8192 * 1024)) ++ self.assertEqual(root.find("capacity").attrib["unit"], "KiB") ++ self.assertEqual(root.find("allocation").text, six.text_type(4096 * 1024)) ++ self.assertEqual(root.find("target/permissions/mode").text, "0775") ++ self.assertEqual(root.find("target/permissions/owner").text, "123") ++ self.assertEqual(root.find("target/permissions/group").text, "456") ++ self.assertEqual(root.find("target/permissions/label").text, "sec_label") ++ self.assertIsNotNone(root.find("target/nocow")) ++ self.assertEqual(root.find("backingStore/path").text, "/backing/image") ++ self.assertEqual(root.find("backingStore/format").get("type"), "raw") + + def test_gen_xml_for_kvm_default_profile(self): +- ''' ++ """ + Test virt._gen_xml(), KVM default profile case +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.attrib['type'], 'kvm') +- self.assertEqual(root.find('vcpu').text, '1') +- self.assertEqual(root.find('memory').text, six.text_type(512 * 1024)) +- self.assertEqual(root.find('memory').attrib['unit'], 'KiB') ++ self.assertEqual(root.attrib["type"], "kvm") ++ self.assertEqual(root.find("vcpu").text, "1") ++ self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").attrib["unit"], "KiB") + +- disks = root.findall('.//disk') ++ disks = root.findall(".//disk") + self.assertEqual(len(disks), 1) + disk = disks[0] +- root_dir = salt.config.DEFAULT_MINION_OPTS.get('root_dir') +- self.assertTrue(disk.find('source').attrib['file'].startswith(root_dir)) +- self.assertTrue('hello_system' in disk.find('source').attrib['file']) +- self.assertEqual(disk.find('target').attrib['dev'], 'vda') +- self.assertEqual(disk.find('target').attrib['bus'], 'virtio') +- self.assertEqual(disk.find('driver').attrib['name'], 'qemu') +- self.assertEqual(disk.find('driver').attrib['type'], 'qcow2') +- +- interfaces = root.findall('.//interface') ++ root_dir = salt.config.DEFAULT_MINION_OPTS.get("root_dir") ++ self.assertTrue(disk.find("source").attrib["file"].startswith(root_dir)) ++ self.assertTrue("hello_system" in disk.find("source").attrib["file"]) ++ self.assertEqual(disk.find("target").attrib["dev"], "vda") ++ self.assertEqual(disk.find("target").attrib["bus"], "virtio") ++ self.assertEqual(disk.find("driver").attrib["name"], "qemu") ++ self.assertEqual(disk.find("driver").attrib["type"], "qcow2") ++ ++ interfaces = root.findall(".//interface") + self.assertEqual(len(interfaces), 1) + iface = interfaces[0] +- self.assertEqual(iface.attrib['type'], 'bridge') +- self.assertEqual(iface.find('source').attrib['bridge'], 'br0') +- self.assertEqual(iface.find('model').attrib['type'], 'virtio') +- +- mac = iface.find('mac').attrib['address'] +- self.assertTrue( +- re.match('^([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$', mac, re.I)) ++ self.assertEqual(iface.attrib["type"], "bridge") ++ self.assertEqual(iface.find("source").attrib["bridge"], "br0") ++ self.assertEqual(iface.find("model").attrib["type"], "virtio") + + def test_gen_xml_for_esxi_default_profile(self): +- ''' ++ """ + Test virt._gen_xml(), ESXi/vmware default profile case +- ''' +- diskp = virt._disk_profile('default', 'vmware', [], 'hello') +- nicp = virt._nic_profile('default', 'vmware') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "vmware", [], "hello") ++ nicp = virt._nic_profile("default", "vmware") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'vmware', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "vmware", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.attrib['type'], 'vmware') +- self.assertEqual(root.find('vcpu').text, '1') +- self.assertEqual(root.find('memory').text, six.text_type(512 * 1024)) +- self.assertEqual(root.find('memory').attrib['unit'], 'KiB') ++ self.assertEqual(root.attrib["type"], "vmware") ++ self.assertEqual(root.find("vcpu").text, "1") ++ self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").attrib["unit"], "KiB") + +- disks = root.findall('.//disk') ++ disks = root.findall(".//disk") + self.assertEqual(len(disks), 1) + disk = disks[0] +- self.assertTrue('[0]' in disk.find('source').attrib['file']) +- self.assertTrue('hello_system' in disk.find('source').attrib['file']) +- self.assertEqual(disk.find('target').attrib['dev'], 'sda') +- self.assertEqual(disk.find('target').attrib['bus'], 'scsi') +- self.assertEqual(disk.find('address').attrib['unit'], '0') ++ self.assertTrue("[0]" in disk.find("source").attrib["file"]) ++ self.assertTrue("hello_system" in disk.find("source").attrib["file"]) ++ self.assertEqual(disk.find("target").attrib["dev"], "sda") ++ self.assertEqual(disk.find("target").attrib["bus"], "scsi") ++ self.assertEqual(disk.find("address").attrib["unit"], "0") + +- interfaces = root.findall('.//interface') ++ interfaces = root.findall(".//interface") + self.assertEqual(len(interfaces), 1) + iface = interfaces[0] +- self.assertEqual(iface.attrib['type'], 'bridge') +- self.assertEqual(iface.find('source').attrib['bridge'], 'DEFAULT') +- self.assertEqual(iface.find('model').attrib['type'], 'e1000') +- +- mac = iface.find('mac').attrib['address'] +- self.assertTrue( +- re.match('^([0-9A-F]{2}[:-]){5}([0-9A-F]{2})$', mac, re.I)) ++ self.assertEqual(iface.attrib["type"], "bridge") ++ self.assertEqual(iface.find("source").attrib["bridge"], "DEFAULT") ++ self.assertEqual(iface.find("model").attrib["type"], "e1000") + + def test_gen_xml_for_xen_default_profile(self): +- ''' ++ """ + Test virt._gen_xml(), XEN PV default profile case +- ''' +- diskp = virt._disk_profile('default', 'xen', [], 'hello') +- nicp = virt._nic_profile('default', 'xen') +- with patch.dict(virt.__grains__, {'os_family': 'Suse'}): ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "xen", [], "hello") ++ nicp = virt._nic_profile("default", "xen") ++ with patch.dict( ++ virt.__grains__, {"os_family": "Suse"} # pylint: disable=no-member ++ ): + xml_data = virt._gen_xml( +- 'hello', ++ self.mock_conn, ++ "hello", + 1, + 512, + diskp, + nicp, +- 'xen', +- 'xen', +- 'x86_64', +- boot=None +- ) ++ "xen", ++ "xen", ++ "x86_64", ++ boot=None, ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.attrib['type'], 'xen') +- self.assertEqual(root.find('vcpu').text, '1') +- self.assertEqual(root.find('memory').text, six.text_type(512 * 1024)) +- self.assertEqual(root.find('memory').attrib['unit'], 'KiB') +- self.assertEqual(root.find('.//kernel').text, '/usr/lib/grub2/x86_64-xen/grub.xen') ++ self.assertEqual(root.attrib["type"], "xen") ++ self.assertEqual(root.find("vcpu").text, "1") ++ self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").attrib["unit"], "KiB") ++ self.assertEqual( ++ root.find(".//kernel").text, "/usr/lib/grub2/x86_64-xen/grub.xen" ++ ) + +- disks = root.findall('.//disk') ++ disks = root.findall(".//disk") + self.assertEqual(len(disks), 1) + disk = disks[0] +- root_dir = salt.config.DEFAULT_MINION_OPTS.get('root_dir') +- self.assertTrue(disk.find('source').attrib['file'].startswith(root_dir)) +- self.assertTrue('hello_system' in disk.find('source').attrib['file']) +- self.assertEqual(disk.find('target').attrib['dev'], 'xvda') +- self.assertEqual(disk.find('target').attrib['bus'], 'xen') +- self.assertEqual(disk.find('driver').attrib['name'], 'qemu') +- self.assertEqual(disk.find('driver').attrib['type'], 'qcow2') +- +- interfaces = root.findall('.//interface') ++ root_dir = salt.config.DEFAULT_MINION_OPTS.get("root_dir") ++ self.assertTrue(disk.find("source").attrib["file"].startswith(root_dir)) ++ self.assertTrue("hello_system" in disk.find("source").attrib["file"]) ++ self.assertEqual(disk.find("target").attrib["dev"], "xvda") ++ self.assertEqual(disk.find("target").attrib["bus"], "xen") ++ self.assertEqual(disk.find("driver").attrib["name"], "qemu") ++ self.assertEqual(disk.find("driver").attrib["type"], "qcow2") ++ ++ interfaces = root.findall(".//interface") + self.assertEqual(len(interfaces), 1) + iface = interfaces[0] +- self.assertEqual(iface.attrib['type'], 'bridge') +- self.assertEqual(iface.find('source').attrib['bridge'], 'br0') +- self.assertIsNone(iface.find('model')) ++ self.assertEqual(iface.attrib["type"], "bridge") ++ self.assertEqual(iface.find("source").attrib["bridge"], "br0") ++ self.assertIsNone(iface.find("model")) + + def test_gen_xml_for_esxi_custom_profile(self): +- ''' ++ """ + Test virt._gen_xml(), ESXi/vmware custom profile case +- ''' ++ """ + disks = { +- 'noeffect': [ +- {'first': {'size': 8192, 'pool': 'datastore1'}}, +- {'second': {'size': 4096, 'pool': 'datastore2'}} ++ "noeffect": [ ++ {"first": {"size": 8192, "pool": "datastore1"}}, ++ {"second": {"size": 4096, "pool": "datastore2"}}, + ] + } + nics = { +- 'noeffect': [ +- {'name': 'eth1', 'source': 'ONENET'}, +- {'name': 'eth2', 'source': 'TWONET'} ++ "noeffect": [ ++ {"name": "eth1", "source": "ONENET"}, ++ {"name": "eth2", "source": "TWONET"}, + ] + } +- with patch.dict(virt.__salt__, # pylint: disable=no-member +- {'config.get': MagicMock(side_effect=[disks, nics])}): +- diskp = virt._disk_profile('noeffect', 'vmware', [], 'hello') +- nicp = virt._nic_profile('noeffect', 'vmware') ++ with patch.dict( ++ virt.__salt__, # pylint: disable=no-member ++ {"config.get": MagicMock(side_effect=[disks, nics])}, ++ ): ++ diskp = virt._disk_profile( ++ self.mock_conn, "noeffect", "vmware", [], "hello" ++ ) ++ nicp = virt._nic_profile("noeffect", "vmware") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'vmware', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "vmware", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.attrib['type'], 'vmware') +- self.assertEqual(root.find('vcpu').text, '1') +- self.assertEqual(root.find('memory').text, six.text_type(512 * 1024)) +- self.assertEqual(root.find('memory').attrib['unit'], 'KiB') +- self.assertTrue(len(root.findall('.//disk')) == 2) +- self.assertTrue(len(root.findall('.//interface')) == 2) ++ self.assertEqual(root.attrib["type"], "vmware") ++ self.assertEqual(root.find("vcpu").text, "1") ++ self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").attrib["unit"], "KiB") ++ self.assertTrue(len(root.findall(".//disk")) == 2) ++ self.assertTrue(len(root.findall(".//interface")) == 2) + + def test_gen_xml_for_kvm_custom_profile(self): +- ''' ++ """ + Test virt._gen_xml(), KVM custom profile case +- ''' ++ """ + disks = { +- 'noeffect': [ +- {'first': {'size': 8192, 'pool': '/var/lib/images'}}, +- {'second': {'size': 4096, 'pool': '/var/lib/images'}} ++ "noeffect": [ ++ {"first": {"size": 8192, "pool": "/var/lib/images"}}, ++ {"second": {"size": 4096, "pool": "/var/lib/images"}}, + ] + } + nics = { +- 'noeffect': [ +- {'name': 'eth1', 'source': 'b2'}, +- {'name': 'eth2', 'source': 'b2'} ++ "noeffect": [ ++ {"name": "eth1", "source": "b2"}, ++ {"name": "eth2", "source": "b2"}, + ] + } +- with patch.dict(virt.__salt__, {'config.get': MagicMock(side_effect=[ # pylint: disable=no-member +- disks, nics])}): +- diskp = virt._disk_profile('noeffect', 'kvm', [], 'hello') +- nicp = virt._nic_profile('noeffect', 'kvm') ++ with patch.dict( ++ virt.__salt__, # pylint: disable=no-member ++ {"config.get": MagicMock(side_effect=[disks, nics])}, ++ ): ++ diskp = virt._disk_profile(self.mock_conn, "noeffect", "kvm", [], "hello") ++ nicp = virt._nic_profile("noeffect", "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- self.assertEqual(root.attrib['type'], 'kvm') +- self.assertEqual(root.find('vcpu').text, '1') +- self.assertEqual(root.find('memory').text, six.text_type(512 * 1024)) +- self.assertEqual(root.find('memory').attrib['unit'], 'KiB') +- self.assertTrue(len(root.findall('.//disk')) == 2) +- self.assertTrue(len(root.findall('.//interface')) == 2) +- +- @patch('salt.modules.virt.pool_info', +- return_value={'mypool': {'target_path': os.path.join(salt.syspaths.ROOT_DIR, 'pools', 'mypool')}}) +- def test_disk_profile_kvm_disk_pool(self, mock_poolinfo): +- ''' +- Test virt._gen_xml(), KVM case with pools defined. +- ''' ++ self.assertEqual(root.attrib["type"], "kvm") ++ self.assertEqual(root.find("vcpu").text, "1") ++ self.assertEqual(root.find("memory").text, six.text_type(512 * 1024)) ++ self.assertEqual(root.find("memory").attrib["unit"], "KiB") ++ disks = root.findall(".//disk") ++ self.assertTrue(len(disks) == 2) ++ self.assertEqual(disks[0].find("target").get("dev"), "vda") ++ self.assertEqual(disks[1].find("target").get("dev"), "vdb") ++ self.assertTrue(len(root.findall(".//interface")) == 2) ++ ++ def test_disk_profile_kvm_disk_pool(self): ++ """ ++ Test virt._disk_profile(), KVM case with pools defined. ++ """ + disks = { +- 'noeffect': [ +- {'first': {'size': 8192, 'pool': 'mypool'}}, +- {'second': {'size': 4096}} ++ "noeffect": [ ++ {"first": {"size": 8192, "pool": "mypool"}}, ++ {"second": {"size": 4096}}, + ] + } + + # pylint: disable=no-member +- with patch.dict(virt.__salt__, {'config.get': MagicMock(side_effect=[ +- disks, +- os.path.join(salt.syspaths.ROOT_DIR, 'default', 'path')])}): ++ with patch.dict( ++ virt.__salt__, ++ { ++ "config.get": MagicMock( ++ side_effect=[ ++ disks, ++ os.path.join(salt.syspaths.ROOT_DIR, "default", "path"), ++ ] ++ ) ++ }, ++ ): + +- diskp = virt._disk_profile('noeffect', 'kvm', [], 'hello') ++ diskp = virt._disk_profile(self.mock_conn, "noeffect", "kvm", [], "hello") + +- pools_path = os.path.join(salt.syspaths.ROOT_DIR, 'pools', 'mypool') + os.sep +- default_path = os.path.join(salt.syspaths.ROOT_DIR, 'default', 'path') + os.sep ++ pools_path = ( ++ os.path.join(salt.syspaths.ROOT_DIR, "pools", "mypool") + os.sep ++ ) ++ default_path = ( ++ os.path.join(salt.syspaths.ROOT_DIR, "default", "path") + os.sep ++ ) + + self.assertEqual(len(diskp), 2) +- self.assertTrue(diskp[0]['source_file'].startswith(pools_path)) +- self.assertTrue(diskp[1]['source_file'].startswith(default_path)) ++ self.assertTrue(diskp[1]["source_file"].startswith(default_path)) + # pylint: enable=no-member + + def test_disk_profile_kvm_disk_external_image(self): +- ''' ++ """ + Test virt._gen_xml(), KVM case with an external image. +- ''' +- diskp = virt._disk_profile(None, 'kvm', [ ++ """ ++ with patch.dict(os.path.__dict__, {"exists": MagicMock(return_value=True)}): ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "mydisk", "source_file": "/path/to/my/image.qcow2"}], ++ "hello", ++ ) ++ ++ self.assertEqual(len(diskp), 1) ++ self.assertEqual(diskp[0]["source_file"], ("/path/to/my/image.qcow2")) ++ ++ def test_disk_profile_cdrom_default(self): ++ """ ++ Test virt._gen_xml(), KVM case with cdrom. ++ """ ++ with patch.dict(os.path.__dict__, {"exists": MagicMock(return_value=True)}): ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [ ++ { ++ "name": "mydisk", ++ "device": "cdrom", ++ "source_file": "/path/to/my.iso", ++ } ++ ], ++ "hello", ++ ) ++ ++ self.assertEqual(len(diskp), 1) ++ self.assertEqual(diskp[0]["model"], "ide") ++ self.assertEqual(diskp[0]["format"], "raw") ++ ++ def test_disk_profile_pool_disk_type(self): ++ """ ++ Test virt._disk_profile(), with a disk pool of disk type ++ """ ++ self.mock_conn.listStoragePools.return_value = ["test-vdb"] ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = """ ++ ++ test-vdb ++ ++ ++ ++ ++ /dev ++ ++ ++ """ ++ ++ # No existing disk case ++ self.mock_conn.storagePoolLookupByName.return_value.listVolumes.return_value = ( ++ [] ++ ) ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "mydisk", "pool": "test-vdb"}], ++ "hello", ++ ) ++ self.assertEqual(diskp[0]["filename"], ("vdb1")) ++ ++ # Append to the end case ++ self.mock_conn.storagePoolLookupByName.return_value.listVolumes.return_value = [ ++ "vdb1", ++ "vdb2", ++ ] ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "mydisk", "pool": "test-vdb"}], ++ "hello", ++ ) ++ self.assertEqual(diskp[0]["filename"], ("vdb3")) ++ ++ # Hole in the middle case ++ self.mock_conn.storagePoolLookupByName.return_value.listVolumes.return_value = [ ++ "vdb1", ++ "vdb3", ++ ] ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "mydisk", "pool": "test-vdb"}], ++ "hello", ++ ) ++ self.assertEqual(diskp[0]["filename"], ("vdb2")) ++ ++ # Reuse existing volume case ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "mydisk", "pool": "test-vdb", "source_file": "vdb1"}], ++ "hello", ++ ) ++ self.assertEqual(diskp[0]["filename"], ("vdb1")) ++ ++ def test_gen_xml_volume(self): ++ """ ++ Test virt._gen_xml(), generating a disk of volume type ++ """ ++ self.mock_conn.listStoragePools.return_value = ["default"] ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = ( ++ "" ++ ) ++ self.mock_conn.storagePoolLookupByName.return_value.listVolumes.return_value = [ ++ "myvolume" ++ ] ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [ ++ {"name": "system", "pool": "default"}, ++ {"name": "data", "pool": "default", "source_file": "myvolume"}, ++ ], ++ "hello", ++ ) ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = ( ++ "" ++ ) ++ nicp = virt._nic_profile(None, "kvm") ++ xml_data = virt._gen_xml( ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) ++ root = ET.fromstring(xml_data) ++ disk = root.findall(".//disk")[0] ++ self.assertEqual(disk.attrib["device"], "disk") ++ self.assertEqual(disk.attrib["type"], "volume") ++ source = disk.find("source") ++ self.assertEqual("default", source.attrib["pool"]) ++ self.assertEqual("hello_system", source.attrib["volume"]) ++ self.assertEqual("myvolume", root.find(".//disk[2]/source").get("volume")) ++ ++ # RBD volume usage auth test case ++ self.mock_conn.listStoragePools.return_value = ["test-rbd"] ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = """ ++ ++ test-rbd ++ ede33e0a-9df0-479f-8afd-55085a01b244 ++ 526133493760 ++ 589928 ++ 515081306112 ++ ++ ++ ++ libvirt-pool ++ ++ ++ ++ ++ ++ """ ++ self.mock_conn.getStoragePoolCapabilities.return_value = """ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ """ ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "system", "pool": "test-rbd"}], ++ "test-vm", ++ ) ++ xml_data = virt._gen_xml( ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) ++ root = ET.fromstring(xml_data) ++ disk = root.findall(".//disk")[0] ++ self.assertDictEqual( + { +- 'name': 'mydisk', +- 'source_file': '/path/to/my/image.qcow2' +- }], 'hello') +- +- self.assertEqual(len(diskp), 1) +- self.assertEqual(diskp[0]['source_file'], ('/path/to/my/image.qcow2')) +- +- @patch('salt.modules.virt.pool_info', return_value={}) +- def test_disk_profile_kvm_disk_pool_notfound(self, mock_poolinfo): +- ''' +- Test virt._gen_xml(), KVM case with pools defined. +- ''' +- disks = { +- 'noeffect': [ +- {'first': {'size': 8192, 'pool': 'default'}}, +- ] +- } +- with patch.dict(virt.__salt__, {'config.get': MagicMock(side_effect=[ # pylint: disable=no-member +- disks, "/default/path/"])}): +- with self.assertRaises(CommandExecutionError): +- virt._disk_profile('noeffect', 'kvm', [], 'hello') +- +- @patch('salt.modules.virt.pool_info', return_value={'target_path': '/dev/disk/by-path'}) +- def test_disk_profile_kvm_disk_pool_invalid(self, mock_poolinfo): +- ''' +- Test virt._gen_xml(), KVM case with pools defined. +- ''' +- disks = { +- 'noeffect': [ +- {'first': {'size': 8192, 'pool': 'default'}}, +- ] +- } +- with patch.dict(virt.__salt__, {'config.get': MagicMock(side_effect=[ # pylint: disable=no-member +- disks, "/default/path/"])}): +- with self.assertRaises(CommandExecutionError): +- virt._disk_profile('noeffect', 'kvm', [], 'hello') ++ "type": "network", ++ "device": "disk", ++ "source": { ++ "protocol": "rbd", ++ "name": "libvirt-pool/test-vm_system", ++ "host": [ ++ {"name": "ses2.tf.local"}, ++ {"name": "ses3.tf.local", "port": "1234"}, ++ ], ++ "auth": { ++ "username": "libvirt", ++ "secret": {"type": "ceph", "usage": "pool_test-rbd"}, ++ }, ++ }, ++ "target": {"dev": "vda", "bus": "virtio"}, ++ "driver": { ++ "name": "qemu", ++ "type": "raw", ++ "cache": "none", ++ "io": "native", ++ }, ++ }, ++ salt.utils.xmlutil.to_dict(disk, True), ++ ) ++ ++ # RBD volume UUID auth test case ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = """ ++ ++ test-rbd ++ ede33e0a-9df0-479f-8afd-55085a01b244 ++ 526133493760 ++ 589928 ++ 515081306112 ++ ++ ++ ++ libvirt-pool ++ ++ ++ ++ ++ ++ """ ++ self.mock_conn.secretLookupByUUIDString.return_value.usageID.return_value = ( ++ "pool_test-rbd" ++ ) ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "system", "pool": "test-rbd"}], ++ "test-vm", ++ ) ++ xml_data = virt._gen_xml( ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) ++ root = ET.fromstring(xml_data) ++ self.assertDictEqual( ++ { ++ "username": "libvirt", ++ "secret": {"type": "ceph", "usage": "pool_test-rbd"}, ++ }, ++ salt.utils.xmlutil.to_dict(root.find(".//disk/source/auth"), True), ++ ) ++ self.mock_conn.secretLookupByUUIDString.assert_called_once_with("some-uuid") ++ ++ # Disk volume test case ++ self.mock_conn.getStoragePoolCapabilities.return_value = """ ++ ++ ++ ++ ++ ++ none ++ linux ++ fat16 ++ ++ ++ ++ ++ """ ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = """ ++ ++ test-vdb ++ ++ ++ ++ ++ ++ """ ++ self.mock_conn.listStoragePools.return_value = ["test-vdb"] ++ self.mock_conn.storagePoolLookupByName.return_value.listVolumes.return_value = [ ++ "vdb1", ++ ] ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [{"name": "system", "pool": "test-vdb"}], ++ "test-vm", ++ ) ++ xml_data = virt._gen_xml( ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) ++ root = ET.fromstring(xml_data) ++ disk = root.findall(".//disk")[0] ++ self.assertEqual(disk.attrib["type"], "volume") ++ source = disk.find("source") ++ self.assertEqual("test-vdb", source.attrib["pool"]) ++ self.assertEqual("vdb2", source.attrib["volume"]) ++ self.assertEqual("raw", disk.find("driver").get("type")) + + def test_gen_xml_cdrom(self): +- ''' ++ """ + Test virt._gen_xml(), generating a cdrom device (different disk type, no source) +- ''' +- diskp = virt._disk_profile(None, 'kvm', [{ +- 'name': 'tested', +- 'device': 'cdrom', +- 'source_file': None, +- 'model': 'ide'}], 'hello') +- nicp = virt._nic_profile(None, 'kvm') ++ """ ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = ( ++ "" ++ ) ++ diskp = virt._disk_profile( ++ self.mock_conn, ++ None, ++ "kvm", ++ [ ++ {"name": "system", "pool": "default"}, ++ { ++ "name": "tested", ++ "device": "cdrom", ++ "source_file": None, ++ "model": "ide", ++ }, ++ { ++ "name": "remote", ++ "device": "cdrom", ++ "source_file": "http://myhost:8080/url/to/image?query=foo&filter=bar", ++ "model": "ide", ++ }, ++ ], ++ "hello", ++ ) ++ nicp = virt._nic_profile(None, "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- disk = root.findall('.//disk')[0] +- self.assertEqual(disk.attrib['device'], 'cdrom') +- self.assertIsNone(disk.find('source')) ++ disk = root.findall(".//disk")[1] ++ self.assertEqual(disk.get("type"), "file") ++ self.assertEqual(disk.attrib["device"], "cdrom") ++ self.assertIsNone(disk.find("source")) ++ self.assertEqual(disk.find("target").get("dev"), "hda") ++ ++ disk = root.findall(".//disk")[2] ++ self.assertEqual(disk.get("type"), "network") ++ self.assertEqual(disk.attrib["device"], "cdrom") ++ self.assertEqual( ++ { ++ "protocol": "http", ++ "name": "/url/to/image", ++ "query": "query=foo&filter=bar", ++ "host": {"name": "myhost", "port": "8080"}, ++ }, ++ salt.utils.xmlutil.to_dict(disk.find("source"), True), ++ ) + + def test_controller_for_esxi(self): +- ''' ++ """ + Test virt._gen_xml() generated device controller for ESXi/vmware +- ''' +- diskp = virt._disk_profile('default', 'vmware', [], 'hello') +- nicp = virt._nic_profile('default', 'vmware') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "vmware", [], "hello") ++ nicp = virt._nic_profile("default", "vmware") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'vmware', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "vmware", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- controllers = root.findall('.//devices/controller') ++ controllers = root.findall(".//devices/controller") + self.assertTrue(len(controllers) == 1) + controller = controllers[0] +- self.assertEqual(controller.attrib['model'], 'lsilogic') ++ self.assertEqual(controller.attrib["model"], "lsilogic") + + def test_controller_for_kvm(self): +- ''' ++ """ + Test virt._gen_xml() generated device controller for KVM +- ''' +- diskp = virt._disk_profile('default', 'kvm', [], 'hello') +- nicp = virt._nic_profile('default', 'kvm') ++ """ ++ diskp = virt._disk_profile(self.mock_conn, "default", "kvm", [], "hello") ++ nicp = virt._nic_profile("default", "kvm") + xml_data = virt._gen_xml( +- 'hello', +- 1, +- 512, +- diskp, +- nicp, +- 'kvm', +- 'hvm', +- 'x86_64', +- ) ++ self.mock_conn, "hello", 1, 512, diskp, nicp, "kvm", "hvm", "x86_64", ++ ) + root = ET.fromstring(xml_data) +- controllers = root.findall('.//devices/controller') ++ controllers = root.findall(".//devices/controller") + # There should be no controller + self.assertTrue(len(controllers) == 0) +- # kvm mac address shoud start with 52:54:00 +- self.assertTrue("mac address='52:54:00" in xml_data) + + def test_diff_disks(self): +- ''' ++ """ + Test virt._diff_disks() +- ''' +- old_disks = ET.fromstring(''' ++ """ ++ old_disks = ET.fromstring( ++ """ + + + +@@ -881,9 +1254,11 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- ''').findall('disk') ++ """ ++ ).findall("disk") + +- new_disks = ET.fromstring(''' ++ new_disks = ET.fromstring( ++ """ + + + +@@ -901,35 +1276,63 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- ''').findall('disk') ++ """ ++ ).findall("disk") + ret = virt._diff_disk_lists(old_disks, new_disks) +- self.assertEqual([disk.find('source').get('file') if disk.find('source') is not None else None +- for disk in ret['unchanged']], []) +- self.assertEqual([disk.find('source').get('file') if disk.find('source') is not None else None +- for disk in ret['new']], +- ['/path/to/img3.qcow2', '/path/to/img0.qcow2', '/path/to/img4.qcow2', None]) +- self.assertEqual([disk.find('target').get('dev') for disk in ret['sorted']], +- ['vda', 'vdb', 'vdc', 'hda']) +- self.assertEqual([disk.find('source').get('file') if disk.find('source') is not None else None +- for disk in ret['sorted']], +- ['/path/to/img3.qcow2', +- '/path/to/img0.qcow2', +- '/path/to/img4.qcow2', +- None]) +- self.assertEqual(ret['new'][1].find('target').get('bus'), 'virtio') +- self.assertEqual([disk.find('source').get('file') if disk.find('source') is not None else None +- for disk in ret['deleted']], +- ['/path/to/img0.qcow2', +- '/path/to/img1.qcow2', +- '/path/to/img2.qcow2', +- '/path/to/img4.qcow2', +- None]) ++ self.assertEqual( ++ [ ++ disk.find("source").get("file") ++ if disk.find("source") is not None ++ else None ++ for disk in ret["unchanged"] ++ ], ++ [], ++ ) ++ self.assertEqual( ++ [ ++ disk.find("source").get("file") ++ if disk.find("source") is not None ++ else None ++ for disk in ret["new"] ++ ], ++ ["/path/to/img3.qcow2", "/path/to/img0.qcow2", "/path/to/img4.qcow2", None], ++ ) ++ self.assertEqual( ++ [disk.find("target").get("dev") for disk in ret["sorted"]], ++ ["vda", "vdb", "vdc", "hda"], ++ ) ++ self.assertEqual( ++ [ ++ disk.find("source").get("file") ++ if disk.find("source") is not None ++ else None ++ for disk in ret["sorted"] ++ ], ++ ["/path/to/img3.qcow2", "/path/to/img0.qcow2", "/path/to/img4.qcow2", None], ++ ) ++ self.assertEqual(ret["new"][1].find("target").get("bus"), "virtio") ++ self.assertEqual( ++ [ ++ disk.find("source").get("file") ++ if disk.find("source") is not None ++ else None ++ for disk in ret["deleted"] ++ ], ++ [ ++ "/path/to/img0.qcow2", ++ "/path/to/img1.qcow2", ++ "/path/to/img2.qcow2", ++ "/path/to/img4.qcow2", ++ None, ++ ], ++ ) + + def test_diff_nics(self): +- ''' ++ """ + Test virt._diff_nics() +- ''' +- old_nics = ET.fromstring(''' ++ """ ++ old_nics = ET.fromstring( ++ """ + + + +@@ -950,9 +1353,11 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): +
+ + +- ''').findall('interface') ++ """ ++ ).findall("interface") + +- new_nics = ET.fromstring(''' ++ new_nics = ET.fromstring( ++ """ + + + +@@ -970,20 +1375,27 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- ''').findall('interface') ++ """ ++ ).findall("interface") + ret = virt._diff_interface_lists(old_nics, new_nics) +- self.assertEqual([nic.find('mac').get('address') for nic in ret['unchanged']], +- ['52:54:00:39:02:b1']) +- self.assertEqual([nic.find('mac').get('address') for nic in ret['new']], +- ['52:54:00:39:02:b2', '52:54:00:39:02:b4']) +- self.assertEqual([nic.find('mac').get('address') for nic in ret['deleted']], +- ['52:54:00:39:02:b2', '52:54:00:39:02:b3']) ++ self.assertEqual( ++ [nic.find("mac").get("address") for nic in ret["unchanged"]], ++ ["52:54:00:39:02:b1"], ++ ) ++ self.assertEqual( ++ [nic.find("mac").get("address") for nic in ret["new"]], ++ ["52:54:00:39:02:b2", "52:54:00:39:02:b4"], ++ ) ++ self.assertEqual( ++ [nic.find("mac").get("address") for nic in ret["deleted"]], ++ ["52:54:00:39:02:b2", "52:54:00:39:02:b3"], ++ ) + + def test_init(self): +- ''' ++ """ + Test init() function +- ''' +- xml = ''' ++ """ ++ xml = """ + + + 44454c4c-3400-105a-8033-b3c04f4b344a +@@ -1101,29 +1513,29 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- ''' ++ """ + self.mock_conn.getCapabilities.return_value = xml # pylint: disable=no-member + +- root_dir = os.path.join(salt.syspaths.ROOT_DIR, 'srv', 'salt-images') ++ root_dir = os.path.join(salt.syspaths.ROOT_DIR, "srv", "salt-images") + + defineMock = MagicMock(return_value=1) + self.mock_conn.defineXML = defineMock + mock_chmod = MagicMock() + mock_run = MagicMock() +- with patch.dict(os.__dict__, {'chmod': mock_chmod, 'makedirs': MagicMock()}): # pylint: disable=no-member +- with patch.dict(virt.__salt__, {'cmd.run': mock_run}): # pylint: disable=no-member ++ with patch.dict( ++ os.__dict__, {"chmod": mock_chmod, "makedirs": MagicMock()} ++ ): # pylint: disable=no-member ++ with patch.dict( ++ virt.__salt__, {"cmd.run": mock_run} ++ ): # pylint: disable=no-member + + # Ensure the init() function allows creating VM without NIC and disk +- virt.init('test vm', +- 2, +- 1234, +- nic=None, +- disk=None, +- seed=False, +- start=False) ++ virt.init( ++ "test vm", 2, 1234, nic=None, disk=None, seed=False, start=False ++ ) + definition = defineMock.call_args_list[0][0][0] +- self.assertFalse(' +- my vm ++ my_vm + 1048576 + 1048576 + 1 +@@ -1226,20 +1733,33 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- ++ + + + +
+ +- ++ + +- ++ + + + +
+ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++
++ + + + +@@ -1266,78 +1786,142 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + + + +- '''.format(root_dir, os.sep) +- domain_mock = self.set_mock_vm('my vm', xml) +- domain_mock.OSType = MagicMock(return_value='hvm') ++ """.format( ++ root_dir, os.sep ++ ) ++ domain_mock = self.set_mock_vm("my_vm", xml) ++ domain_mock.OSType = MagicMock(return_value="hvm") + define_mock = MagicMock(return_value=True) + self.mock_conn.defineXML = define_mock + + # No parameter passed case +- self.assertEqual({ +- 'definition': False, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm')) ++ self.assertEqual( ++ { ++ "definition": False, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm"), ++ ) + + # Same parameters passed than in default virt.defined state case +- self.assertEqual({ +- 'definition': False, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', +- cpu=None, +- mem=None, +- disk_profile=None, +- disks=None, +- nic_profile=None, +- interfaces=None, +- graphics=None, +- live=True, +- connection=None, +- username=None, +- password=None, +- boot=None)) ++ self.assertEqual( ++ { ++ "definition": False, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update( ++ "my_vm", ++ cpu=None, ++ mem=None, ++ disk_profile=None, ++ disks=None, ++ nic_profile=None, ++ interfaces=None, ++ graphics=None, ++ live=True, ++ connection=None, ++ username=None, ++ password=None, ++ boot=None, ++ ), ++ ) + + # Update vcpus case + setvcpus_mock = MagicMock(return_value=0) + domain_mock.setVcpusFlags = setvcpus_mock +- self.assertEqual({ +- 'definition': True, +- 'cpu': True, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', cpu=2)) ++ self.assertEqual( ++ { ++ "definition": True, ++ "cpu": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", cpu=2), ++ ) + setxml = ET.fromstring(define_mock.call_args[0][0]) +- self.assertEqual(setxml.find('vcpu').text, '2') ++ self.assertEqual(setxml.find("vcpu").text, "2") + self.assertEqual(setvcpus_mock.call_args[0][0], 2) + + boot = { +- 'kernel': '/root/f8-i386-vmlinuz', +- 'initrd': '/root/f8-i386-initrd', +- 'cmdline': +- 'console=ttyS0 ks=http://example.com/f8-i386/os/' ++ "kernel": "/root/f8-i386-vmlinuz", ++ "initrd": "/root/f8-i386-initrd", ++ "cmdline": "console=ttyS0 ks=http://example.com/f8-i386/os/", ++ } ++ ++ boot_uefi = { ++ "loader": "/usr/share/OVMF/OVMF_CODE.fd", ++ "nvram": "/usr/share/OVMF/OVMF_VARS.ms.fd", ++ } ++ ++ invalid_boot = { ++ "loader": "/usr/share/OVMF/OVMF_CODE.fd", ++ "initrd": "/root/f8-i386-initrd", + } + + # Update with boot parameter case +- self.assertEqual({ +- 'definition': True, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', boot=boot)) ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", boot=boot), ++ ) ++ setxml = ET.fromstring(define_mock.call_args[0][0]) ++ self.assertEqual(setxml.find("os").find("kernel").text, "/root/f8-i386-vmlinuz") ++ self.assertEqual(setxml.find("os").find("initrd").text, "/root/f8-i386-initrd") ++ self.assertEqual( ++ setxml.find("os").find("cmdline").text, ++ "console=ttyS0 ks=http://example.com/f8-i386/os/", ++ ) ++ setxml = ET.fromstring(define_mock.call_args[0][0]) ++ self.assertEqual(setxml.find("os").find("kernel").text, "/root/f8-i386-vmlinuz") ++ self.assertEqual(setxml.find("os").find("initrd").text, "/root/f8-i386-initrd") ++ self.assertEqual( ++ setxml.find("os").find("cmdline").text, ++ "console=ttyS0 ks=http://example.com/f8-i386/os/", ++ ) ++ ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", boot=boot_uefi), ++ ) ++ setxml = ET.fromstring(define_mock.call_args[0][0]) ++ self.assertEqual( ++ setxml.find("os").find("loader").text, "/usr/share/OVMF/OVMF_CODE.fd" ++ ) ++ self.assertEqual(setxml.find("os").find("loader").attrib.get("readonly"), "yes") ++ self.assertEqual(setxml.find("os").find("loader").attrib["type"], "pflash") ++ self.assertEqual( ++ setxml.find("os").find("nvram").attrib["template"], ++ "/usr/share/OVMF/OVMF_VARS.ms.fd", ++ ) ++ ++ with self.assertRaises(SaltInvocationError): ++ virt.update("my_vm", boot=invalid_boot) + + # Update memory case + setmem_mock = MagicMock(return_value=0) + domain_mock.setMemoryFlags = setmem_mock + +- self.assertEqual({ +- 'definition': True, +- 'mem': True, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', mem=2048)) ++ self.assertEqual( ++ { ++ "definition": True, ++ "mem": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", mem=2048), ++ ) + setxml = ET.fromstring(define_mock.call_args[0][0]) +- self.assertEqual(setxml.find('memory').text, '2048') +- self.assertEqual(setxml.find('memory').get('unit'), 'MiB') ++ self.assertEqual(setxml.find("memory").text, "2048") ++ self.assertEqual(setxml.find("memory").get("unit"), "MiB") + self.assertEqual(setmem_mock.call_args[0][0], 2048 * 1024) + + # Update disks case +@@ -1347,122 +1931,630 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): + domain_mock.detachDevice = devdetach_mock + mock_chmod = MagicMock() + mock_run = MagicMock() +- with patch.dict(os.__dict__, {'chmod': mock_chmod, 'makedirs': MagicMock()}): # pylint: disable=no-member +- with patch.dict(virt.__salt__, {'cmd.run': mock_run}): # pylint: disable=no-member +- ret = virt.update('my vm', disk_profile='default', disks=[ +- {'name': 'cddrive', 'device': 'cdrom', 'source_file': None, 'model': 'ide'}, +- {'name': 'added', 'size': 2048}]) ++ with patch.dict( ++ os.__dict__, {"chmod": mock_chmod, "makedirs": MagicMock()} ++ ): # pylint: disable=no-member ++ with patch.dict( ++ virt.__salt__, {"cmd.run": mock_run} ++ ): # pylint: disable=no-member ++ ret = virt.update( ++ "my_vm", ++ disk_profile="default", ++ disks=[ ++ { ++ "name": "cddrive", ++ "device": "cdrom", ++ "source_file": None, ++ "model": "ide", ++ }, ++ {"name": "added", "size": 2048}, ++ ], ++ ) + added_disk_path = os.path.join( +- virt.__salt__['config.get']('virt:images'), 'my vm_added.qcow2') # pylint: disable=no-member +- self.assertEqual(mock_run.call_args[0][0], +- 'qemu-img create -f qcow2 "{0}" 2048M'.format(added_disk_path)) ++ virt.__salt__["config.get"]("virt:images"), "my_vm_added.qcow2" ++ ) # pylint: disable=no-member ++ self.assertEqual( ++ mock_run.call_args[0][0], ++ 'qemu-img create -f qcow2 "{0}" 2048M'.format(added_disk_path), ++ ) + self.assertEqual(mock_chmod.call_args[0][0], added_disk_path) + self.assertListEqual( +- [None, os.path.join(root_dir, 'my vm_added.qcow2')], +- [ET.fromstring(disk).find('source').get('file') if str(disk).find(' -1 else None +- for disk in ret['disk']['attached']]) ++ [None, os.path.join(root_dir, "my_vm_added.qcow2")], ++ [ ++ ET.fromstring(disk).find("source").get("file") ++ if str(disk).find(" -1 ++ else None ++ for disk in ret["disk"]["attached"] ++ ], ++ ) + + self.assertListEqual( +- [os.path.join(root_dir, 'my vm_data.qcow2')], +- [ET.fromstring(disk).find('source').get('file') for disk in ret['disk']['detached']]) ++ ["my_vm_data", "libvirt-pool/my_vm_data2"], ++ [ ++ ET.fromstring(disk).find("source").get("volume") ++ or ET.fromstring(disk).find("source").get("name") ++ for disk in ret["disk"]["detached"] ++ ], ++ ) + self.assertEqual(devattach_mock.call_count, 2) +- devdetach_mock.assert_called_once() ++ self.assertEqual(devdetach_mock.call_count, 2) + + # Update nics case +- yaml_config = ''' ++ yaml_config = """ + virt: + nic: + myprofile: + - network: default + name: eth0 +- ''' ++ """ + mock_config = salt.utils.yaml.safe_load(yaml_config) + devattach_mock.reset_mock() + devdetach_mock.reset_mock() +- with patch.dict(salt.modules.config.__opts__, mock_config): # pylint: disable=no-member +- ret = virt.update('my vm', nic_profile='myprofile', +- interfaces=[{'name': 'eth0', 'type': 'network', 'source': 'default', +- 'mac': '52:54:00:39:02:b1'}, +- {'name': 'eth1', 'type': 'network', 'source': 'newnet'}]) +- self.assertEqual(['newnet'], +- [ET.fromstring(nic).find('source').get('network') for nic in ret['interface']['attached']]) +- self.assertEqual(['oldnet'], +- [ET.fromstring(nic).find('source').get('network') for nic in ret['interface']['detached']]) ++ with patch.dict( ++ salt.modules.config.__opts__, mock_config # pylint: disable=no-member ++ ): ++ ret = virt.update( ++ "my_vm", ++ nic_profile="myprofile", ++ interfaces=[ ++ { ++ "name": "eth0", ++ "type": "network", ++ "source": "default", ++ "mac": "52:54:00:39:02:b1", ++ }, ++ {"name": "eth1", "type": "network", "source": "newnet"}, ++ ], ++ ) ++ self.assertEqual( ++ ["newnet"], ++ [ ++ ET.fromstring(nic).find("source").get("network") ++ for nic in ret["interface"]["attached"] ++ ], ++ ) ++ self.assertEqual( ++ ["oldnet"], ++ [ ++ ET.fromstring(nic).find("source").get("network") ++ for nic in ret["interface"]["detached"] ++ ], ++ ) + devattach_mock.assert_called_once() + devdetach_mock.assert_called_once() + + # Remove nics case + devattach_mock.reset_mock() + devdetach_mock.reset_mock() +- ret = virt.update('my vm', nic_profile=None, interfaces=[]) +- self.assertEqual([], ret['interface']['attached']) +- self.assertEqual(2, len(ret['interface']['detached'])) ++ ret = virt.update("my_vm", nic_profile=None, interfaces=[]) ++ self.assertEqual([], ret["interface"]["attached"]) ++ self.assertEqual(2, len(ret["interface"]["detached"])) + devattach_mock.assert_not_called() + devdetach_mock.assert_called() + + # Remove disks case (yeah, it surely is silly) + devattach_mock.reset_mock() + devdetach_mock.reset_mock() +- ret = virt.update('my vm', disk_profile=None, disks=[]) +- self.assertEqual([], ret['disk']['attached']) +- self.assertEqual(2, len(ret['disk']['detached'])) ++ ret = virt.update("my_vm", disk_profile=None, disks=[]) ++ self.assertEqual([], ret["disk"]["attached"]) ++ self.assertEqual(3, len(ret["disk"]["detached"])) + devattach_mock.assert_not_called() + devdetach_mock.assert_called() + + # Graphics change test case +- self.assertEqual({ +- 'definition': True, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', graphics={'type': 'vnc'})) ++ self.assertEqual( ++ { ++ "definition": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", graphics={"type": "vnc"}), ++ ) + setxml = ET.fromstring(define_mock.call_args[0][0]) +- self.assertEqual('vnc', setxml.find('devices/graphics').get('type')) ++ self.assertEqual("vnc", setxml.find("devices/graphics").get("type")) + + # Update with no diff case +- self.assertEqual({ +- 'definition': False, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', cpu=1, mem=1024, +- disk_profile='default', disks=[{'name': 'data', 'size': 2048}], +- nic_profile='myprofile', +- interfaces=[{'name': 'eth0', 'type': 'network', 'source': 'default', +- 'mac': '52:54:00:39:02:b1'}, +- {'name': 'eth1', 'type': 'network', 'source': 'oldnet', +- 'mac': '52:54:00:39:02:b2'}], +- graphics={'type': 'spice', +- 'listen': {'type': 'address', 'address': '127.0.0.1'}})) ++ pool_mock = MagicMock() ++ default_pool_desc = "" ++ rbd_pool_desc = """ ++ ++ test-rbd ++ ++ ++ ++ libvirt-pool ++ ++ ++ ++ ++ ++ """ ++ pool_mock.XMLDesc.side_effect = [ ++ default_pool_desc, ++ rbd_pool_desc, ++ default_pool_desc, ++ rbd_pool_desc, ++ ] ++ self.mock_conn.storagePoolLookupByName.return_value = pool_mock ++ self.mock_conn.listStoragePools.return_value = ["test-rbd", "default"] ++ self.assertEqual( ++ { ++ "definition": False, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update( ++ "my_vm", ++ cpu=1, ++ mem=1024, ++ disk_profile="default", ++ disks=[ ++ {"name": "data", "size": 2048, "pool": "default"}, ++ { ++ "name": "data2", ++ "size": 4096, ++ "pool": "test-rbd", ++ "format": "raw", ++ }, ++ ], ++ nic_profile="myprofile", ++ interfaces=[ ++ { ++ "name": "eth0", ++ "type": "network", ++ "source": "default", ++ "mac": "52:54:00:39:02:b1", ++ }, ++ {"name": "eth1", "type": "network", "source": "oldnet"}, ++ ], ++ graphics={ ++ "type": "spice", ++ "listen": {"type": "address", "address": "127.0.0.1"}, ++ }, ++ ), ++ ) + + # Failed XML description update case +- self.mock_conn.defineXML.side_effect = self.mock_libvirt.libvirtError("Test error") ++ self.mock_conn.defineXML.side_effect = self.mock_libvirt.libvirtError( ++ "Test error" ++ ) + setmem_mock.reset_mock() + with self.assertRaises(self.mock_libvirt.libvirtError): +- virt.update('my vm', mem=2048) ++ virt.update("my_vm", mem=2048) + + # Failed single update failure case + self.mock_conn.defineXML = MagicMock(return_value=True) +- setmem_mock.side_effect = self.mock_libvirt.libvirtError("Failed to live change memory") +- self.assertEqual({ +- 'definition': True, +- 'errors': ['Failed to live change memory'], +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', mem=2048)) ++ setmem_mock.side_effect = self.mock_libvirt.libvirtError( ++ "Failed to live change memory" ++ ) ++ self.assertEqual( ++ { ++ "definition": True, ++ "errors": ["Failed to live change memory"], ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", mem=2048), ++ ) + + # Failed multiple updates failure case +- self.assertEqual({ +- 'definition': True, +- 'errors': ['Failed to live change memory'], +- 'cpu': True, +- 'disk': {'attached': [], 'detached': []}, +- 'interface': {'attached': [], 'detached': []} +- }, virt.update('my vm', cpu=4, mem=2048)) ++ self.assertEqual( ++ { ++ "definition": True, ++ "errors": ["Failed to live change memory"], ++ "cpu": True, ++ "disk": {"attached": [], "detached": [], "updated": []}, ++ "interface": {"attached": [], "detached": []}, ++ }, ++ virt.update("my_vm", cpu=4, mem=2048), ++ ) ++ ++ def test_update_backing_store(self): ++ """ ++ Test updating a disk with a backing store ++ """ ++ xml = """ ++ ++ my_vm ++ 1048576 ++ 1048576 ++ 1 ++ ++ hvm ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ """ ++ domain_mock = self.set_mock_vm("my_vm", xml) ++ domain_mock.OSType.return_value = "hvm" ++ self.mock_conn.defineXML.return_value = True ++ updatedev_mock = MagicMock(return_value=0) ++ domain_mock.updateDeviceFlags = updatedev_mock ++ self.mock_conn.listStoragePools.return_value = ["default"] ++ self.mock_conn.storagePoolLookupByName.return_value.XMLDesc.return_value = ( ++ "" ++ ) ++ ++ ret = virt.update( ++ "my_vm", ++ disks=[ ++ { ++ "name": "system", ++ "pool": "default", ++ "backing_store_path": "/path/to/base.qcow2", ++ "backing_store_format": "qcow2", ++ }, ++ ], ++ ) ++ self.assertFalse(ret["definition"]) ++ self.assertFalse(ret["disk"]["attached"]) ++ self.assertFalse(ret["disk"]["detached"]) ++ ++ def test_update_removables(self): ++ """ ++ Test attaching, detaching, changing removable devices ++ """ ++ xml = """ ++ ++ my_vm ++ 1048576 ++ 1048576 ++ 1 ++ ++ hvm ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ """ ++ domain_mock = self.set_mock_vm("my_vm", xml) ++ domain_mock.OSType.return_value = "hvm" ++ self.mock_conn.defineXML.return_value = True ++ updatedev_mock = MagicMock(return_value=0) ++ domain_mock.updateDeviceFlags = updatedev_mock ++ ++ ret = virt.update( ++ "my_vm", ++ disks=[ ++ { ++ "name": "dvd1", ++ "device": "cdrom", ++ "source_file": None, ++ "model": "ide", ++ }, ++ { ++ "name": "dvd2", ++ "device": "cdrom", ++ "source_file": "/srv/dvd-image-4.iso", ++ "model": "ide", ++ }, ++ { ++ "name": "dvd3", ++ "device": "cdrom", ++ "source_file": "/srv/dvd-image-2.iso", ++ "model": "ide", ++ }, ++ { ++ "name": "dvd4", ++ "device": "cdrom", ++ "source_file": "/srv/dvd-image-5.iso", ++ "model": "ide", ++ }, ++ { ++ "name": "dvd5", ++ "device": "cdrom", ++ "source_file": "/srv/dvd-image-6.iso", ++ "model": "ide", ++ }, ++ ], ++ ) ++ ++ self.assertTrue(ret["definition"]) ++ self.assertFalse(ret["disk"]["attached"]) ++ self.assertFalse(ret["disk"]["detached"]) ++ self.assertEqual( ++ [ ++ { ++ "type": "file", ++ "device": "cdrom", ++ "driver": { ++ "name": "qemu", ++ "type": "raw", ++ "cache": "none", ++ "io": "native", ++ }, ++ "backingStore": None, ++ "target": {"dev": "hda", "bus": "ide"}, ++ "readonly": None, ++ "alias": {"name": "ide0-0-0"}, ++ "address": { ++ "type": "drive", ++ "controller": "0", ++ "bus": "0", ++ "target": "0", ++ "unit": "0", ++ }, ++ }, ++ { ++ "type": "file", ++ "device": "cdrom", ++ "driver": { ++ "name": "qemu", ++ "type": "raw", ++ "cache": "none", ++ "io": "native", ++ }, ++ "target": {"dev": "hdb", "bus": "ide"}, ++ "readonly": None, ++ "alias": {"name": "ide0-0-1"}, ++ "address": { ++ "type": "drive", ++ "controller": "0", ++ "bus": "0", ++ "target": "0", ++ "unit": "1", ++ }, ++ "source": {"file": "/srv/dvd-image-4.iso"}, ++ }, ++ { ++ "type": "file", ++ "device": "cdrom", ++ "driver": { ++ "name": "qemu", ++ "type": "raw", ++ "cache": "none", ++ "io": "native", ++ }, ++ "backingStore": None, ++ "target": {"dev": "hdd", "bus": "ide"}, ++ "readonly": None, ++ "alias": {"name": "ide0-0-3"}, ++ "address": { ++ "type": "drive", ++ "controller": "0", ++ "bus": "0", ++ "target": "0", ++ "unit": "3", ++ }, ++ "source": {"file": "/srv/dvd-image-5.iso"}, ++ }, ++ { ++ "type": "file", ++ "device": "cdrom", ++ "driver": { ++ "name": "qemu", ++ "type": "raw", ++ "cache": "none", ++ "io": "native", ++ }, ++ "backingStore": None, ++ "target": {"dev": "hde", "bus": "ide"}, ++ "readonly": None, ++ "source": {"file": "/srv/dvd-image-6.iso"}, ++ }, ++ ], ++ [ ++ salt.utils.xmlutil.to_dict(ET.fromstring(disk), True) ++ for disk in ret["disk"]["updated"] ++ ], ++ ) ++ ++ def test_update_existing_boot_params(self): ++ """ ++ Test virt.update() with existing boot parameters. ++ """ ++ root_dir = os.path.join(salt.syspaths.ROOT_DIR, "srv", "salt-images") ++ xml_boot = """ ++ ++ vm_with_boot_param ++ 1048576 ++ 1048576 ++ 1 ++ ++ hvm ++ /boot/oldkernel ++ /boot/initrdold.img ++ console=ttyS0 ks=http://example.com/old/os/ ++ /usr/share/old/OVMF_CODE.fd ++ /usr/share/old/OVMF_VARS.ms.fd ++ ++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++ ++ ++ ++
++ ++ ++ ++ ++