From 01f9005feb9cd0c0f47a4520d29a488c6814d07170d48a52a5e5b511feb4fcc1 Mon Sep 17 00:00:00 2001 From: Li Zhang Date: Wed, 6 Apr 2022 08:07:07 +0000 Subject: [PATCH] Accepting request 966963 from home:lizhang:branches:Virtualization - Backport aqmp patches from upstream which can fix iotest issues * Patches added: python-aqmp-add-__del__-method-to-legacy.patch python-aqmp-add-_session_guard.patch python-aqmp-add-SocketAddrT-to-package-r.patch python-aqmp-add-socket-bind-step-to-lega.patch python-aqmp-add-start_server-and-accept-.patch python-aqmp-copy-type-definitions-from-q.patch python-aqmp-drop-_bind_hack.patch python-aqmp-fix-docstring-typo.patch python-aqmp-Fix-negotiation-with-pre-oob.patch python-aqmp-fix-race-condition-in-legacy.patch Python-aqmp-fix-type-definitions-for-myp.patch python-aqmp-handle-asyncio.TimeoutError-.patch python-aqmp-refactor-_do_accept-into-two.patch python-aqmp-remove-_new_session-and-_est.patch python-aqmp-rename-accept-to-start_serve.patch python-aqmp-rename-AQMPError-to-QMPError.patch python-aqmp-split-_client_connected_cb-o.patch python-aqmp-squelch-pylint-warning-for-t.patch python-aqmp-stop-the-server-during-disco.patch python-introduce-qmp-shell-wrap-convenie.patch python-machine-raise-VMLaunchFailure-exc.patch python-move-qmp-shell-under-the-AQMP-pac.patch python-move-qmp-utilities-to-python-qemu.patch python-qmp-switch-qmp-shell-to-AQMP.patch python-support-recording-QMP-session-to-.patch python-upgrade-mypy-to-0.780.patch - Drop the patches which are workaround to fix iotest issues * Patches dropped: Revert-python-iotests-replace-qmp-with-a.patch Revert-python-machine-add-instance-disam.patch Revert-python-machine-add-sock_dir-prope.patch Revert-python-machine-handle-fast-QEMU-t.patch Revert-python-machine-move-more-variable.patch Revert-python-machine-remove-_remove_mon.patch OBS-URL: https://build.opensuse.org/request/show/966963 OBS-URL: https://build.opensuse.org/package/show/Virtualization/qemu?expand=0&rev=708 --- ...on-aqmp-fix-type-definitions-for-myp.patch | 48 + ...rt-python-iotests-replace-qmp-with-a.patch | 39 - ...rt-python-machine-add-instance-disam.patch | 28 - ...rt-python-machine-add-sock_dir-prope.patch | 70 - ...rt-python-machine-handle-fast-QEMU-t.patch | 66 - ...rt-python-machine-move-more-variable.patch | 56 - ...rt-python-machine-remove-_remove_mon.patch | 41 - bundles.tar.xz | 4 +- ...-Support-SGX-numa-in-the-monitor-and.patch | 1 - ...on-aqmp-Fix-negotiation-with-pre-oob.patch | 36 + ...on-aqmp-add-SocketAddrT-to-package-r.patch | 42 + ...on-aqmp-add-__del__-method-to-legacy.patch | 64 + python-aqmp-add-_session_guard.patch | 140 ++ ...on-aqmp-add-socket-bind-step-to-lega.patch | 135 ++ ...on-aqmp-add-start_server-and-accept-.patch | 158 ++ ...on-aqmp-copy-type-definitions-from-q.patch | 134 + python-aqmp-drop-_bind_hack.patch | 132 + python-aqmp-fix-docstring-typo.patch | 27 + ...on-aqmp-fix-race-condition-in-legacy.patch | 59 + ...on-aqmp-handle-asyncio.TimeoutError-.patch | 46 + ...on-aqmp-refactor-_do_accept-into-two.patch | 102 + ...on-aqmp-remove-_new_session-and-_est.patch | 231 ++ ...on-aqmp-rename-AQMPError-to-QMPError.patch | 221 ++ ...on-aqmp-rename-accept-to-start_serve.patch | 163 ++ ...on-aqmp-split-_client_connected_cb-o.patch | 149 ++ ...on-aqmp-squelch-pylint-warning-for-t.patch | 37 + ...on-aqmp-stop-the-server-during-disco.patch | 54 + ...on-introduce-qmp-shell-wrap-convenie.patch | 167 ++ ...on-machine-raise-VMLaunchFailure-exc.patch | 126 + ...on-move-qmp-shell-under-the-AQMP-pac.patch | 1143 +++++++++ ...on-move-qmp-utilities-to-python-qemu.patch | 2153 +++++++++++++++++ python-qmp-switch-qmp-shell-to-AQMP.patch | 121 + ...on-support-recording-QMP-session-to-.patch | 184 ++ python-upgrade-mypy-to-0.780.patch | 232 ++ qemu.changes | 44 + qemu.spec | 60 +- 36 files changed, 6200 insertions(+), 313 deletions(-) create mode 100644 Python-aqmp-fix-type-definitions-for-myp.patch delete mode 100644 Revert-python-iotests-replace-qmp-with-a.patch delete mode 100644 Revert-python-machine-add-instance-disam.patch delete mode 100644 Revert-python-machine-add-sock_dir-prope.patch delete mode 100644 Revert-python-machine-handle-fast-QEMU-t.patch delete mode 100644 Revert-python-machine-move-more-variable.patch delete mode 100644 Revert-python-machine-remove-_remove_mon.patch create mode 100644 python-aqmp-Fix-negotiation-with-pre-oob.patch create mode 100644 python-aqmp-add-SocketAddrT-to-package-r.patch create mode 100644 python-aqmp-add-__del__-method-to-legacy.patch create mode 100644 python-aqmp-add-_session_guard.patch create mode 100644 python-aqmp-add-socket-bind-step-to-lega.patch create mode 100644 python-aqmp-add-start_server-and-accept-.patch create mode 100644 python-aqmp-copy-type-definitions-from-q.patch create mode 100644 python-aqmp-drop-_bind_hack.patch create mode 100644 python-aqmp-fix-docstring-typo.patch create mode 100644 python-aqmp-fix-race-condition-in-legacy.patch create mode 100644 python-aqmp-handle-asyncio.TimeoutError-.patch create mode 100644 python-aqmp-refactor-_do_accept-into-two.patch create mode 100644 python-aqmp-remove-_new_session-and-_est.patch create mode 100644 python-aqmp-rename-AQMPError-to-QMPError.patch create mode 100644 python-aqmp-rename-accept-to-start_serve.patch create mode 100644 python-aqmp-split-_client_connected_cb-o.patch create mode 100644 python-aqmp-squelch-pylint-warning-for-t.patch create mode 100644 python-aqmp-stop-the-server-during-disco.patch create mode 100644 python-introduce-qmp-shell-wrap-convenie.patch create mode 100644 python-machine-raise-VMLaunchFailure-exc.patch create mode 100644 python-move-qmp-shell-under-the-AQMP-pac.patch create mode 100644 python-move-qmp-utilities-to-python-qemu.patch create mode 100644 python-qmp-switch-qmp-shell-to-AQMP.patch create mode 100644 python-support-recording-QMP-session-to-.patch create mode 100644 python-upgrade-mypy-to-0.780.patch diff --git a/Python-aqmp-fix-type-definitions-for-myp.patch b/Python-aqmp-fix-type-definitions-for-myp.patch new file mode 100644 index 00000000..dfca8007 --- /dev/null +++ b/Python-aqmp-fix-type-definitions-for-myp.patch @@ -0,0 +1,48 @@ +From: John Snow +Date: Mon, 10 Jan 2022 14:13:48 -0500 +Subject: Python/aqmp: fix type definitions for mypy 0.920 + +Git-commit: 42d73f2894ea1855df5a25d58e0d9eac6023dcc3 + +0.920 (Released 2021-12-15) is not entirely happy with the +way that I was defining _FutureT: + +qemu/aqmp/protocol.py:601: error: Item "object" of the upper bound +"Optional[Future[Any]]" of type variable "_FutureT" has no attribute +"done" + +Update it with something a little mechanically simpler that works better +across a wider array of mypy versions. + +Signed-off-by: John Snow +Message-id: 20220110191349.1841027-3-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 5190b33b13df24fc2ca4aed934ed..c4fbe35a0e41c589059ec4fa37a8 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -43,8 +43,8 @@ from .util import ( + + + T = TypeVar('T') ++_U = TypeVar('_U') + _TaskFN = Callable[[], Awaitable[None]] # aka ``async def func() -> None`` +-_FutureT = TypeVar('_FutureT', bound=Optional['asyncio.Future[Any]']) + + + class Runstate(Enum): +@@ -591,7 +591,8 @@ class AsyncProtocol(Generic[T]): + """ + Fully reset this object to a clean state and return to `IDLE`. + """ +- def _paranoid_task_erase(task: _FutureT) -> Optional[_FutureT]: ++ def _paranoid_task_erase(task: Optional['asyncio.Future[_U]'] ++ ) -> Optional['asyncio.Future[_U]']: + # Help to erase a task, ENSURING it is fully quiesced first. + assert (task is None) or task.done() + return None if (task and task.done()) else task diff --git a/Revert-python-iotests-replace-qmp-with-a.patch b/Revert-python-iotests-replace-qmp-with-a.patch deleted file mode 100644 index 1c4b5aee..00000000 --- a/Revert-python-iotests-replace-qmp-with-a.patch +++ /dev/null @@ -1,39 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 12:04:16 +0200 -Subject: Revert "python, iotests: replace qmp with aqmp" - -References: bsc#1197528 bsc#1197150 - -aqmp is still not stable, it causes failures. -This reverts commit 76cd358671e6b8e7c435ec65b1c44200254514a9. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 7 +------ - 1 file changed, 1 insertion(+), 6 deletions(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index a487c397459a1fa6889276ab2538..a0cf69786b4bb7e851b5eeb2517b 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -41,6 +41,7 @@ from typing import ( - ) - - from qemu.qmp import ( # pylint: disable=import-error -+ QEMUMonitorProtocol, - QMPMessage, - QMPReturnValue, - SocketAddrT, -@@ -49,12 +50,6 @@ from qemu.qmp import ( # pylint: disable=import-error - from . import console_socket - - --if os.environ.get('QEMU_PYTHON_LEGACY_QMP'): -- from qemu.qmp import QEMUMonitorProtocol --else: -- from qemu.aqmp.legacy import QEMUMonitorProtocol -- -- - LOG = logging.getLogger(__name__) - - diff --git a/Revert-python-machine-add-instance-disam.patch b/Revert-python-machine-add-instance-disam.patch deleted file mode 100644 index 810e70e5..00000000 --- a/Revert-python-machine-add-instance-disam.patch +++ /dev/null @@ -1,28 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 12:00:29 +0200 -Subject: Revert "python/machine: add instance disambiguator to default - nickname" - -References: bsc#1197528 bsc#1197150 - -To improve testsuit, these patches still need more testing. -This reverts commit 72b17fe715056c96ea73f187ab46721788b3a782. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index ad529fd92a6022150fd0156d005d..ea9e07805df10a57115dace06885 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -133,7 +133,7 @@ class QEMUMachine: - self._wrapper = wrapper - self._qmp_timer = qmp_timer - -- self._name = name or f"qemu-{os.getpid()}-{id(self):02x}" -+ self._name = name or "qemu-%d" % os.getpid() - self._temp_dir: Optional[str] = None - self._base_temp_dir = base_temp_dir - self._sock_dir = sock_dir diff --git a/Revert-python-machine-add-sock_dir-prope.patch b/Revert-python-machine-add-sock_dir-prope.patch deleted file mode 100644 index dd7ddd98..00000000 --- a/Revert-python-machine-add-sock_dir-prope.patch +++ /dev/null @@ -1,70 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 12:02:45 +0200 -Subject: Revert "python/machine: add @sock_dir property" - -References: bsc#1197528 bsc#1197150 - -To improve testsuit, these patches still need more testing. -This reverts commit 87bf1fe5cbffefe6b7ee13a7015ae285250ad2db. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 17 ++++------------- - 1 file changed, 4 insertions(+), 13 deletions(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index b1dd77b53885629eba452cdd1bc2..a487c397459a1fa6889276ab2538 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -134,9 +134,8 @@ class QEMUMachine: - self._qmp_timer = qmp_timer - - self._name = name or "qemu-%d" % os.getpid() -- self._temp_dir: Optional[str] = None - self._base_temp_dir = base_temp_dir -- self._sock_dir = sock_dir -+ self._sock_dir = sock_dir or self._base_temp_dir - self._log_dir = log_dir - - if monitor_address is not None: -@@ -144,7 +143,7 @@ class QEMUMachine: - self._remove_monitor_sockfile = False - else: - self._monitor_address = os.path.join( -- self.sock_dir, f"{self._name}-monitor.sock" -+ self._sock_dir, f"{self._name}-monitor.sock" - ) - self._remove_monitor_sockfile = True - -@@ -164,13 +163,14 @@ class QEMUMachine: - self._qmp_set = True # Enable QMP monitor by default. - self._qmp_connection: Optional[QEMUMonitorProtocol] = None - self._qemu_full_args: Tuple[str, ...] = () -+ self._temp_dir: Optional[str] = None - self._launched = False - self._machine: Optional[str] = None - self._console_index = 0 - self._console_set = False - self._console_device_type: Optional[str] = None - self._console_address = os.path.join( -- self.sock_dir, f"{self._name}-console.sock" -+ self._sock_dir, f"{self._name}-console.sock" - ) - self._console_socket: Optional[socket.socket] = None - self._remove_files: List[str] = [] -@@ -816,15 +816,6 @@ class QEMUMachine: - dir=self._base_temp_dir) - return self._temp_dir - -- @property -- def sock_dir(self) -> str: -- """ -- Returns the directory used for sockfiles by this machine. -- """ -- if self._sock_dir: -- return self._sock_dir -- return self.temp_dir -- - @property - def log_dir(self) -> str: - """ diff --git a/Revert-python-machine-handle-fast-QEMU-t.patch b/Revert-python-machine-handle-fast-QEMU-t.patch deleted file mode 100644 index 15e84ea6..00000000 --- a/Revert-python-machine-handle-fast-QEMU-t.patch +++ /dev/null @@ -1,66 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 11:51:54 +0200 -Subject: Revert "python/machine: handle "fast" QEMU terminations" - -References: bsc#1197528 bsc#1197150 - -This patch causes iotest failures, it needs to revert. -This reverts commit 1611e6cf4e7163f6102b37010a8b7e7120f468b5. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 19 +++++++------------ - 1 file changed, 7 insertions(+), 12 deletions(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index 67ab06ca2b6daa531b7c0ad9f7c2..f92e73de4010d10c9e062259c706 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -349,6 +349,9 @@ class QEMUMachine: - Called to cleanup the VM instance after the process has exited. - May also be called after a failed launch. - """ -+ # Comprehensive reset for the failed launch case: -+ self._early_cleanup() -+ - try: - self._close_qmp_connection() - except Exception as err: # pylint: disable=broad-except -@@ -397,16 +400,9 @@ class QEMUMachine: - - try: - self._launch() -+ self._launched = True - except: -- # We may have launched the process but it may -- # have exited before we could connect via QMP. -- # Assume the VM didn't launch or is exiting. -- # If we don't wait for the process, exitcode() may still be -- # 'None' by the time control is ceded back to the caller. -- if self._launched: -- self.wait() -- else: -- self._post_shutdown() -+ self._post_shutdown() - - LOG.debug('Error launching VM') - if self._qemu_full_args: -@@ -430,7 +426,6 @@ class QEMUMachine: - stderr=subprocess.STDOUT, - shell=False, - close_fds=False) -- self._launched = True - self._post_launch() - - def _close_qmp_connection(self) -> None: -@@ -462,8 +457,8 @@ class QEMUMachine: - """ - Perform any cleanup that needs to happen before the VM exits. - -- This method may be called twice upon shutdown, once each by soft -- and hard shutdown in failover scenarios. -+ May be invoked by both soft and hard shutdown in failover scenarios. -+ Called additionally by _post_shutdown for comprehensive cleanup. - """ - # If we keep the console socket open, we may deadlock waiting - # for QEMU to exit, while QEMU is waiting for the socket to diff --git a/Revert-python-machine-move-more-variable.patch b/Revert-python-machine-move-more-variable.patch deleted file mode 100644 index 09504a4b..00000000 --- a/Revert-python-machine-move-more-variable.patch +++ /dev/null @@ -1,56 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 11:57:11 +0200 -Subject: Revert "python/machine: move more variable initializations to - _pre_launch" - -References: bsc#1197528 bsc#1197150 - -To improve testsuit, these patches still need more testing. -This reverts commit b1ca99199320fcc010f407b84ac00d96e7e4baa1. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index f92e73de4010d10c9e062259c706..ad529fd92a6022150fd0156d005d 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -327,14 +327,6 @@ class QEMUMachine: - self._qemu_log_path = os.path.join(self.log_dir, self._name + ".log") - self._qemu_log_file = open(self._qemu_log_path, 'wb') - -- self._iolog = None -- self._qemu_full_args = tuple(chain( -- self._wrapper, -- [self._binary], -- self._base_args, -- self._args -- )) -- - def _post_launch(self) -> None: - if self._qmp_connection: - self._qmp.accept(self._qmp_timer) -@@ -398,6 +390,8 @@ class QEMUMachine: - if self._launched: - raise QEMUMachineError('VM already launched') - -+ self._iolog = None -+ self._qemu_full_args = () - try: - self._launch() - self._launched = True -@@ -416,6 +410,12 @@ class QEMUMachine: - Launch the VM and establish a QMP connection - """ - self._pre_launch() -+ self._qemu_full_args = tuple( -+ chain(self._wrapper, -+ [self._binary], -+ self._base_args, -+ self._args) -+ ) - LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) - - # Cleaning up of this subprocess is guaranteed by _do_shutdown. diff --git a/Revert-python-machine-remove-_remove_mon.patch b/Revert-python-machine-remove-_remove_mon.patch deleted file mode 100644 index fe749df9..00000000 --- a/Revert-python-machine-remove-_remove_mon.patch +++ /dev/null @@ -1,41 +0,0 @@ -From: Li Zhang -Date: Tue, 29 Mar 2022 12:01:34 +0200 -Subject: Revert "python/machine: remove _remove_monitor_sockfile property" - -References: bsc#1197528 bsc#1197150 - -To improve testsuit, these patches still need more testing. -This reverts commit 6eeb3de7e1aff91ce6e092a39f85946d12664385. - -Signed-off-by: Li Zhang ---- - python/qemu/machine/machine.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py -index ea9e07805df10a57115dace06885..b1dd77b53885629eba452cdd1bc2 100644 ---- a/python/qemu/machine/machine.py -+++ b/python/qemu/machine/machine.py -@@ -141,10 +141,12 @@ class QEMUMachine: - - if monitor_address is not None: - self._monitor_address = monitor_address -+ self._remove_monitor_sockfile = False - else: - self._monitor_address = os.path.join( - self.sock_dir, f"{self._name}-monitor.sock" - ) -+ self._remove_monitor_sockfile = True - - self._console_log_path = console_log - if self._console_log_path: -@@ -313,7 +315,8 @@ class QEMUMachine: - self._remove_files.append(self._console_address) - - if self._qmp_set: -- if isinstance(self._monitor_address, str): -+ if self._remove_monitor_sockfile: -+ assert isinstance(self._monitor_address, str) - self._remove_files.append(self._monitor_address) - self._qmp_connection = QEMUMonitorProtocol( - self._monitor_address, diff --git a/bundles.tar.xz b/bundles.tar.xz index 21cf82db..23016b2f 100644 --- a/bundles.tar.xz +++ b/bundles.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5b5a6e9d7d9c66f8f445153de1b4665c8abede132da0dae6c4231fac0cba49dc -size 98764 +oid sha256:0ecf0e91f78b91cabf0df72e0dd9c54c9d2d016e581abd8364eea6cc6103df94 +size 135948 diff --git a/numa-Support-SGX-numa-in-the-monitor-and.patch b/numa-Support-SGX-numa-in-the-monitor-and.patch index 02783741..efea423b 100644 --- a/numa-Support-SGX-numa-in-the-monitor-and.patch +++ b/numa-Support-SGX-numa-in-the-monitor-and.patch @@ -33,7 +33,6 @@ The QMP interface show: Signed-off-by: Yang Zhong Message-Id: <20211101162009.62161-4-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini -(cherry picked from commit 4755927ae12547c2e7cb22c5fa1b39038c6c11b1) Signed-off-by: Li Zhang --- hw/i386/sgx.c | 51 +++++++++++++++++++++++++++++++++++-------- diff --git a/python-aqmp-Fix-negotiation-with-pre-oob.patch b/python-aqmp-Fix-negotiation-with-pre-oob.patch new file mode 100644 index 00000000..ca589640 --- /dev/null +++ b/python-aqmp-Fix-negotiation-with-pre-oob.patch @@ -0,0 +1,36 @@ +From: John Snow +Date: Mon, 31 Jan 2022 23:11:31 -0500 +Subject: python/aqmp: Fix negotiation with pre-"oob" QEMU + +Git-commit: fa73e6e4ca1a93c5bbf9d05fb2a25736ab810b35 + +QEMU versions prior to the "oob" capability *also* can't accept the +"enable" keyword argument at all. Fix the handshake process with older +QEMU versions. + +Signed-off-by: John Snow +Reviewed-by: Hanna Reitz +Reviewed-by: Kevin Wolf +Message-id: 20220201041134.1237016-2-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/qmp_client.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py +index 8105e29fa8f04297ec9390ec25ea..6b43e1dbbe38eded19fd0115e8bc 100644 +--- a/python/qemu/aqmp/qmp_client.py ++++ b/python/qemu/aqmp/qmp_client.py +@@ -292,9 +292,9 @@ class QMPClient(AsyncProtocol[Message], Events): + """ + self.logger.debug("Negotiating capabilities ...") + +- arguments: Dict[str, List[str]] = {'enable': []} ++ arguments: Dict[str, List[str]] = {} + if self._greeting and 'oob' in self._greeting.QMP.capabilities: +- arguments['enable'].append('oob') ++ arguments.setdefault('enable', []).append('oob') + msg = self.make_execute_msg('qmp_capabilities', arguments=arguments) + + # It's not safe to use execute() here, because the reader/writers diff --git a/python-aqmp-add-SocketAddrT-to-package-r.patch b/python-aqmp-add-SocketAddrT-to-package-r.patch new file mode 100644 index 00000000..c6368c41 --- /dev/null +++ b/python-aqmp-add-SocketAddrT-to-package-r.patch @@ -0,0 +1,42 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:48 -0500 +Subject: python/aqmp: add SocketAddrT to package root + +Git-commit: 728dcac5e356ce5b948943f21c0c72a1b2d96122 + +It's a commonly needed definition, it can be re-exported by the root. + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/__init__.py | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py +index 880d5b6fa7f2c966542b12e25571..c6fa2dda58fdf4f1a867e677eadb 100644 +--- a/python/qemu/aqmp/__init__.py ++++ b/python/qemu/aqmp/__init__.py +@@ -26,7 +26,12 @@ import logging + from .error import AQMPError + from .events import EventListener + from .message import Message +-from .protocol import ConnectError, Runstate, StateError ++from .protocol import ( ++ ConnectError, ++ Runstate, ++ SocketAddrT, ++ StateError, ++) + from .qmp_client import ExecInterruptedError, ExecuteError, QMPClient + + +@@ -48,4 +53,7 @@ __all__ = ( + 'ConnectError', + 'ExecuteError', + 'ExecInterruptedError', ++ ++ # Type aliases ++ 'SocketAddrT', + ) diff --git a/python-aqmp-add-__del__-method-to-legacy.patch b/python-aqmp-add-__del__-method-to-legacy.patch new file mode 100644 index 00000000..1c276329 --- /dev/null +++ b/python-aqmp-add-__del__-method-to-legacy.patch @@ -0,0 +1,64 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:45 -0500 +Subject: python/aqmp: add __del__ method to legacy interface + +Git-commit: 3bc72e3aed76e0326703db81964b13f1da075cbf + +asyncio can complain *very* loudly if you forget to back out of things +gracefully before the garbage collector starts destroying objects that +contain live references to asyncio Tasks. + +The usual fix is just to remember to call aqmp.disconnect(), but for the +sake of the legacy wrapper and quick, one-off scripts where a graceful +shutdown is not necessarily of paramount imporance, add a courtesy +cleanup that will trigger prior to seeing screenfuls of confusing +asyncio tracebacks. + +Note that we can't *always* save you from yourself; depending on when +the GC runs, you might just seriously be out of luck. The best we can do +in this case is to gently remind you to clean up after yourself. + +(Still much better than multiple pages of incomprehensible python +warnings for the crime of forgetting to put your toys away.) + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 9e7b9fb80b95ad5d442fea0dbbab..2ccb136b02c1fcf586507400c056 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -16,6 +16,8 @@ from typing import ( + import qemu.qmp + from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT + ++from .error import AQMPError ++from .protocol import Runstate + from .qmp_client import QMPClient + + +@@ -136,3 +138,19 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + + def send_fd_scm(self, fd: int) -> None: + self._aqmp.send_fd_scm(fd) ++ ++ def __del__(self) -> None: ++ if self._aqmp.runstate == Runstate.IDLE: ++ return ++ ++ if not self._aloop.is_running(): ++ self.close() ++ else: ++ # Garbage collection ran while the event loop was running. ++ # Nothing we can do about it now, but if we don't raise our ++ # own error, the user will be treated to a lot of traceback ++ # they might not understand. ++ raise AQMPError( ++ "QEMUMonitorProtocol.close()" ++ " was not called before object was garbage collected" ++ ) diff --git a/python-aqmp-add-_session_guard.patch b/python-aqmp-add-_session_guard.patch new file mode 100644 index 00000000..f6bb696f --- /dev/null +++ b/python-aqmp-add-_session_guard.patch @@ -0,0 +1,140 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:39 -0500 +Subject: python/aqmp: add _session_guard() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 40196c23939758abc5300e85333e676196e3ba6d + +In _new_session, there's a fairly complex except clause that's used to +give semantic errors to callers of accept() and connect(). We need to +create a new two-step replacement for accept(), so factoring out this +piece of logic will be useful. + +Bolster the comments and docstring here to try and demystify what's +going on in this fairly delicate piece of Python magic. + +(If we were using Python 3.7+, this would be an @asynccontextmanager. We +don't have that very nice piece of magic, however, so this must take an +Awaitable to manage the Exception contexts properly. We pay the price +for platform compatibility.) + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-2-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 89 +++++++++++++++++++++++++----------- + 1 file changed, 62 insertions(+), 27 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 33358f5cd72b61bd060b8dea6091..009883f64d011e44dd003e9dcde3 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -317,6 +317,62 @@ class AsyncProtocol(Generic[T]): + # Section: Session machinery + # -------------------------- + ++ async def _session_guard(self, coro: Awaitable[None], emsg: str) -> None: ++ """ ++ Async guard function used to roll back to `IDLE` on any error. ++ ++ On any Exception, the state machine will be reset back to ++ `IDLE`. Most Exceptions will be wrapped with `ConnectError`, but ++ `BaseException` events will be left alone (This includes ++ asyncio.CancelledError, even prior to Python 3.8). ++ ++ :param error_message: ++ Human-readable string describing what connection phase failed. ++ ++ :raise BaseException: ++ When `BaseException` occurs in the guarded block. ++ :raise ConnectError: ++ When any other error is encountered in the guarded block. ++ """ ++ # Note: After Python 3.6 support is removed, this should be an ++ # @asynccontextmanager instead of accepting a callback. ++ try: ++ await coro ++ except BaseException as err: ++ self.logger.error("%s: %s", emsg, exception_summary(err)) ++ self.logger.debug("%s:\n%s\n", emsg, pretty_traceback()) ++ try: ++ # Reset the runstate back to IDLE. ++ await self.disconnect() ++ except: ++ # We don't expect any Exceptions from the disconnect function ++ # here, because we failed to connect in the first place. ++ # The disconnect() function is intended to perform ++ # only cannot-fail cleanup here, but you never know. ++ emsg = ( ++ "Unexpected bottom half exception. " ++ "This is a bug in the QMP library. " ++ "Please report it to and " ++ "CC: John Snow ." ++ ) ++ self.logger.critical("%s:\n%s\n", emsg, pretty_traceback()) ++ raise ++ ++ # CancelledError is an Exception with special semantic meaning; ++ # We do NOT want to wrap it up under ConnectError. ++ # NB: CancelledError is not a BaseException before Python 3.8 ++ if isinstance(err, asyncio.CancelledError): ++ raise ++ ++ # Any other kind of error can be treated as some kind of connection ++ # failure broadly. Inspect the 'exc' field to explore the root ++ # cause in greater detail. ++ if isinstance(err, Exception): ++ raise ConnectError(emsg, err) from err ++ ++ # Raise BaseExceptions un-wrapped, they're more important. ++ raise ++ + @property + def _runstate_event(self) -> asyncio.Event: + # asyncio.Event() objects should not be created prior to entrance into +@@ -371,34 +427,13 @@ class AsyncProtocol(Generic[T]): + """ + assert self.runstate == Runstate.IDLE + +- try: +- phase = "connection" +- await self._establish_connection(address, ssl, accept) +- +- phase = "session" +- await self._establish_session() ++ await self._session_guard( ++ self._establish_connection(address, ssl, accept), ++ 'Failed to establish connection') + +- except BaseException as err: +- emsg = f"Failed to establish {phase}" +- self.logger.error("%s: %s", emsg, exception_summary(err)) +- self.logger.debug("%s:\n%s\n", emsg, pretty_traceback()) +- try: +- # Reset from CONNECTING back to IDLE. +- await self.disconnect() +- except: +- emsg = "Unexpected bottom half exception" +- self.logger.critical("%s:\n%s\n", emsg, pretty_traceback()) +- raise +- +- # NB: CancelledError is not a BaseException before Python 3.8 +- if isinstance(err, asyncio.CancelledError): +- raise +- +- if isinstance(err, Exception): +- raise ConnectError(emsg, err) from err +- +- # Raise BaseExceptions un-wrapped, they're more important. +- raise ++ await self._session_guard( ++ self._establish_session(), ++ 'Failed to establish session') + + assert self.runstate == Runstate.RUNNING + diff --git a/python-aqmp-add-socket-bind-step-to-lega.patch b/python-aqmp-add-socket-bind-step-to-lega.patch new file mode 100644 index 00000000..c0a6b792 --- /dev/null +++ b/python-aqmp-add-socket-bind-step-to-lega.patch @@ -0,0 +1,135 @@ +From: John Snow +Date: Mon, 31 Jan 2022 23:11:34 -0500 +Subject: python/aqmp: add socket bind step to legacy.py + +Git-commit: b0b662bb2b340d63529672b5bdae596a6243c4d0 + +The synchronous QMP library would bind to the server address during +__init__(). The new library delays this to the accept() call, because +binding occurs inside of the call to start_[unix_]server(), which is an +async method -- so it cannot happen during __init__ anymore. + +Python 3.7+ adds the ability to create the server (and thus the bind() +call) and begin the active listening in separate steps, but we don't +have that functionality in 3.6, our current minimum. + +Therefore ... Add a temporary workaround that allows the synchronous +version of the client to bind the socket in advance, guaranteeing that +there will be a UNIX socket in the filesystem ready for the QEMU client +to connect to without a race condition. + +(Yes, it's a bit ugly. Fixing it more nicely will have to wait until our +minimum Python version is 3.7+.) + +Signed-off-by: John Snow +Reviewed-by: Kevin Wolf +Message-id: 20220201041134.1237016-5-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 3 +++ + python/qemu/aqmp/protocol.py | 41 +++++++++++++++++++++++++++++++++--- + 2 files changed, 41 insertions(+), 3 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 0890f95b16875ecb815ed4560bc7..6baa5f3409a6b459c67097d3c2a0 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -56,6 +56,9 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + self._address = address + self._timeout: Optional[float] = None + ++ if server: ++ self._aqmp._bind_hack(address) # pylint: disable=protected-access ++ + _T = TypeVar('_T') + + def _sync( +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 50e973c2f2dc9c5fa759380ab3e9..33358f5cd72b61bd060b8dea6091 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -15,6 +15,7 @@ from asyncio import StreamReader, StreamWriter + from enum import Enum + from functools import wraps + import logging ++import socket + from ssl import SSLContext + from typing import ( + Any, +@@ -238,6 +239,9 @@ class AsyncProtocol(Generic[T]): + self._runstate = Runstate.IDLE + self._runstate_changed: Optional[asyncio.Event] = None + ++ # Workaround for bind() ++ self._sock: Optional[socket.socket] = None ++ + def __repr__(self) -> str: + cls_name = type(self).__name__ + tokens = [] +@@ -427,6 +431,34 @@ class AsyncProtocol(Generic[T]): + else: + await self._do_connect(address, ssl) + ++ def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None: ++ """ ++ Used to create a socket in advance of accept(). ++ ++ This is a workaround to ensure that we can guarantee timing of ++ precisely when a socket exists to avoid a connection attempt ++ bouncing off of nothing. ++ ++ Python 3.7+ adds a feature to separate the server creation and ++ listening phases instead, and should be used instead of this ++ hack. ++ """ ++ if isinstance(address, tuple): ++ family = socket.AF_INET ++ else: ++ family = socket.AF_UNIX ++ ++ sock = socket.socket(family, socket.SOCK_STREAM) ++ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ++ ++ try: ++ sock.bind(address) ++ except: ++ sock.close() ++ raise ++ ++ self._sock = sock ++ + @upper_half + async def _do_accept(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: +@@ -464,24 +496,27 @@ class AsyncProtocol(Generic[T]): + if isinstance(address, tuple): + coro = asyncio.start_server( + _client_connected_cb, +- host=address[0], +- port=address[1], ++ host=None if self._sock else address[0], ++ port=None if self._sock else address[1], + ssl=ssl, + backlog=1, + limit=self._limit, ++ sock=self._sock, + ) + else: + coro = asyncio.start_unix_server( + _client_connected_cb, +- path=address, ++ path=None if self._sock else address, + ssl=ssl, + backlog=1, + limit=self._limit, ++ sock=self._sock, + ) + + server = await coro # Starts listening + await connected.wait() # Waits for the callback to fire (and finish) + assert server is None ++ self._sock = None + + self.logger.debug("Connection accepted.") + diff --git a/python-aqmp-add-start_server-and-accept-.patch b/python-aqmp-add-start_server-and-accept-.patch new file mode 100644 index 00000000..ca98d7b2 --- /dev/null +++ b/python-aqmp-add-start_server-and-accept-.patch @@ -0,0 +1,158 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:46 -0500 +Subject: python/aqmp: add start_server() and accept() methods +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 481607c7d35de2bc4d9bec7f4734036fc467f330 + +Add start_server() and accept() methods that can be used instead of +start_server_and_accept() to allow more fine-grained control over the +incoming connection process. + +(Eagle-eyed reviewers will surely notice that it's a bit weird that +"CONNECTING" is a state that's shared between both the start_server() +and connect() states. That's absolutely true, and it's very true that +checking on the presence of _accepted as an indicator of state is a +hack. That's also very certainly true. But ... this keeps client code an +awful lot simpler, as it doesn't have to care exactly *how* the +connection is being made, just that it *is*. Is it worth disrupting that +simplicity in order to provide a better state guard on `accept()`? Hm.) + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-9-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 67 +++++++++++++++++++++++++++++++++--- + python/tests/protocol.py | 7 ++++ + 2 files changed, 69 insertions(+), 5 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index cdbc9cba0d15dea19cc1c60ca3c3..2ecba1455571a35e0e6c565e3641 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -280,6 +280,8 @@ class AsyncProtocol(Generic[T]): + Accept a connection and begin processing message queues. + + If this call fails, `runstate` is guaranteed to be set back to `IDLE`. ++ This method is precisely equivalent to calling `start_server()` ++ followed by `accept()`. + + :param address: + Address to listen on; UNIX socket path or TCP address/port. +@@ -294,9 +296,62 @@ class AsyncProtocol(Generic[T]): + protocol-level failure occurs while establishing a new + session, the wrapped error may also be an `QMPError`. + """ ++ await self.start_server(address, ssl) ++ await self.accept() ++ assert self.runstate == Runstate.RUNNING ++ ++ @upper_half ++ @require(Runstate.IDLE) ++ async def start_server(self, address: SocketAddrT, ++ ssl: Optional[SSLContext] = None) -> None: ++ """ ++ Start listening for an incoming connection, but do not wait for a peer. ++ ++ This method starts listening for an incoming connection, but ++ does not block waiting for a peer. This call will return ++ immediately after binding and listening on a socket. A later ++ call to `accept()` must be made in order to finalize the ++ incoming connection. ++ ++ :param address: ++ Address to listen on; UNIX socket path or TCP address/port. ++ :param ssl: SSL context to use, if any. ++ ++ :raise StateError: When the `Runstate` is not `IDLE`. ++ :raise ConnectError: ++ When the server could not start listening on this address. ++ ++ This exception will wrap a more concrete one. In most cases, ++ the wrapped exception will be `OSError`. ++ """ + await self._session_guard( + self._do_start_server(address, ssl), + 'Failed to establish connection') ++ assert self.runstate == Runstate.CONNECTING ++ ++ @upper_half ++ @require(Runstate.CONNECTING) ++ async def accept(self) -> None: ++ """ ++ Accept an incoming connection and begin processing message queues. ++ ++ If this call fails, `runstate` is guaranteed to be set back to `IDLE`. ++ ++ :raise StateError: When the `Runstate` is not `CONNECTING`. ++ :raise QMPError: When `start_server()` was not called yet. ++ :raise ConnectError: ++ When a connection or session cannot be established. ++ ++ This exception will wrap a more concrete one. In most cases, ++ the wrapped exception will be `OSError` or `EOFError`. If a ++ protocol-level failure occurs while establishing a new ++ session, the wrapped error may also be an `QMPError`. ++ """ ++ if self._accepted is None: ++ raise QMPError("Cannot call accept() before start_server().") ++ await self._session_guard( ++ self._do_accept(), ++ 'Failed to establish connection') + await self._session_guard( + self._establish_session(), + 'Failed to establish session') +@@ -512,7 +567,12 @@ class AsyncProtocol(Generic[T]): + async def _do_start_server(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: + """ +- Acting as the transport server, accept a single connection. ++ Start listening for an incoming connection, but do not wait for a peer. ++ ++ This method starts listening for an incoming connection, but does not ++ block waiting for a peer. This call will return immediately after ++ binding and listening to a socket. A later call to accept() must be ++ made in order to finalize the incoming connection. + + :param address: + Address to listen on; UNIX socket path or TCP address/port. +@@ -554,10 +614,7 @@ class AsyncProtocol(Generic[T]): + # This will start the server (bind(2), listen(2)). It will also + # call accept(2) if we yield, but we don't block on that here. + self._server = await coro +- +- # Just for this one commit, wait for a peer. +- # This gets split out in the next patch. +- await self._do_accept() ++ self.logger.debug("Server listening on %s", address) + + @upper_half + async def _do_accept(self) -> None: +diff --git a/python/tests/protocol.py b/python/tests/protocol.py +index 5e442e1efbd19bf95a95de371060..d6849ad3062081c62b29bf89dd2a 100644 +--- a/python/tests/protocol.py ++++ b/python/tests/protocol.py +@@ -43,11 +43,18 @@ class NullProtocol(AsyncProtocol[None]): + + async def _do_start_server(self, address, ssl=None): + if self.fake_session: ++ self._accepted = asyncio.Event() + self._set_state(Runstate.CONNECTING) + await asyncio.sleep(0) + else: + await super()._do_start_server(address, ssl) + ++ async def _do_accept(self): ++ if self.fake_session: ++ self._accepted = None ++ else: ++ await super()._do_accept() ++ + async def _do_connect(self, address, ssl=None): + if self.fake_session: + self._set_state(Runstate.CONNECTING) diff --git a/python-aqmp-copy-type-definitions-from-q.patch b/python-aqmp-copy-type-definitions-from-q.patch new file mode 100644 index 00000000..f4e837f1 --- /dev/null +++ b/python-aqmp-copy-type-definitions-from-q.patch @@ -0,0 +1,134 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:47 -0500 +Subject: python/aqmp: copy type definitions from qmp + +Git-commit: 0e6bfd8b96e407db7b0cb5e8c14cc315a7154f53 + +Copy the remaining type definitions from QMP into the qemu.aqmp.legacy +module. Now, users that require the legacy interface don't need to +import anything else but qemu.aqmp.legacy wrapper. + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 22 ++++++++++++++++++++-- + python/qemu/aqmp/protocol.py | 16 ++++++++++------ + 2 files changed, 30 insertions(+), 8 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 2ccb136b02c1fcf586507400c056..9431fe933019b1e1c221ea3ab7bb 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -6,7 +6,9 @@ This class pretends to be qemu.qmp.QEMUMonitorProtocol. + + import asyncio + from typing import ( ++ Any, + Awaitable, ++ Dict, + List, + Optional, + TypeVar, +@@ -14,13 +16,29 @@ from typing import ( + ) + + import qemu.qmp +-from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT + + from .error import AQMPError +-from .protocol import Runstate ++from .protocol import Runstate, SocketAddrT + from .qmp_client import QMPClient + + ++#: QMPMessage is an entire QMP message of any kind. ++QMPMessage = Dict[str, Any] ++ ++#: QMPReturnValue is the 'return' value of a command. ++QMPReturnValue = object ++ ++#: QMPObject is any object in a QMP message. ++QMPObject = Dict[str, object] ++ ++# QMPMessage can be outgoing commands or incoming events/returns. ++# QMPReturnValue is usually a dict/json object, but due to QAPI's ++# 'returns-whitelist', it can actually be anything. ++# ++# {'return': {}} is a QMPMessage, ++# {} is the QMPReturnValue. ++ ++ + # pylint: disable=missing-docstring + + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index c4fbe35a0e41c589059ec4fa37a8..5b4f2f0d0a81a0d2902358e9b799 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -46,6 +46,10 @@ T = TypeVar('T') + _U = TypeVar('_U') + _TaskFN = Callable[[], Awaitable[None]] # aka ``async def func() -> None`` + ++InternetAddrT = Tuple[str, int] ++UnixAddrT = str ++SocketAddrT = Union[UnixAddrT, InternetAddrT] ++ + + class Runstate(Enum): + """Protocol session runstate.""" +@@ -257,7 +261,7 @@ class AsyncProtocol(Generic[T]): + + @upper_half + @require(Runstate.IDLE) +- async def accept(self, address: Union[str, Tuple[str, int]], ++ async def accept(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: + """ + Accept a connection and begin processing message queues. +@@ -275,7 +279,7 @@ class AsyncProtocol(Generic[T]): + + @upper_half + @require(Runstate.IDLE) +- async def connect(self, address: Union[str, Tuple[str, int]], ++ async def connect(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: + """ + Connect to the server and begin processing message queues. +@@ -337,7 +341,7 @@ class AsyncProtocol(Generic[T]): + + @upper_half + async def _new_session(self, +- address: Union[str, Tuple[str, int]], ++ address: SocketAddrT, + ssl: Optional[SSLContext] = None, + accept: bool = False) -> None: + """ +@@ -397,7 +401,7 @@ class AsyncProtocol(Generic[T]): + @upper_half + async def _establish_connection( + self, +- address: Union[str, Tuple[str, int]], ++ address: SocketAddrT, + ssl: Optional[SSLContext] = None, + accept: bool = False + ) -> None: +@@ -424,7 +428,7 @@ class AsyncProtocol(Generic[T]): + await self._do_connect(address, ssl) + + @upper_half +- async def _do_accept(self, address: Union[str, Tuple[str, int]], ++ async def _do_accept(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: + """ + Acting as the transport server, accept a single connection. +@@ -482,7 +486,7 @@ class AsyncProtocol(Generic[T]): + self.logger.debug("Connection accepted.") + + @upper_half +- async def _do_connect(self, address: Union[str, Tuple[str, int]], ++ async def _do_connect(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: + """ + Acting as the transport client, initiate a connection to a server. diff --git a/python-aqmp-drop-_bind_hack.patch b/python-aqmp-drop-_bind_hack.patch new file mode 100644 index 00000000..b0414c02 --- /dev/null +++ b/python-aqmp-drop-_bind_hack.patch @@ -0,0 +1,132 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:48 -0500 +Subject: python/aqmp: drop _bind_hack() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 4c1fe7003c9b373acb0791b4356e2285a10365c0 + +_bind_hack() was a quick fix to allow async QMP to call bind(2) prior to +calling listen(2) and accept(2). This wasn't sufficient to fully address +the race condition present in synchronous clients. + +With the race condition in legacy.py fixed (see the previous commit), +there are no longer any users of _bind_hack(). Drop it. + +Fixes: b0b662bb2b3 +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-11-jsnow@redhat.com +[Expanded commit message. --js] +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 2 +- + python/qemu/aqmp/protocol.py | 41 +++--------------------------------- + 2 files changed, 4 insertions(+), 39 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index cb50e60564823fdc0aeeb194c5e3..46026e9fdc6c8859a8c944076c71 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -57,7 +57,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + self._timeout: Optional[float] = None + + if server: +- self._sync(self._aqmp.start_server(address)) ++ self._sync(self._aqmp.start_server(self._address)) + + _T = TypeVar('_T') + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 2ecba1455571a35e0e6c565e3641..36fae57f277826d5cd4026e0d079 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -18,7 +18,6 @@ from asyncio import StreamReader, StreamWriter + from enum import Enum + from functools import wraps + import logging +-import socket + from ssl import SSLContext + from typing import ( + Any, +@@ -242,9 +241,6 @@ class AsyncProtocol(Generic[T]): + self._runstate = Runstate.IDLE + self._runstate_changed: Optional[asyncio.Event] = None + +- # Workaround for bind() +- self._sock: Optional[socket.socket] = None +- + # Server state for start_server() and _incoming() + self._server: Optional[asyncio.AbstractServer] = None + self._accepted: Optional[asyncio.Event] = None +@@ -535,34 +531,6 @@ class AsyncProtocol(Generic[T]): + self._reader, self._writer = (reader, writer) + self._accepted.set() + +- def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None: +- """ +- Used to create a socket in advance of accept(). +- +- This is a workaround to ensure that we can guarantee timing of +- precisely when a socket exists to avoid a connection attempt +- bouncing off of nothing. +- +- Python 3.7+ adds a feature to separate the server creation and +- listening phases instead, and should be used instead of this +- hack. +- """ +- if isinstance(address, tuple): +- family = socket.AF_INET +- else: +- family = socket.AF_UNIX +- +- sock = socket.socket(family, socket.SOCK_STREAM) +- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +- +- try: +- sock.bind(address) +- except: +- sock.close() +- raise +- +- self._sock = sock +- + @upper_half + async def _do_start_server(self, address: SocketAddrT, + ssl: Optional[SSLContext] = None) -> None: +@@ -589,21 +557,19 @@ class AsyncProtocol(Generic[T]): + if isinstance(address, tuple): + coro = asyncio.start_server( + self._incoming, +- host=None if self._sock else address[0], +- port=None if self._sock else address[1], ++ host=address[0], ++ port=address[1], + ssl=ssl, + backlog=1, + limit=self._limit, +- sock=self._sock, + ) + else: + coro = asyncio.start_unix_server( + self._incoming, +- path=None if self._sock else address, ++ path=address, + ssl=ssl, + backlog=1, + limit=self._limit, +- sock=self._sock, + ) + + # Allow runstate watchers to witness 'CONNECTING' state; some +@@ -630,7 +596,6 @@ class AsyncProtocol(Generic[T]): + await self._accepted.wait() + assert self._server is None + self._accepted = None +- self._sock = None + + self.logger.debug("Connection accepted.") + diff --git a/python-aqmp-fix-docstring-typo.patch b/python-aqmp-fix-docstring-typo.patch new file mode 100644 index 00000000..3095afa0 --- /dev/null +++ b/python-aqmp-fix-docstring-typo.patch @@ -0,0 +1,27 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:44 -0500 +Subject: python/aqmp: fix docstring typo + +Git-commit: dc6877bd2ea04a38700adcea2359d5d20c1082a6 + +Reported-by: Vladimir Sementsov-Ogievskiy +Signed-off-by: John Snow +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/__init__.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py +index c6fa2dda58fdf4f1a867e677eadb..05f467c1415360169e9c66c8d27e 100644 +--- a/python/qemu/aqmp/__init__.py ++++ b/python/qemu/aqmp/__init__.py +@@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by QEMU, the + QEMU Guest Agent, and the QEMU Storage Daemon. + + `QMPClient` provides the main functionality of this package. All errors +-raised by this library dervive from `AQMPError`, see `aqmp.error` for ++raised by this library derive from `AQMPError`, see `aqmp.error` for + additional detail. See `aqmp.events` for an in-depth tutorial on + managing QMP events. + """ diff --git a/python-aqmp-fix-race-condition-in-legacy.patch b/python-aqmp-fix-race-condition-in-legacy.patch new file mode 100644 index 00000000..11243712 --- /dev/null +++ b/python-aqmp-fix-race-condition-in-legacy.patch @@ -0,0 +1,59 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:47 -0500 +Subject: python/aqmp: fix race condition in legacy.py +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 673856f9d889dc50b6a1a7964df960c4f00c7c93 + +legacy.py provides a synchronous model. iotests frequently uses this +paradigm: + + - create QMP client object + - start QEMU process + - await connection from QEMU process + +In the switch from sync to async QMP, the QMP client object stopped +calling bind() and listen() during the QMP object creation step, which +creates a race condition if the QEMU process dials in too quickly. + +With refactoring out of the way, restore the former behavior of calling +bind() and listen() during __init__() to fix this race condition. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-10-jsnow@redhat.com +[Expanded commit message. --js] +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 7 ++----- + 1 file changed, 2 insertions(+), 5 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index dca1e76ed4994959caf542031363..cb50e60564823fdc0aeeb194c5e3 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -57,7 +57,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + self._timeout: Optional[float] = None + + if server: +- self._aqmp._bind_hack(address) # pylint: disable=protected-access ++ self._sync(self._aqmp.start_server(address)) + + _T = TypeVar('_T') + +@@ -90,10 +90,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + self._aqmp.await_greeting = True + self._aqmp.negotiate = True + +- self._sync( +- self._aqmp.start_server_and_accept(self._address), +- timeout +- ) ++ self._sync(self._aqmp.accept(), timeout) + + ret = self._get_greeting() + assert ret is not None diff --git a/python-aqmp-handle-asyncio.TimeoutError-.patch b/python-aqmp-handle-asyncio.TimeoutError-.patch new file mode 100644 index 00000000..1a2c21cf --- /dev/null +++ b/python-aqmp-handle-asyncio.TimeoutError-.patch @@ -0,0 +1,46 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:46 -0500 +Subject: python/aqmp: handle asyncio.TimeoutError on execute() + +Git-commit: 3b5bf136f5798a4ea2c66875d6337ca3d6b79434 + +This exception can be injected into any await statement. If we are +canceled via timeout, we want to clear the pending execution record on +our way out. + +Signed-off-by: John Snow +Reviewed-by: Beraldo Leal +Reviewed-by: Vladimir Sementsov-Ogievskiy +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/qmp_client.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py +index 6b43e1dbbe38eded19fd0115e8bc..45864f288e4fe5a7505c0022ed13 100644 +--- a/python/qemu/aqmp/qmp_client.py ++++ b/python/qemu/aqmp/qmp_client.py +@@ -435,7 +435,11 @@ class QMPClient(AsyncProtocol[Message], Events): + msg_id = msg['id'] + + self._pending[msg_id] = asyncio.Queue(maxsize=1) +- await self._outgoing.put(msg) ++ try: ++ await self._outgoing.put(msg) ++ except: ++ del self._pending[msg_id] ++ raise + + return msg_id + +@@ -452,9 +456,9 @@ class QMPClient(AsyncProtocol[Message], Events): + was lost, or some other problem. + """ + queue = self._pending[msg_id] +- result = await queue.get() + + try: ++ result = await queue.get() + if isinstance(result, ExecInterruptedError): + raise result + return result diff --git a/python-aqmp-refactor-_do_accept-into-two.patch b/python-aqmp-refactor-_do_accept-into-two.patch new file mode 100644 index 00000000..20770868 --- /dev/null +++ b/python-aqmp-refactor-_do_accept-into-two.patch @@ -0,0 +1,102 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:44 -0500 +Subject: python/aqmp: refactor _do_accept() into two distinct steps +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 5e9902a030ab832b0b6577764c65ce6a6f874af6 + +Refactor _do_accept() into _do_start_server() and _do_accept(). As of +this commit, the former calls the latter, but in subsequent commits +they'll be split apart. + +(So please forgive the misnomer for _do_start_server(); it will live up +to its name shortly, and the docstring will be updated then too. I'm +just cutting down on some churn.) + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-7-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 29 ++++++++++++++++++++++++----- + python/tests/protocol.py | 4 ++-- + 2 files changed, 26 insertions(+), 7 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 631bcdaa554f4a104af4e25a3c61..e2bdad542dc0ef451dd200a1d679 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -295,7 +295,7 @@ class AsyncProtocol(Generic[T]): + session, the wrapped error may also be an `QMPError`. + """ + await self._session_guard( +- self._do_accept(address, ssl), ++ self._do_start_server(address, ssl), + 'Failed to establish connection') + await self._session_guard( + self._establish_session(), +@@ -509,8 +509,8 @@ class AsyncProtocol(Generic[T]): + self._sock = sock + + @upper_half +- async def _do_accept(self, address: SocketAddrT, +- ssl: Optional[SSLContext] = None) -> None: ++ async def _do_start_server(self, address: SocketAddrT, ++ ssl: Optional[SSLContext] = None) -> None: + """ + Acting as the transport server, accept a single connection. + +@@ -551,9 +551,28 @@ class AsyncProtocol(Generic[T]): + # otherwise yield. + await asyncio.sleep(0) + +- self._server = await coro # Starts listening +- await self._accepted.wait() # Waits for the callback to finish ++ # This will start the server (bind(2), listen(2)). It will also ++ # call accept(2) if we yield, but we don't block on that here. ++ self._server = await coro ++ ++ # Just for this one commit, wait for a peer. ++ # This gets split out in the next patch. ++ await self._do_accept() ++ ++ @upper_half ++ async def _do_accept(self) -> None: ++ """ ++ Wait for and accept an incoming connection. ++ ++ Requires that we have not yet accepted an incoming connection ++ from the upper_half, but it's OK if the server is no longer ++ running because the bottom_half has already accepted the ++ connection. ++ """ ++ assert self._accepted is not None ++ await self._accepted.wait() + assert self._server is None ++ self._accepted = None + self._sock = None + + self.logger.debug("Connection accepted.") +diff --git a/python/tests/protocol.py b/python/tests/protocol.py +index 8dd26c4ed1e0973b8058604c2373..5e442e1efbd19bf95a95de371060 100644 +--- a/python/tests/protocol.py ++++ b/python/tests/protocol.py +@@ -41,12 +41,12 @@ class NullProtocol(AsyncProtocol[None]): + self.trigger_input = asyncio.Event() + await super()._establish_session() + +- async def _do_accept(self, address, ssl=None): ++ async def _do_start_server(self, address, ssl=None): + if self.fake_session: + self._set_state(Runstate.CONNECTING) + await asyncio.sleep(0) + else: +- await super()._do_accept(address, ssl) ++ await super()._do_start_server(address, ssl) + + async def _do_connect(self, address, ssl=None): + if self.fake_session: diff --git a/python-aqmp-remove-_new_session-and-_est.patch b/python-aqmp-remove-_new_session-and-_est.patch new file mode 100644 index 00000000..0558ce78 --- /dev/null +++ b/python-aqmp-remove-_new_session-and-_est.patch @@ -0,0 +1,231 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:41 -0500 +Subject: python/aqmp: remove _new_session and _establish_connection +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 68a6cf3ffe3532c0655efbbf5910bd99a1b4a3fa + +These two methods attempted to entirely envelop the logic of +establishing a connection to a peer start to finish. However, we need to +break apart the incoming connection step into more granular steps. We +will no longer be able to reasonably constrain the logic inside of these +helper functions. + +So, remove them - with _session_guard(), they no longer serve a real +purpose. + +Although the public API doesn't change, the internal API does. Now that +there are no intermediary methods between e.g. connect() and +_do_connect(), there's no hook where the runstate is set. As a result, +the test suite changes a little to cope with the new semantics of +_do_accept() and _do_connect(). + +Lastly, take some pieces of the now-deleted docstrings and move +them up to the public interface level. They were a little more detailed, +and it won't hurt to keep them. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-4-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 117 ++++++++++++++--------------------- + python/tests/protocol.py | 10 ++- + 2 files changed, 53 insertions(+), 74 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 73719257e058b7e9e4d8a281bcd9..b7e5e635d886db0efc85f829f42e 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -275,13 +275,25 @@ class AsyncProtocol(Generic[T]): + If this call fails, `runstate` is guaranteed to be set back to `IDLE`. + + :param address: +- Address to listen to; UNIX socket path or TCP address/port. ++ Address to listen on; UNIX socket path or TCP address/port. + :param ssl: SSL context to use, if any. + + :raise StateError: When the `Runstate` is not `IDLE`. +- :raise ConnectError: If a connection could not be accepted. ++ :raise ConnectError: ++ When a connection or session cannot be established. ++ ++ This exception will wrap a more concrete one. In most cases, ++ the wrapped exception will be `OSError` or `EOFError`. If a ++ protocol-level failure occurs while establishing a new ++ session, the wrapped error may also be an `QMPError`. + """ +- await self._new_session(address, ssl, accept=True) ++ await self._session_guard( ++ self._do_accept(address, ssl), ++ 'Failed to establish connection') ++ await self._session_guard( ++ self._establish_session(), ++ 'Failed to establish session') ++ assert self.runstate == Runstate.RUNNING + + @upper_half + @require(Runstate.IDLE) +@@ -297,9 +309,21 @@ class AsyncProtocol(Generic[T]): + :param ssl: SSL context to use, if any. + + :raise StateError: When the `Runstate` is not `IDLE`. +- :raise ConnectError: If a connection cannot be made to the server. ++ :raise ConnectError: ++ When a connection or session cannot be established. ++ ++ This exception will wrap a more concrete one. In most cases, ++ the wrapped exception will be `OSError` or `EOFError`. If a ++ protocol-level failure occurs while establishing a new ++ session, the wrapped error may also be an `QMPError`. + """ +- await self._new_session(address, ssl) ++ await self._session_guard( ++ self._do_connect(address, ssl), ++ 'Failed to establish connection') ++ await self._session_guard( ++ self._establish_session(), ++ 'Failed to establish session') ++ assert self.runstate == Runstate.RUNNING + + @upper_half + async def disconnect(self) -> None: +@@ -401,73 +425,6 @@ class AsyncProtocol(Generic[T]): + self._runstate_event.set() + self._runstate_event.clear() + +- @upper_half +- async def _new_session(self, +- address: SocketAddrT, +- ssl: Optional[SSLContext] = None, +- accept: bool = False) -> None: +- """ +- Establish a new connection and initialize the session. +- +- Connect or accept a new connection, then begin the protocol +- session machinery. If this call fails, `runstate` is guaranteed +- to be set back to `IDLE`. +- +- :param address: +- Address to connect to/listen on; +- UNIX socket path or TCP address/port. +- :param ssl: SSL context to use, if any. +- :param accept: Accept a connection instead of connecting when `True`. +- +- :raise ConnectError: +- When a connection or session cannot be established. +- +- This exception will wrap a more concrete one. In most cases, +- the wrapped exception will be `OSError` or `EOFError`. If a +- protocol-level failure occurs while establishing a new +- session, the wrapped error may also be an `QMPError`. +- """ +- assert self.runstate == Runstate.IDLE +- +- await self._session_guard( +- self._establish_connection(address, ssl, accept), +- 'Failed to establish connection') +- +- await self._session_guard( +- self._establish_session(), +- 'Failed to establish session') +- +- assert self.runstate == Runstate.RUNNING +- +- @upper_half +- async def _establish_connection( +- self, +- address: SocketAddrT, +- ssl: Optional[SSLContext] = None, +- accept: bool = False +- ) -> None: +- """ +- Establish a new connection. +- +- :param address: +- Address to connect to/listen on; +- UNIX socket path or TCP address/port. +- :param ssl: SSL context to use, if any. +- :param accept: Accept a connection instead of connecting when `True`. +- """ +- assert self.runstate == Runstate.IDLE +- self._set_state(Runstate.CONNECTING) +- +- # Allow runstate watchers to witness 'CONNECTING' state; some +- # failures in the streaming layer are synchronous and will not +- # otherwise yield. +- await asyncio.sleep(0) +- +- if accept: +- await self._do_accept(address, ssl) +- else: +- await self._do_connect(address, ssl) +- + def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None: + """ + Used to create a socket in advance of accept(). +@@ -508,6 +465,9 @@ class AsyncProtocol(Generic[T]): + + :raise OSError: For stream-related errors. + """ ++ assert self.runstate == Runstate.IDLE ++ self._set_state(Runstate.CONNECTING) ++ + self.logger.debug("Awaiting connection on %s ...", address) + connected = asyncio.Event() + server: Optional[asyncio.AbstractServer] = None +@@ -550,6 +510,11 @@ class AsyncProtocol(Generic[T]): + sock=self._sock, + ) + ++ # Allow runstate watchers to witness 'CONNECTING' state; some ++ # failures in the streaming layer are synchronous and will not ++ # otherwise yield. ++ await asyncio.sleep(0) ++ + server = await coro # Starts listening + await connected.wait() # Waits for the callback to fire (and finish) + assert server is None +@@ -569,6 +534,14 @@ class AsyncProtocol(Generic[T]): + + :raise OSError: For stream-related errors. + """ ++ assert self.runstate == Runstate.IDLE ++ self._set_state(Runstate.CONNECTING) ++ ++ # Allow runstate watchers to witness 'CONNECTING' state; some ++ # failures in the streaming layer are synchronous and will not ++ # otherwise yield. ++ await asyncio.sleep(0) ++ + self.logger.debug("Connecting to %s ...", address) + + if isinstance(address, tuple): +diff --git a/python/tests/protocol.py b/python/tests/protocol.py +index 354d6559b9d1e3dc3ad29598af3c..8dd26c4ed1e0973b8058604c2373 100644 +--- a/python/tests/protocol.py ++++ b/python/tests/protocol.py +@@ -42,11 +42,17 @@ class NullProtocol(AsyncProtocol[None]): + await super()._establish_session() + + async def _do_accept(self, address, ssl=None): +- if not self.fake_session: ++ if self.fake_session: ++ self._set_state(Runstate.CONNECTING) ++ await asyncio.sleep(0) ++ else: + await super()._do_accept(address, ssl) + + async def _do_connect(self, address, ssl=None): +- if not self.fake_session: ++ if self.fake_session: ++ self._set_state(Runstate.CONNECTING) ++ await asyncio.sleep(0) ++ else: + await super()._do_connect(address, ssl) + + async def _do_recv(self) -> None: diff --git a/python-aqmp-rename-AQMPError-to-QMPError.patch b/python-aqmp-rename-AQMPError-to-QMPError.patch new file mode 100644 index 00000000..272398f0 --- /dev/null +++ b/python-aqmp-rename-AQMPError-to-QMPError.patch @@ -0,0 +1,221 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:49 -0500 +Subject: python/aqmp: rename AQMPError to QMPError + +Git-commit: 6e7751dc388df6daf425db0e245d4d3a10859803 + +This is in preparation for renaming qemu.aqmp to qemu.qmp. I should have +done this from this from the very beginning, but it's a convenient time +to make sure this churn is taken care of. + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/__init__.py | 6 +++--- + python/qemu/aqmp/error.py | 12 ++++++------ + python/qemu/aqmp/events.py | 4 ++-- + python/qemu/aqmp/legacy.py | 4 ++-- + python/qemu/aqmp/protocol.py | 8 ++++---- + python/qemu/aqmp/qmp_client.py | 8 ++++---- + 6 files changed, 21 insertions(+), 21 deletions(-) + +diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py +index 05f467c1415360169e9c66c8d27e..4c22c380790fd0f86402e402628b 100644 +--- a/python/qemu/aqmp/__init__.py ++++ b/python/qemu/aqmp/__init__.py +@@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by QEMU, the + QEMU Guest Agent, and the QEMU Storage Daemon. + + `QMPClient` provides the main functionality of this package. All errors +-raised by this library derive from `AQMPError`, see `aqmp.error` for ++raised by this library derive from `QMPError`, see `aqmp.error` for + additional detail. See `aqmp.events` for an in-depth tutorial on + managing QMP events. + """ +@@ -23,7 +23,7 @@ managing QMP events. + + import logging + +-from .error import AQMPError ++from .error import QMPError + from .events import EventListener + from .message import Message + from .protocol import ( +@@ -48,7 +48,7 @@ __all__ = ( + 'Runstate', + + # Exceptions, most generic to most explicit +- 'AQMPError', ++ 'QMPError', + 'StateError', + 'ConnectError', + 'ExecuteError', +diff --git a/python/qemu/aqmp/error.py b/python/qemu/aqmp/error.py +index 781f49b00877893d7a88f755c67f..24ba4d505410b5fe56390e3d4e02 100644 +--- a/python/qemu/aqmp/error.py ++++ b/python/qemu/aqmp/error.py +@@ -1,21 +1,21 @@ + """ +-AQMP Error Classes ++QMP Error Classes + + This package seeks to provide semantic error classes that are intended + to be used directly by clients when they would like to handle particular + semantic failures (e.g. "failed to connect") without needing to know the + enumeration of possible reasons for that failure. + +-AQMPError serves as the ancestor for all exceptions raised by this ++QMPError serves as the ancestor for all exceptions raised by this + package, and is suitable for use in handling semantic errors from this + library. In most cases, individual public methods will attempt to catch + and re-encapsulate various exceptions to provide a semantic + error-handling interface. + +-.. admonition:: AQMP Exception Hierarchy Reference ++.. admonition:: QMP Exception Hierarchy Reference + + | `Exception` +- | +-- `AQMPError` ++ | +-- `QMPError` + | +-- `ConnectError` + | +-- `StateError` + | +-- `ExecInterruptedError` +@@ -31,11 +31,11 @@ error-handling interface. + """ + + +-class AQMPError(Exception): ++class QMPError(Exception): + """Abstract error class for all errors originating from this package.""" + + +-class ProtocolError(AQMPError): ++class ProtocolError(QMPError): + """ + Abstract error class for protocol failures. + +diff --git a/python/qemu/aqmp/events.py b/python/qemu/aqmp/events.py +index 5f7150c78d49d9513978103dc9a7..f3d4e2b5e853c39db9e016009db0 100644 +--- a/python/qemu/aqmp/events.py ++++ b/python/qemu/aqmp/events.py +@@ -443,7 +443,7 @@ from typing import ( + Union, + ) + +-from .error import AQMPError ++from .error import QMPError + from .message import Message + + +@@ -451,7 +451,7 @@ EventNames = Union[str, Iterable[str], None] + EventFilter = Callable[[Message], bool] + + +-class ListenerError(AQMPError): ++class ListenerError(QMPError): + """ + Generic error class for `EventListener`-related problems. + """ +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 9431fe933019b1e1c221ea3ab7bb..27df22818a76190e872f08c0852e 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -17,7 +17,7 @@ from typing import ( + + import qemu.qmp + +-from .error import AQMPError ++from .error import QMPError + from .protocol import Runstate, SocketAddrT + from .qmp_client import QMPClient + +@@ -168,7 +168,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + # Nothing we can do about it now, but if we don't raise our + # own error, the user will be treated to a lot of traceback + # they might not understand. +- raise AQMPError( ++ raise QMPError( + "QEMUMonitorProtocol.close()" + " was not called before object was garbage collected" + ) +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 5b4f2f0d0a81a0d2902358e9b799..50e973c2f2dc9c5fa759380ab3e9 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -29,7 +29,7 @@ from typing import ( + cast, + ) + +-from .error import AQMPError ++from .error import QMPError + from .util import ( + bottom_half, + create_task, +@@ -65,7 +65,7 @@ class Runstate(Enum): + DISCONNECTING = 3 + + +-class ConnectError(AQMPError): ++class ConnectError(QMPError): + """ + Raised when the initial connection process has failed. + +@@ -90,7 +90,7 @@ class ConnectError(AQMPError): + return f"{self.error_message}: {cause}" + + +-class StateError(AQMPError): ++class StateError(QMPError): + """ + An API command (connect, execute, etc) was issued at an inappropriate time. + +@@ -363,7 +363,7 @@ class AsyncProtocol(Generic[T]): + This exception will wrap a more concrete one. In most cases, + the wrapped exception will be `OSError` or `EOFError`. If a + protocol-level failure occurs while establishing a new +- session, the wrapped error may also be an `AQMPError`. ++ session, the wrapped error may also be an `QMPError`. + """ + assert self.runstate == Runstate.IDLE + +diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py +index 45864f288e4fe5a7505c0022ed13..90a8737f03a997f6813ee7cbcaac 100644 +--- a/python/qemu/aqmp/qmp_client.py ++++ b/python/qemu/aqmp/qmp_client.py +@@ -20,7 +20,7 @@ from typing import ( + cast, + ) + +-from .error import AQMPError, ProtocolError ++from .error import ProtocolError, QMPError + from .events import Events + from .message import Message + from .models import ErrorResponse, Greeting +@@ -66,7 +66,7 @@ class NegotiationError(_WrappedProtocolError): + """ + + +-class ExecuteError(AQMPError): ++class ExecuteError(QMPError): + """ + Exception raised by `QMPClient.execute()` on RPC failure. + +@@ -87,7 +87,7 @@ class ExecuteError(AQMPError): + self.error_class: str = error_response.error.class_ + + +-class ExecInterruptedError(AQMPError): ++class ExecInterruptedError(QMPError): + """ + Exception raised by `execute()` (et al) when an RPC is interrupted. + +@@ -641,7 +641,7 @@ class QMPClient(AsyncProtocol[Message], Events): + sock = self._writer.transport.get_extra_info('socket') + + if sock.family != socket.AF_UNIX: +- raise AQMPError("Sending file descriptors requires a UNIX socket.") ++ raise QMPError("Sending file descriptors requires a UNIX socket.") + + if not hasattr(sock, 'sendmsg'): + # We need to void the warranty sticker. diff --git a/python-aqmp-rename-accept-to-start_serve.patch b/python-aqmp-rename-accept-to-start_serve.patch new file mode 100644 index 00000000..47b4ba97 --- /dev/null +++ b/python-aqmp-rename-accept-to-start_serve.patch @@ -0,0 +1,163 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:40 -0500 +Subject: python/aqmp: rename 'accept()' to 'start_server_and_accept()' +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 0ba4e76b23fed77d09be7f56da783ab3f0b2d497 + +Previously, I had a method named "accept()" that under-the-hood calls +bind(2), listen(2) *and* accept(2). I meant this as a simplification and +counterpart to the one-shot "connect()" method. + +This is confusing to readers who expect accept() to mean *just* +accept(2). Since I need to split apart the "accept()" method into +multiple methods anyway (one of which strongly resembling accept(2)), it +feels pertinent to rename this method *now*. + +Rename this all-in-one method "start_server_and_accept()" instead. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-3-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 2 +- + python/qemu/aqmp/protocol.py | 6 ++++-- + python/tests/protocol.py | 24 ++++++++++++------------ + 3 files changed, 17 insertions(+), 15 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 6baa5f3409a6b459c67097d3c2a0..dca1e76ed4994959caf542031363 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -91,7 +91,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): + self._aqmp.negotiate = True + + self._sync( +- self._aqmp.accept(self._address), ++ self._aqmp.start_server_and_accept(self._address), + timeout + ) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 009883f64d011e44dd003e9dcde3..73719257e058b7e9e4d8a281bcd9 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -265,8 +265,10 @@ class AsyncProtocol(Generic[T]): + + @upper_half + @require(Runstate.IDLE) +- async def accept(self, address: SocketAddrT, +- ssl: Optional[SSLContext] = None) -> None: ++ async def start_server_and_accept( ++ self, address: SocketAddrT, ++ ssl: Optional[SSLContext] = None ++ ) -> None: + """ + Accept a connection and begin processing message queues. + +diff --git a/python/tests/protocol.py b/python/tests/protocol.py +index 5cd7938be35ec61a1d412728f64e..354d6559b9d1e3dc3ad29598af3c 100644 +--- a/python/tests/protocol.py ++++ b/python/tests/protocol.py +@@ -413,14 +413,14 @@ class Accept(Connect): + assert family in ('INET', 'UNIX') + + if family == 'INET': +- await self.proto.accept(('example.com', 1)) ++ await self.proto.start_server_and_accept(('example.com', 1)) + elif family == 'UNIX': +- await self.proto.accept('/dev/null') ++ await self.proto.start_server_and_accept('/dev/null') + + async def _hanging_connection(self): + with TemporaryDirectory(suffix='.aqmp') as tmpdir: + sock = os.path.join(tmpdir, type(self.proto).__name__ + ".sock") +- await self.proto.accept(sock) ++ await self.proto.start_server_and_accept(sock) + + + class FakeSession(TestBase): +@@ -449,13 +449,13 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testFakeAccept(self): + """Test the full state lifecycle (via accept) with a no-op session.""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + self.assertEqual(self.proto.runstate, Runstate.RUNNING) + + @TestBase.async_test + async def testFakeRecv(self): + """Test receiving a fake/null message.""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + logname = self.proto.logger.name + with self.assertLogs(logname, level='DEBUG') as context: +@@ -471,7 +471,7 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testFakeSend(self): + """Test sending a fake/null message.""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + logname = self.proto.logger.name + with self.assertLogs(logname, level='DEBUG') as context: +@@ -493,7 +493,7 @@ class FakeSession(TestBase): + ): + with self.assertRaises(StateError) as context: + if accept: +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + else: + await self.proto.connect('/not/a/real/path') + +@@ -504,7 +504,7 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testAcceptRequireRunning(self): + """Test that accept() cannot be called when Runstate=RUNNING""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + await self._prod_session_api( + Runstate.RUNNING, +@@ -515,7 +515,7 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testConnectRequireRunning(self): + """Test that connect() cannot be called when Runstate=RUNNING""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + await self._prod_session_api( + Runstate.RUNNING, +@@ -526,7 +526,7 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testAcceptRequireDisconnecting(self): + """Test that accept() cannot be called when Runstate=DISCONNECTING""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + # Cheat: force a disconnect. + await self.proto.simulate_disconnect() +@@ -541,7 +541,7 @@ class FakeSession(TestBase): + @TestBase.async_test + async def testConnectRequireDisconnecting(self): + """Test that connect() cannot be called when Runstate=DISCONNECTING""" +- await self.proto.accept('/not/a/real/path') ++ await self.proto.start_server_and_accept('/not/a/real/path') + + # Cheat: force a disconnect. + await self.proto.simulate_disconnect() +@@ -576,7 +576,7 @@ class SimpleSession(TestBase): + async def testSmoke(self): + with TemporaryDirectory(suffix='.aqmp') as tmpdir: + sock = os.path.join(tmpdir, type(self.proto).__name__ + ".sock") +- server_task = create_task(self.server.accept(sock)) ++ server_task = create_task(self.server.start_server_and_accept(sock)) + + # give the server a chance to start listening [...] + await asyncio.sleep(0) diff --git a/python-aqmp-split-_client_connected_cb-o.patch b/python-aqmp-split-_client_connected_cb-o.patch new file mode 100644 index 00000000..677a35ee --- /dev/null +++ b/python-aqmp-split-_client_connected_cb-o.patch @@ -0,0 +1,149 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:42 -0500 +Subject: python/aqmp: split _client_connected_cb() out as _incoming() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 830e6fd36e2aef37b158a10dea6c3853ce43b20c + +As part of disentangling the monolithic nature of _do_accept(), split +out the incoming callback to prepare for factoring out the "wait for a +peer" step. Namely, this means using an event signal we can wait on from +outside of this method. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-5-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 83 +++++++++++++++++++++++++----------- + 1 file changed, 58 insertions(+), 25 deletions(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index b7e5e635d886db0efc85f829f42e..56f05b90308c44a86d0978fd2ce6 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -242,6 +242,10 @@ class AsyncProtocol(Generic[T]): + # Workaround for bind() + self._sock: Optional[socket.socket] = None + ++ # Server state for start_server() and _incoming() ++ self._server: Optional[asyncio.AbstractServer] = None ++ self._accepted: Optional[asyncio.Event] = None ++ + def __repr__(self) -> str: + cls_name = type(self).__name__ + tokens = [] +@@ -425,6 +429,54 @@ class AsyncProtocol(Generic[T]): + self._runstate_event.set() + self._runstate_event.clear() + ++ @bottom_half # However, it does not run from the R/W tasks. ++ async def _stop_server(self) -> None: ++ """ ++ Stop listening for / accepting new incoming connections. ++ """ ++ if self._server is None: ++ return ++ ++ try: ++ self.logger.debug("Stopping server.") ++ self._server.close() ++ await self._server.wait_closed() ++ self.logger.debug("Server stopped.") ++ finally: ++ self._server = None ++ ++ @bottom_half # However, it does not run from the R/W tasks. ++ async def _incoming(self, ++ reader: asyncio.StreamReader, ++ writer: asyncio.StreamWriter) -> None: ++ """ ++ Accept an incoming connection and signal the upper_half. ++ ++ This method does the minimum necessary to accept a single ++ incoming connection. It signals back to the upper_half ASAP so ++ that any errors during session initialization can occur ++ naturally in the caller's stack. ++ ++ :param reader: Incoming `asyncio.StreamReader` ++ :param writer: Incoming `asyncio.StreamWriter` ++ """ ++ peer = writer.get_extra_info('peername', 'Unknown peer') ++ self.logger.debug("Incoming connection from %s", peer) ++ ++ if self._reader or self._writer: ++ # Sadly, we can have more than one pending connection ++ # because of https://bugs.python.org/issue46715 ++ # Close any extra connections we don't actually want. ++ self.logger.warning("Extraneous connection inadvertently accepted") ++ writer.close() ++ return ++ ++ # A connection has been accepted; stop listening for new ones. ++ assert self._accepted is not None ++ await self._stop_server() ++ self._reader, self._writer = (reader, writer) ++ self._accepted.set() ++ + def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None: + """ + Used to create a socket in advance of accept(). +@@ -469,30 +521,11 @@ class AsyncProtocol(Generic[T]): + self._set_state(Runstate.CONNECTING) + + self.logger.debug("Awaiting connection on %s ...", address) +- connected = asyncio.Event() +- server: Optional[asyncio.AbstractServer] = None +- +- async def _client_connected_cb(reader: asyncio.StreamReader, +- writer: asyncio.StreamWriter) -> None: +- """Used to accept a single incoming connection, see below.""" +- nonlocal server +- nonlocal connected +- +- # A connection has been accepted; stop listening for new ones. +- assert server is not None +- server.close() +- await server.wait_closed() +- server = None +- +- # Register this client as being connected +- self._reader, self._writer = (reader, writer) +- +- # Signal back: We've accepted a client! +- connected.set() ++ self._accepted = asyncio.Event() + + if isinstance(address, tuple): + coro = asyncio.start_server( +- _client_connected_cb, ++ self._incoming, + host=None if self._sock else address[0], + port=None if self._sock else address[1], + ssl=ssl, +@@ -502,7 +535,7 @@ class AsyncProtocol(Generic[T]): + ) + else: + coro = asyncio.start_unix_server( +- _client_connected_cb, ++ self._incoming, + path=None if self._sock else address, + ssl=ssl, + backlog=1, +@@ -515,9 +548,9 @@ class AsyncProtocol(Generic[T]): + # otherwise yield. + await asyncio.sleep(0) + +- server = await coro # Starts listening +- await connected.wait() # Waits for the callback to fire (and finish) +- assert server is None ++ self._server = await coro # Starts listening ++ await self._accepted.wait() # Waits for the callback to finish ++ assert self._server is None + self._sock = None + + self.logger.debug("Connection accepted.") diff --git a/python-aqmp-squelch-pylint-warning-for-t.patch b/python-aqmp-squelch-pylint-warning-for-t.patch new file mode 100644 index 00000000..830c71f3 --- /dev/null +++ b/python-aqmp-squelch-pylint-warning-for-t.patch @@ -0,0 +1,37 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:43 -0500 +Subject: python/aqmp: squelch pylint warning for too many lines +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 1b9c8cb6ce6b5c5911eb715b2d5b0a2671999dde + +I would really like to keep this under 1000 lines, I promise. Doesn't +look like it's gonna happen. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-6-jsnow@redhat.com +Signed-off-by: John Snow +(cherry picked from commit 1b9c8cb6ce6b5c5911eb715b2d5b0a2671999dde) +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index 56f05b90308c44a86d0978fd2ce6..631bcdaa554f4a104af4e25a3c61 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -10,6 +10,9 @@ In this package, it is used as the implementation for the `QMPClient` + class. + """ + ++# It's all the docstrings ... ! It's long for a good reason ^_^; ++# pylint: disable=too-many-lines ++ + import asyncio + from asyncio import StreamReader, StreamWriter + from enum import Enum diff --git a/python-aqmp-stop-the-server-during-disco.patch b/python-aqmp-stop-the-server-during-disco.patch new file mode 100644 index 00000000..d5696264 --- /dev/null +++ b/python-aqmp-stop-the-server-during-disco.patch @@ -0,0 +1,54 @@ +From: John Snow +Date: Fri, 25 Feb 2022 15:59:45 -0500 +Subject: python/aqmp: stop the server during disconnect() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 32c5abf051d06ff103d9d30eb6a7f3e8bf582334 + +Before we allow the full separation of starting the server and accepting +new connections, make sure that the disconnect cleans up the server and +its new state, too. + +Signed-off-by: John Snow +Acked-by: Kevin Wolf +Reviewed-by: Daniel P. Berrangé +Message-id: 20220225205948.3693480-8-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/protocol.py | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py +index e2bdad542dc0ef451dd200a1d679..cdbc9cba0d15dea19cc1c60ca3c3 100644 +--- a/python/qemu/aqmp/protocol.py ++++ b/python/qemu/aqmp/protocol.py +@@ -432,7 +432,7 @@ class AsyncProtocol(Generic[T]): + self._runstate_event.set() + self._runstate_event.clear() + +- @bottom_half # However, it does not run from the R/W tasks. ++ @bottom_half + async def _stop_server(self) -> None: + """ + Stop listening for / accepting new incoming connections. +@@ -709,6 +709,7 @@ class AsyncProtocol(Generic[T]): + + self._reader = None + self._writer = None ++ self._accepted = None + + # NB: _runstate_changed cannot be cleared because we still need it to + # send the final runstate changed event ...! +@@ -732,6 +733,9 @@ class AsyncProtocol(Generic[T]): + def _done(task: Optional['asyncio.Future[Any]']) -> bool: + return task is not None and task.done() + ++ # If the server is running, stop it. ++ await self._stop_server() ++ + # Are we already in an error pathway? If either of the tasks are + # already done, or if we have no tasks but a reader/writer; we + # must be. diff --git a/python-introduce-qmp-shell-wrap-convenie.patch b/python-introduce-qmp-shell-wrap-convenie.patch new file mode 100644 index 00000000..32429819 --- /dev/null +++ b/python-introduce-qmp-shell-wrap-convenie.patch @@ -0,0 +1,167 @@ +From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= +Date: Fri, 28 Jan 2022 16:11:56 +0000 +Subject: python: introduce qmp-shell-wrap convenience tool +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 439125293cc9cfb684eb4db23db04199f5f435a2 + +With the current 'qmp-shell' tool developers must first spawn QEMU with +a suitable -qmp arg and then spawn qmp-shell in a separate terminal +pointing to the right socket. + +With 'qmp-shell-wrap' developers can ignore QMP sockets entirely and +just pass the QEMU command and arguments they want. The program will +listen on a UNIX socket and tell QEMU to connect QMP to that. + +For example, this: + + # qmp-shell-wrap -- qemu-system-x86_64 -display none + +Is roughly equivalent of running: + + # qemu-system-x86_64 -display none -qmp qmp-shell-1234 & + # qmp-shell qmp-shell-1234 + +Except that 'qmp-shell-wrap' switches the socket peers around so that +it is the UNIX socket server and QEMU is the socket client. This makes +QEMU reliably go away when qmp-shell-wrap exits, closing the server +socket. + +Signed-off-by: Daniel P. Berrangé +Message-id: 20220128161157.36261-2-berrange@redhat.com +[Edited for rebase. --js] +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/qmp_shell.py | 65 ++++++++++++++++++++++++++++++++--- + python/setup.cfg | 1 + + scripts/qmp/qmp-shell-wrap | 11 ++++++ + 3 files changed, 73 insertions(+), 4 deletions(-) + +diff --git a/python/qemu/aqmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py +index d11bf54b00e5d56616ae57be0006..c60df787fcd50bf8a0109e5f5cd3 100644 +--- a/python/qemu/aqmp/qmp_shell.py ++++ b/python/qemu/aqmp/qmp_shell.py +@@ -86,6 +86,7 @@ import logging + import os + import re + import readline ++from subprocess import Popen + import sys + from typing import ( + Iterator, +@@ -167,8 +168,10 @@ class QMPShell(QEMUMonitorProtocol): + :param verbose: Echo outgoing QMP messages to console. + """ + def __init__(self, address: SocketAddrT, +- pretty: bool = False, verbose: bool = False): +- super().__init__(address) ++ pretty: bool = False, ++ verbose: bool = False, ++ server: bool = False): ++ super().__init__(address, server=server) + self._greeting: Optional[QMPMessage] = None + self._completer = QMPCompleter() + self._transmode = False +@@ -409,8 +412,10 @@ class HMPShell(QMPShell): + :param verbose: Echo outgoing QMP messages to console. + """ + def __init__(self, address: SocketAddrT, +- pretty: bool = False, verbose: bool = False): +- super().__init__(address, pretty, verbose) ++ pretty: bool = False, ++ verbose: bool = False, ++ server: bool = False): ++ super().__init__(address, pretty, verbose, server) + self._cpu_index = 0 + + def _cmd_completion(self) -> None: +@@ -533,5 +538,57 @@ def main() -> None: + pass + + ++def main_wrap() -> None: ++ """ ++ qmp-shell-wrap entry point: parse command line arguments and ++ start the REPL. ++ """ ++ parser = argparse.ArgumentParser() ++ parser.add_argument('-H', '--hmp', action='store_true', ++ help='Use HMP interface') ++ parser.add_argument('-v', '--verbose', action='store_true', ++ help='Verbose (echo commands sent and received)') ++ parser.add_argument('-p', '--pretty', action='store_true', ++ help='Pretty-print JSON') ++ ++ parser.add_argument('command', nargs=argparse.REMAINDER, ++ help='QEMU command line to invoke') ++ ++ args = parser.parse_args() ++ ++ cmd = args.command ++ if len(cmd) != 0 and cmd[0] == '--': ++ cmd = cmd[1:] ++ if len(cmd) == 0: ++ cmd = ["qemu-system-x86_64"] ++ ++ sockpath = "qmp-shell-wrap-%d" % os.getpid() ++ cmd += ["-qmp", "unix:%s" % sockpath] ++ ++ shell_class = HMPShell if args.hmp else QMPShell ++ ++ try: ++ address = shell_class.parse_address(sockpath) ++ except QMPBadPortError: ++ parser.error(f"Bad port number: {sockpath}") ++ return # pycharm doesn't know error() is noreturn ++ ++ try: ++ with shell_class(address, args.pretty, args.verbose, True) as qemu: ++ with Popen(cmd): ++ ++ try: ++ qemu.accept() ++ except ConnectError as err: ++ if isinstance(err.exc, OSError): ++ die(f"Couldn't connect to {args.qmp_server}: {err!s}") ++ die(str(err)) ++ ++ for _ in qemu.repl(): ++ pass ++ finally: ++ os.unlink(sockpath) ++ ++ + if __name__ == '__main__': + main() +diff --git a/python/setup.cfg b/python/setup.cfg +index 0063c757b78638ef651a362af338..bec54e8b0d663191e2b7afbfa350 100644 +--- a/python/setup.cfg ++++ b/python/setup.cfg +@@ -68,6 +68,7 @@ console_scripts = + qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse] + qemu-ga-client = qemu.utils.qemu_ga_client:main + qmp-shell = qemu.aqmp.qmp_shell:main ++ qmp-shell-wrap = qemu.aqmp.qmp_shell:main_wrap + aqmp-tui = qemu.aqmp.aqmp_tui:main [tui] + + [flake8] +diff --git a/scripts/qmp/qmp-shell-wrap b/scripts/qmp/qmp-shell-wrap +new file mode 100755 +index 0000000000000000000000000000000000000000..9e94da114f5f87588639f6b2cc636391e80c3864 +--- /dev/null ++++ b/scripts/qmp/qmp-shell-wrap +@@ -0,0 +1,11 @@ ++#!/usr/bin/env python3 ++ ++import os ++import sys ++ ++sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) ++from qemu.qmp import qmp_shell ++ ++ ++if __name__ == '__main__': ++ qmp_shell.main_wrap() diff --git a/python-machine-raise-VMLaunchFailure-exc.patch b/python-machine-raise-VMLaunchFailure-exc.patch new file mode 100644 index 00000000..4d6207cb --- /dev/null +++ b/python-machine-raise-VMLaunchFailure-exc.patch @@ -0,0 +1,126 @@ +From: John Snow +Date: Mon, 31 Jan 2022 23:11:32 -0500 +Subject: python/machine: raise VMLaunchFailure exception from launch() + +Git-commit: 50465f94d211beabfbfc80e4f85ec4fad0757570 + +This allows us to pack in some extra information about the failure, +which guarantees that if the caller did not *intentionally* cause a +failure (by capturing this Exception), some pretty good clues will be +printed at the bottom of the traceback information. + +This will help make failures in the event of a non-negative return code +more obvious when they go unhandled; the current behavior in +_post_shutdown() is to print a warning message only in the event of +signal-based terminations (for negative return codes). + +(Note: In Python, catching BaseException instead of Exception catches a +broader array of Exception events, including SystemExit and +KeyboardInterrupt. We do not want to "wrap" such exceptions as a +VMLaunchFailure, because that will 'downgrade' the exception from a +BaseException to a regular Exception. We do, however, want to perform +cleanup in either case, so catch on the broadest scope and +wrap-and-re-raise only in the more targeted scope.) + +Signed-off-by: John Snow +Reviewed-by: Hanna Reitz +Reviewed-by: Kevin Wolf +Message-id: 20220201041134.1237016-3-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/machine/machine.py | 45 ++++++++++++++++++++--- + tests/qemu-iotests/tests/mirror-top-perms | 3 +- + 2 files changed, 40 insertions(+), 8 deletions(-) + +diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py +index 67ab06ca2b6daa531b7c0ad9f7c2..a5972fab4d2b1fddfa2d5a7db882 100644 +--- a/python/qemu/machine/machine.py ++++ b/python/qemu/machine/machine.py +@@ -74,6 +74,35 @@ class QEMUMachineAddDeviceError(QEMUMachineError): + """ + + ++class VMLaunchFailure(QEMUMachineError): ++ """ ++ Exception raised when a VM launch was attempted, but failed. ++ """ ++ def __init__(self, exitcode: Optional[int], ++ command: str, output: Optional[str]): ++ super().__init__(exitcode, command, output) ++ self.exitcode = exitcode ++ self.command = command ++ self.output = output ++ ++ def __str__(self) -> str: ++ ret = '' ++ if self.__cause__ is not None: ++ name = type(self.__cause__).__name__ ++ reason = str(self.__cause__) ++ if reason: ++ ret += f"{name}: {reason}" ++ else: ++ ret += f"{name}" ++ ret += '\n' ++ ++ if self.exitcode is not None: ++ ret += f"\tExit code: {self.exitcode}\n" ++ ret += f"\tCommand: {self.command}\n" ++ ret += f"\tOutput: {self.output}\n" ++ return ret ++ ++ + class AbnormalShutdown(QEMUMachineError): + """ + Exception raised when a graceful shutdown was requested, but not performed. +@@ -397,7 +426,7 @@ class QEMUMachine: + + try: + self._launch() +- except: ++ except BaseException as exc: + # We may have launched the process but it may + # have exited before we could connect via QMP. + # Assume the VM didn't launch or is exiting. +@@ -408,11 +437,15 @@ class QEMUMachine: + else: + self._post_shutdown() + +- LOG.debug('Error launching VM') +- if self._qemu_full_args: +- LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) +- if self._iolog: +- LOG.debug('Output: %r', self._iolog) ++ if isinstance(exc, Exception): ++ raise VMLaunchFailure( ++ exitcode=self.exitcode(), ++ command=' '.join(self._qemu_full_args), ++ output=self._iolog ++ ) from exc ++ ++ # Don't wrap 'BaseException'; doing so would downgrade ++ # that exception. However, we still want to clean up. + raise + + def _launch(self) -> None: +diff --git a/tests/qemu-iotests/tests/mirror-top-perms b/tests/qemu-iotests/tests/mirror-top-perms +index 0a51a613f39764b2b3ab3fa460ef..b5849978c4158c35e18480186ea2 100755 +--- a/tests/qemu-iotests/tests/mirror-top-perms ++++ b/tests/qemu-iotests/tests/mirror-top-perms +@@ -21,7 +21,6 @@ + + import os + +-from qemu.aqmp import ConnectError + from qemu.machine import machine + from qemu.qmp import QMPConnectError + +@@ -107,7 +106,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase): + self.vm_b.launch() + print('ERROR: VM B launched successfully, ' + 'this should not have happened') +- except (QMPConnectError, ConnectError): ++ except (QMPConnectError, machine.VMLaunchFailure): + assert 'Is another process using the image' in self.vm_b.get_log() + + result = self.vm.qmp('block-job-cancel', diff --git a/python-move-qmp-shell-under-the-AQMP-pac.patch b/python-move-qmp-shell-under-the-AQMP-pac.patch new file mode 100644 index 00000000..4dc8a6e8 --- /dev/null +++ b/python-move-qmp-shell-under-the-AQMP-pac.patch @@ -0,0 +1,1143 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:55 -0500 +Subject: python: move qmp-shell under the AQMP package + +Git-commit: fd9c3a6219b0470c356c8486188052d353846806 + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/README.rst | 2 +- + python/qemu/aqmp/qmp_shell.py | 537 ++++++++++++++++++++++++++++++++++ + python/qemu/qmp/qmp_shell.py | 537 ---------------------------------- + python/setup.cfg | 2 +- + scripts/qmp/qmp-shell | 2 +- + 5 files changed, 540 insertions(+), 540 deletions(-) + +diff --git a/python/README.rst b/python/README.rst +index 9c1fceaee73b11cb313b71a9a558..fcf74f69eae011aafd1e089e1f92 100644 +--- a/python/README.rst ++++ b/python/README.rst +@@ -59,7 +59,7 @@ Package installation also normally provides executable console scripts, + so that tools like ``qmp-shell`` are always available via $PATH. To + invoke them without installation, you can invoke e.g.: + +-``> PYTHONPATH=~/src/qemu/python python3 -m qemu.qmp.qmp_shell`` ++``> PYTHONPATH=~/src/qemu/python python3 -m qemu.aqmp.qmp_shell`` + + The mappings between console script name and python module path can be + found in ``setup.cfg``. +diff --git a/python/qemu/aqmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py +new file mode 100644 +index 0000000000000000000000000000000000000000..d11bf54b00e5d56616ae57be0006b9b0629dd9f4 +--- /dev/null ++++ b/python/qemu/aqmp/qmp_shell.py +@@ -0,0 +1,537 @@ ++# ++# Copyright (C) 2009, 2010 Red Hat Inc. ++# ++# Authors: ++# Luiz Capitulino ++# ++# This work is licensed under the terms of the GNU GPL, version 2. See ++# the COPYING file in the top-level directory. ++# ++ ++""" ++Low-level QEMU shell on top of QMP. ++ ++usage: qmp-shell [-h] [-H] [-N] [-v] [-p] qmp_server ++ ++positional arguments: ++ qmp_server < UNIX socket path | TCP address:port > ++ ++optional arguments: ++ -h, --help show this help message and exit ++ -H, --hmp Use HMP interface ++ -N, --skip-negotiation ++ Skip negotiate (for qemu-ga) ++ -v, --verbose Verbose (echo commands sent and received) ++ -p, --pretty Pretty-print JSON ++ ++ ++Start QEMU with: ++ ++# qemu [...] -qmp unix:./qmp-sock,server ++ ++Run the shell: ++ ++$ qmp-shell ./qmp-sock ++ ++Commands have the following format: ++ ++ < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] ++ ++For example: ++ ++(QEMU) device_add driver=e1000 id=net1 ++{'return': {}} ++(QEMU) ++ ++key=value pairs also support Python or JSON object literal subset notations, ++without spaces. Dictionaries/objects {} are supported as are arrays []. ++ ++ example-command arg-name1={'key':'value','obj'={'prop':"value"}} ++ ++Both JSON and Python formatting should work, including both styles of ++string literal quotes. Both paradigms of literal values should work, ++including null/true/false for JSON and None/True/False for Python. ++ ++ ++Transactions have the following multi-line format: ++ ++ transaction( ++ action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ] ++ ... ++ action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ] ++ ) ++ ++One line transactions are also supported: ++ ++ transaction( action-name1 ... ) ++ ++For example: ++ ++ (QEMU) transaction( ++ TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1 ++ TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0 ++ TRANS> ) ++ {"return": {}} ++ (QEMU) ++ ++Use the -v and -p options to activate the verbose and pretty-print options, ++which will echo back the properly formatted JSON-compliant QMP that is being ++sent to QEMU, which is useful for debugging and documentation generation. ++""" ++ ++import argparse ++import ast ++import json ++import logging ++import os ++import re ++import readline ++import sys ++from typing import ( ++ Iterator, ++ List, ++ NoReturn, ++ Optional, ++ Sequence, ++) ++ ++from qemu.aqmp import ConnectError, QMPError, SocketAddrT ++from qemu.aqmp.legacy import ( ++ QEMUMonitorProtocol, ++ QMPBadPortError, ++ QMPMessage, ++ QMPObject, ++) ++ ++ ++LOG = logging.getLogger(__name__) ++ ++ ++class QMPCompleter: ++ """ ++ QMPCompleter provides a readline library tab-complete behavior. ++ """ ++ # NB: Python 3.9+ will probably allow us to subclass list[str] directly, ++ # but pylint as of today does not know that List[str] is simply 'list'. ++ def __init__(self) -> None: ++ self._matches: List[str] = [] ++ ++ def append(self, value: str) -> None: ++ """Append a new valid completion to the list of possibilities.""" ++ return self._matches.append(value) ++ ++ def complete(self, text: str, state: int) -> Optional[str]: ++ """readline.set_completer() callback implementation.""" ++ for cmd in self._matches: ++ if cmd.startswith(text): ++ if state == 0: ++ return cmd ++ state -= 1 ++ return None ++ ++ ++class QMPShellError(QMPError): ++ """ ++ QMP Shell Base error class. ++ """ ++ ++ ++class FuzzyJSON(ast.NodeTransformer): ++ """ ++ This extension of ast.NodeTransformer filters literal "true/false/null" ++ values in a Python AST and replaces them by proper "True/False/None" values ++ that Python can properly evaluate. ++ """ ++ ++ @classmethod ++ def visit_Name(cls, # pylint: disable=invalid-name ++ node: ast.Name) -> ast.AST: ++ """ ++ Transform Name nodes with certain values into Constant (keyword) nodes. ++ """ ++ if node.id == 'true': ++ return ast.Constant(value=True) ++ if node.id == 'false': ++ return ast.Constant(value=False) ++ if node.id == 'null': ++ return ast.Constant(value=None) ++ return node ++ ++ ++class QMPShell(QEMUMonitorProtocol): ++ """ ++ QMPShell provides a basic readline-based QMP shell. ++ ++ :param address: Address of the QMP server. ++ :param pretty: Pretty-print QMP messages. ++ :param verbose: Echo outgoing QMP messages to console. ++ """ ++ def __init__(self, address: SocketAddrT, ++ pretty: bool = False, verbose: bool = False): ++ super().__init__(address) ++ self._greeting: Optional[QMPMessage] = None ++ self._completer = QMPCompleter() ++ self._transmode = False ++ self._actions: List[QMPMessage] = [] ++ self._histfile = os.path.join(os.path.expanduser('~'), ++ '.qmp-shell_history') ++ self.pretty = pretty ++ self.verbose = verbose ++ ++ def close(self) -> None: ++ # Hook into context manager of parent to save shell history. ++ self._save_history() ++ super().close() ++ ++ def _fill_completion(self) -> None: ++ cmds = self.cmd('query-commands') ++ if 'error' in cmds: ++ return ++ for cmd in cmds['return']: ++ self._completer.append(cmd['name']) ++ ++ def _completer_setup(self) -> None: ++ self._completer = QMPCompleter() ++ self._fill_completion() ++ readline.set_history_length(1024) ++ readline.set_completer(self._completer.complete) ++ readline.parse_and_bind("tab: complete") ++ # NB: default delimiters conflict with some command names ++ # (eg. query-), clearing everything as it doesn't seem to matter ++ readline.set_completer_delims('') ++ try: ++ readline.read_history_file(self._histfile) ++ except FileNotFoundError: ++ pass ++ except IOError as err: ++ msg = f"Failed to read history '{self._histfile}': {err!s}" ++ LOG.warning(msg) ++ ++ def _save_history(self) -> None: ++ try: ++ readline.write_history_file(self._histfile) ++ except IOError as err: ++ msg = f"Failed to save history file '{self._histfile}': {err!s}" ++ LOG.warning(msg) ++ ++ @classmethod ++ def _parse_value(cls, val: str) -> object: ++ try: ++ return int(val) ++ except ValueError: ++ pass ++ ++ if val.lower() == 'true': ++ return True ++ if val.lower() == 'false': ++ return False ++ if val.startswith(('{', '[')): ++ # Try first as pure JSON: ++ try: ++ return json.loads(val) ++ except ValueError: ++ pass ++ # Try once again as FuzzyJSON: ++ try: ++ tree = ast.parse(val, mode='eval') ++ transformed = FuzzyJSON().visit(tree) ++ return ast.literal_eval(transformed) ++ except (SyntaxError, ValueError): ++ pass ++ return val ++ ++ def _cli_expr(self, ++ tokens: Sequence[str], ++ parent: QMPObject) -> None: ++ for arg in tokens: ++ (key, sep, val) = arg.partition('=') ++ if sep != '=': ++ raise QMPShellError( ++ f"Expected a key=value pair, got '{arg!s}'" ++ ) ++ ++ value = self._parse_value(val) ++ optpath = key.split('.') ++ curpath = [] ++ for path in optpath[:-1]: ++ curpath.append(path) ++ obj = parent.get(path, {}) ++ if not isinstance(obj, dict): ++ msg = 'Cannot use "{:s}" as both leaf and non-leaf key' ++ raise QMPShellError(msg.format('.'.join(curpath))) ++ parent[path] = obj ++ parent = obj ++ if optpath[-1] in parent: ++ if isinstance(parent[optpath[-1]], dict): ++ msg = 'Cannot use "{:s}" as both leaf and non-leaf key' ++ raise QMPShellError(msg.format('.'.join(curpath))) ++ raise QMPShellError(f'Cannot set "{key}" multiple times') ++ parent[optpath[-1]] = value ++ ++ def _build_cmd(self, cmdline: str) -> Optional[QMPMessage]: ++ """ ++ Build a QMP input object from a user provided command-line in the ++ following format: ++ ++ < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] ++ """ ++ argument_regex = r'''(?:[^\s"']|"(?:\\.|[^"])*"|'(?:\\.|[^'])*')+''' ++ cmdargs = re.findall(argument_regex, cmdline) ++ qmpcmd: QMPMessage ++ ++ # Transactional CLI entry: ++ if cmdargs and cmdargs[0] == 'transaction(': ++ self._transmode = True ++ self._actions = [] ++ cmdargs.pop(0) ++ ++ # Transactional CLI exit: ++ if cmdargs and cmdargs[0] == ')' and self._transmode: ++ self._transmode = False ++ if len(cmdargs) > 1: ++ msg = 'Unexpected input after close of Transaction sub-shell' ++ raise QMPShellError(msg) ++ qmpcmd = { ++ 'execute': 'transaction', ++ 'arguments': {'actions': self._actions} ++ } ++ return qmpcmd ++ ++ # No args, or no args remaining ++ if not cmdargs: ++ return None ++ ++ if self._transmode: ++ # Parse and cache this Transactional Action ++ finalize = False ++ action = {'type': cmdargs[0], 'data': {}} ++ if cmdargs[-1] == ')': ++ cmdargs.pop(-1) ++ finalize = True ++ self._cli_expr(cmdargs[1:], action['data']) ++ self._actions.append(action) ++ return self._build_cmd(')') if finalize else None ++ ++ # Standard command: parse and return it to be executed. ++ qmpcmd = {'execute': cmdargs[0], 'arguments': {}} ++ self._cli_expr(cmdargs[1:], qmpcmd['arguments']) ++ return qmpcmd ++ ++ def _print(self, qmp_message: object) -> None: ++ jsobj = json.dumps(qmp_message, ++ indent=4 if self.pretty else None, ++ sort_keys=self.pretty) ++ print(str(jsobj)) ++ ++ def _execute_cmd(self, cmdline: str) -> bool: ++ try: ++ qmpcmd = self._build_cmd(cmdline) ++ except QMPShellError as err: ++ print( ++ f"Error while parsing command line: {err!s}\n" ++ "command format: " ++ "[arg-name1=arg1] ... [arg-nameN=argN", ++ file=sys.stderr ++ ) ++ return True ++ # For transaction mode, we may have just cached the action: ++ if qmpcmd is None: ++ return True ++ if self.verbose: ++ self._print(qmpcmd) ++ resp = self.cmd_obj(qmpcmd) ++ if resp is None: ++ print('Disconnected') ++ return False ++ self._print(resp) ++ return True ++ ++ def connect(self, negotiate: bool = True) -> None: ++ self._greeting = super().connect(negotiate) ++ self._completer_setup() ++ ++ def show_banner(self, ++ msg: str = 'Welcome to the QMP low-level shell!') -> None: ++ """ ++ Print to stdio a greeting, and the QEMU version if available. ++ """ ++ print(msg) ++ if not self._greeting: ++ print('Connected') ++ return ++ version = self._greeting['QMP']['version']['qemu'] ++ print("Connected to QEMU {major}.{minor}.{micro}\n".format(**version)) ++ ++ @property ++ def prompt(self) -> str: ++ """ ++ Return the current shell prompt, including a trailing space. ++ """ ++ if self._transmode: ++ return 'TRANS> ' ++ return '(QEMU) ' ++ ++ def read_exec_command(self) -> bool: ++ """ ++ Read and execute a command. ++ ++ @return True if execution was ok, return False if disconnected. ++ """ ++ try: ++ cmdline = input(self.prompt) ++ except EOFError: ++ print() ++ return False ++ ++ if cmdline == '': ++ for event in self.get_events(): ++ print(event) ++ return True ++ ++ return self._execute_cmd(cmdline) ++ ++ def repl(self) -> Iterator[None]: ++ """ ++ Return an iterator that implements the REPL. ++ """ ++ self.show_banner() ++ while self.read_exec_command(): ++ yield ++ self.close() ++ ++ ++class HMPShell(QMPShell): ++ """ ++ HMPShell provides a basic readline-based HMP shell, tunnelled via QMP. ++ ++ :param address: Address of the QMP server. ++ :param pretty: Pretty-print QMP messages. ++ :param verbose: Echo outgoing QMP messages to console. ++ """ ++ def __init__(self, address: SocketAddrT, ++ pretty: bool = False, verbose: bool = False): ++ super().__init__(address, pretty, verbose) ++ self._cpu_index = 0 ++ ++ def _cmd_completion(self) -> None: ++ for cmd in self._cmd_passthrough('help')['return'].split('\r\n'): ++ if cmd and cmd[0] != '[' and cmd[0] != '\t': ++ name = cmd.split()[0] # drop help text ++ if name == 'info': ++ continue ++ if name.find('|') != -1: ++ # Command in the form 'foobar|f' or 'f|foobar', take the ++ # full name ++ opt = name.split('|') ++ if len(opt[0]) == 1: ++ name = opt[1] ++ else: ++ name = opt[0] ++ self._completer.append(name) ++ self._completer.append('help ' + name) # help completion ++ ++ def _info_completion(self) -> None: ++ for cmd in self._cmd_passthrough('info')['return'].split('\r\n'): ++ if cmd: ++ self._completer.append('info ' + cmd.split()[1]) ++ ++ def _other_completion(self) -> None: ++ # special cases ++ self._completer.append('help info') ++ ++ def _fill_completion(self) -> None: ++ self._cmd_completion() ++ self._info_completion() ++ self._other_completion() ++ ++ def _cmd_passthrough(self, cmdline: str, ++ cpu_index: int = 0) -> QMPMessage: ++ return self.cmd_obj({ ++ 'execute': 'human-monitor-command', ++ 'arguments': { ++ 'command-line': cmdline, ++ 'cpu-index': cpu_index ++ } ++ }) ++ ++ def _execute_cmd(self, cmdline: str) -> bool: ++ if cmdline.split()[0] == "cpu": ++ # trap the cpu command, it requires special setting ++ try: ++ idx = int(cmdline.split()[1]) ++ if 'return' not in self._cmd_passthrough('info version', idx): ++ print('bad CPU index') ++ return True ++ self._cpu_index = idx ++ except ValueError: ++ print('cpu command takes an integer argument') ++ return True ++ resp = self._cmd_passthrough(cmdline, self._cpu_index) ++ if resp is None: ++ print('Disconnected') ++ return False ++ assert 'return' in resp or 'error' in resp ++ if 'return' in resp: ++ # Success ++ if len(resp['return']) > 0: ++ print(resp['return'], end=' ') ++ else: ++ # Error ++ print('%s: %s' % (resp['error']['class'], resp['error']['desc'])) ++ return True ++ ++ def show_banner(self, msg: str = 'Welcome to the HMP shell!') -> None: ++ QMPShell.show_banner(self, msg) ++ ++ ++def die(msg: str) -> NoReturn: ++ """Write an error to stderr, then exit with a return code of 1.""" ++ sys.stderr.write('ERROR: %s\n' % msg) ++ sys.exit(1) ++ ++ ++def main() -> None: ++ """ ++ qmp-shell entry point: parse command line arguments and start the REPL. ++ """ ++ parser = argparse.ArgumentParser() ++ parser.add_argument('-H', '--hmp', action='store_true', ++ help='Use HMP interface') ++ parser.add_argument('-N', '--skip-negotiation', action='store_true', ++ help='Skip negotiate (for qemu-ga)') ++ parser.add_argument('-v', '--verbose', action='store_true', ++ help='Verbose (echo commands sent and received)') ++ parser.add_argument('-p', '--pretty', action='store_true', ++ help='Pretty-print JSON') ++ ++ default_server = os.environ.get('QMP_SOCKET') ++ parser.add_argument('qmp_server', action='store', ++ default=default_server, ++ help='< UNIX socket path | TCP address:port >') ++ ++ args = parser.parse_args() ++ if args.qmp_server is None: ++ parser.error("QMP socket or TCP address must be specified") ++ ++ shell_class = HMPShell if args.hmp else QMPShell ++ ++ try: ++ address = shell_class.parse_address(args.qmp_server) ++ except QMPBadPortError: ++ parser.error(f"Bad port number: {args.qmp_server}") ++ return # pycharm doesn't know error() is noreturn ++ ++ with shell_class(address, args.pretty, args.verbose) as qemu: ++ try: ++ qemu.connect(negotiate=not args.skip_negotiation) ++ except ConnectError as err: ++ if isinstance(err.exc, OSError): ++ die(f"Couldn't connect to {args.qmp_server}: {err!s}") ++ die(str(err)) ++ ++ for _ in qemu.repl(): ++ pass ++ ++ ++if __name__ == '__main__': ++ main() +diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/qmp/qmp_shell.py +deleted file mode 100644 +index d11bf54b00e5d56616ae57be0006b9b0629dd9f4..0000000000000000000000000000000000000000 +--- a/python/qemu/qmp/qmp_shell.py ++++ /dev/null +@@ -1,537 +0,0 @@ +-# +-# Copyright (C) 2009, 2010 Red Hat Inc. +-# +-# Authors: +-# Luiz Capitulino +-# +-# This work is licensed under the terms of the GNU GPL, version 2. See +-# the COPYING file in the top-level directory. +-# +- +-""" +-Low-level QEMU shell on top of QMP. +- +-usage: qmp-shell [-h] [-H] [-N] [-v] [-p] qmp_server +- +-positional arguments: +- qmp_server < UNIX socket path | TCP address:port > +- +-optional arguments: +- -h, --help show this help message and exit +- -H, --hmp Use HMP interface +- -N, --skip-negotiation +- Skip negotiate (for qemu-ga) +- -v, --verbose Verbose (echo commands sent and received) +- -p, --pretty Pretty-print JSON +- +- +-Start QEMU with: +- +-# qemu [...] -qmp unix:./qmp-sock,server +- +-Run the shell: +- +-$ qmp-shell ./qmp-sock +- +-Commands have the following format: +- +- < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] +- +-For example: +- +-(QEMU) device_add driver=e1000 id=net1 +-{'return': {}} +-(QEMU) +- +-key=value pairs also support Python or JSON object literal subset notations, +-without spaces. Dictionaries/objects {} are supported as are arrays []. +- +- example-command arg-name1={'key':'value','obj'={'prop':"value"}} +- +-Both JSON and Python formatting should work, including both styles of +-string literal quotes. Both paradigms of literal values should work, +-including null/true/false for JSON and None/True/False for Python. +- +- +-Transactions have the following multi-line format: +- +- transaction( +- action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ] +- ... +- action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ] +- ) +- +-One line transactions are also supported: +- +- transaction( action-name1 ... ) +- +-For example: +- +- (QEMU) transaction( +- TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1 +- TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0 +- TRANS> ) +- {"return": {}} +- (QEMU) +- +-Use the -v and -p options to activate the verbose and pretty-print options, +-which will echo back the properly formatted JSON-compliant QMP that is being +-sent to QEMU, which is useful for debugging and documentation generation. +-""" +- +-import argparse +-import ast +-import json +-import logging +-import os +-import re +-import readline +-import sys +-from typing import ( +- Iterator, +- List, +- NoReturn, +- Optional, +- Sequence, +-) +- +-from qemu.aqmp import ConnectError, QMPError, SocketAddrT +-from qemu.aqmp.legacy import ( +- QEMUMonitorProtocol, +- QMPBadPortError, +- QMPMessage, +- QMPObject, +-) +- +- +-LOG = logging.getLogger(__name__) +- +- +-class QMPCompleter: +- """ +- QMPCompleter provides a readline library tab-complete behavior. +- """ +- # NB: Python 3.9+ will probably allow us to subclass list[str] directly, +- # but pylint as of today does not know that List[str] is simply 'list'. +- def __init__(self) -> None: +- self._matches: List[str] = [] +- +- def append(self, value: str) -> None: +- """Append a new valid completion to the list of possibilities.""" +- return self._matches.append(value) +- +- def complete(self, text: str, state: int) -> Optional[str]: +- """readline.set_completer() callback implementation.""" +- for cmd in self._matches: +- if cmd.startswith(text): +- if state == 0: +- return cmd +- state -= 1 +- return None +- +- +-class QMPShellError(QMPError): +- """ +- QMP Shell Base error class. +- """ +- +- +-class FuzzyJSON(ast.NodeTransformer): +- """ +- This extension of ast.NodeTransformer filters literal "true/false/null" +- values in a Python AST and replaces them by proper "True/False/None" values +- that Python can properly evaluate. +- """ +- +- @classmethod +- def visit_Name(cls, # pylint: disable=invalid-name +- node: ast.Name) -> ast.AST: +- """ +- Transform Name nodes with certain values into Constant (keyword) nodes. +- """ +- if node.id == 'true': +- return ast.Constant(value=True) +- if node.id == 'false': +- return ast.Constant(value=False) +- if node.id == 'null': +- return ast.Constant(value=None) +- return node +- +- +-class QMPShell(QEMUMonitorProtocol): +- """ +- QMPShell provides a basic readline-based QMP shell. +- +- :param address: Address of the QMP server. +- :param pretty: Pretty-print QMP messages. +- :param verbose: Echo outgoing QMP messages to console. +- """ +- def __init__(self, address: SocketAddrT, +- pretty: bool = False, verbose: bool = False): +- super().__init__(address) +- self._greeting: Optional[QMPMessage] = None +- self._completer = QMPCompleter() +- self._transmode = False +- self._actions: List[QMPMessage] = [] +- self._histfile = os.path.join(os.path.expanduser('~'), +- '.qmp-shell_history') +- self.pretty = pretty +- self.verbose = verbose +- +- def close(self) -> None: +- # Hook into context manager of parent to save shell history. +- self._save_history() +- super().close() +- +- def _fill_completion(self) -> None: +- cmds = self.cmd('query-commands') +- if 'error' in cmds: +- return +- for cmd in cmds['return']: +- self._completer.append(cmd['name']) +- +- def _completer_setup(self) -> None: +- self._completer = QMPCompleter() +- self._fill_completion() +- readline.set_history_length(1024) +- readline.set_completer(self._completer.complete) +- readline.parse_and_bind("tab: complete") +- # NB: default delimiters conflict with some command names +- # (eg. query-), clearing everything as it doesn't seem to matter +- readline.set_completer_delims('') +- try: +- readline.read_history_file(self._histfile) +- except FileNotFoundError: +- pass +- except IOError as err: +- msg = f"Failed to read history '{self._histfile}': {err!s}" +- LOG.warning(msg) +- +- def _save_history(self) -> None: +- try: +- readline.write_history_file(self._histfile) +- except IOError as err: +- msg = f"Failed to save history file '{self._histfile}': {err!s}" +- LOG.warning(msg) +- +- @classmethod +- def _parse_value(cls, val: str) -> object: +- try: +- return int(val) +- except ValueError: +- pass +- +- if val.lower() == 'true': +- return True +- if val.lower() == 'false': +- return False +- if val.startswith(('{', '[')): +- # Try first as pure JSON: +- try: +- return json.loads(val) +- except ValueError: +- pass +- # Try once again as FuzzyJSON: +- try: +- tree = ast.parse(val, mode='eval') +- transformed = FuzzyJSON().visit(tree) +- return ast.literal_eval(transformed) +- except (SyntaxError, ValueError): +- pass +- return val +- +- def _cli_expr(self, +- tokens: Sequence[str], +- parent: QMPObject) -> None: +- for arg in tokens: +- (key, sep, val) = arg.partition('=') +- if sep != '=': +- raise QMPShellError( +- f"Expected a key=value pair, got '{arg!s}'" +- ) +- +- value = self._parse_value(val) +- optpath = key.split('.') +- curpath = [] +- for path in optpath[:-1]: +- curpath.append(path) +- obj = parent.get(path, {}) +- if not isinstance(obj, dict): +- msg = 'Cannot use "{:s}" as both leaf and non-leaf key' +- raise QMPShellError(msg.format('.'.join(curpath))) +- parent[path] = obj +- parent = obj +- if optpath[-1] in parent: +- if isinstance(parent[optpath[-1]], dict): +- msg = 'Cannot use "{:s}" as both leaf and non-leaf key' +- raise QMPShellError(msg.format('.'.join(curpath))) +- raise QMPShellError(f'Cannot set "{key}" multiple times') +- parent[optpath[-1]] = value +- +- def _build_cmd(self, cmdline: str) -> Optional[QMPMessage]: +- """ +- Build a QMP input object from a user provided command-line in the +- following format: +- +- < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ] +- """ +- argument_regex = r'''(?:[^\s"']|"(?:\\.|[^"])*"|'(?:\\.|[^'])*')+''' +- cmdargs = re.findall(argument_regex, cmdline) +- qmpcmd: QMPMessage +- +- # Transactional CLI entry: +- if cmdargs and cmdargs[0] == 'transaction(': +- self._transmode = True +- self._actions = [] +- cmdargs.pop(0) +- +- # Transactional CLI exit: +- if cmdargs and cmdargs[0] == ')' and self._transmode: +- self._transmode = False +- if len(cmdargs) > 1: +- msg = 'Unexpected input after close of Transaction sub-shell' +- raise QMPShellError(msg) +- qmpcmd = { +- 'execute': 'transaction', +- 'arguments': {'actions': self._actions} +- } +- return qmpcmd +- +- # No args, or no args remaining +- if not cmdargs: +- return None +- +- if self._transmode: +- # Parse and cache this Transactional Action +- finalize = False +- action = {'type': cmdargs[0], 'data': {}} +- if cmdargs[-1] == ')': +- cmdargs.pop(-1) +- finalize = True +- self._cli_expr(cmdargs[1:], action['data']) +- self._actions.append(action) +- return self._build_cmd(')') if finalize else None +- +- # Standard command: parse and return it to be executed. +- qmpcmd = {'execute': cmdargs[0], 'arguments': {}} +- self._cli_expr(cmdargs[1:], qmpcmd['arguments']) +- return qmpcmd +- +- def _print(self, qmp_message: object) -> None: +- jsobj = json.dumps(qmp_message, +- indent=4 if self.pretty else None, +- sort_keys=self.pretty) +- print(str(jsobj)) +- +- def _execute_cmd(self, cmdline: str) -> bool: +- try: +- qmpcmd = self._build_cmd(cmdline) +- except QMPShellError as err: +- print( +- f"Error while parsing command line: {err!s}\n" +- "command format: " +- "[arg-name1=arg1] ... [arg-nameN=argN", +- file=sys.stderr +- ) +- return True +- # For transaction mode, we may have just cached the action: +- if qmpcmd is None: +- return True +- if self.verbose: +- self._print(qmpcmd) +- resp = self.cmd_obj(qmpcmd) +- if resp is None: +- print('Disconnected') +- return False +- self._print(resp) +- return True +- +- def connect(self, negotiate: bool = True) -> None: +- self._greeting = super().connect(negotiate) +- self._completer_setup() +- +- def show_banner(self, +- msg: str = 'Welcome to the QMP low-level shell!') -> None: +- """ +- Print to stdio a greeting, and the QEMU version if available. +- """ +- print(msg) +- if not self._greeting: +- print('Connected') +- return +- version = self._greeting['QMP']['version']['qemu'] +- print("Connected to QEMU {major}.{minor}.{micro}\n".format(**version)) +- +- @property +- def prompt(self) -> str: +- """ +- Return the current shell prompt, including a trailing space. +- """ +- if self._transmode: +- return 'TRANS> ' +- return '(QEMU) ' +- +- def read_exec_command(self) -> bool: +- """ +- Read and execute a command. +- +- @return True if execution was ok, return False if disconnected. +- """ +- try: +- cmdline = input(self.prompt) +- except EOFError: +- print() +- return False +- +- if cmdline == '': +- for event in self.get_events(): +- print(event) +- return True +- +- return self._execute_cmd(cmdline) +- +- def repl(self) -> Iterator[None]: +- """ +- Return an iterator that implements the REPL. +- """ +- self.show_banner() +- while self.read_exec_command(): +- yield +- self.close() +- +- +-class HMPShell(QMPShell): +- """ +- HMPShell provides a basic readline-based HMP shell, tunnelled via QMP. +- +- :param address: Address of the QMP server. +- :param pretty: Pretty-print QMP messages. +- :param verbose: Echo outgoing QMP messages to console. +- """ +- def __init__(self, address: SocketAddrT, +- pretty: bool = False, verbose: bool = False): +- super().__init__(address, pretty, verbose) +- self._cpu_index = 0 +- +- def _cmd_completion(self) -> None: +- for cmd in self._cmd_passthrough('help')['return'].split('\r\n'): +- if cmd and cmd[0] != '[' and cmd[0] != '\t': +- name = cmd.split()[0] # drop help text +- if name == 'info': +- continue +- if name.find('|') != -1: +- # Command in the form 'foobar|f' or 'f|foobar', take the +- # full name +- opt = name.split('|') +- if len(opt[0]) == 1: +- name = opt[1] +- else: +- name = opt[0] +- self._completer.append(name) +- self._completer.append('help ' + name) # help completion +- +- def _info_completion(self) -> None: +- for cmd in self._cmd_passthrough('info')['return'].split('\r\n'): +- if cmd: +- self._completer.append('info ' + cmd.split()[1]) +- +- def _other_completion(self) -> None: +- # special cases +- self._completer.append('help info') +- +- def _fill_completion(self) -> None: +- self._cmd_completion() +- self._info_completion() +- self._other_completion() +- +- def _cmd_passthrough(self, cmdline: str, +- cpu_index: int = 0) -> QMPMessage: +- return self.cmd_obj({ +- 'execute': 'human-monitor-command', +- 'arguments': { +- 'command-line': cmdline, +- 'cpu-index': cpu_index +- } +- }) +- +- def _execute_cmd(self, cmdline: str) -> bool: +- if cmdline.split()[0] == "cpu": +- # trap the cpu command, it requires special setting +- try: +- idx = int(cmdline.split()[1]) +- if 'return' not in self._cmd_passthrough('info version', idx): +- print('bad CPU index') +- return True +- self._cpu_index = idx +- except ValueError: +- print('cpu command takes an integer argument') +- return True +- resp = self._cmd_passthrough(cmdline, self._cpu_index) +- if resp is None: +- print('Disconnected') +- return False +- assert 'return' in resp or 'error' in resp +- if 'return' in resp: +- # Success +- if len(resp['return']) > 0: +- print(resp['return'], end=' ') +- else: +- # Error +- print('%s: %s' % (resp['error']['class'], resp['error']['desc'])) +- return True +- +- def show_banner(self, msg: str = 'Welcome to the HMP shell!') -> None: +- QMPShell.show_banner(self, msg) +- +- +-def die(msg: str) -> NoReturn: +- """Write an error to stderr, then exit with a return code of 1.""" +- sys.stderr.write('ERROR: %s\n' % msg) +- sys.exit(1) +- +- +-def main() -> None: +- """ +- qmp-shell entry point: parse command line arguments and start the REPL. +- """ +- parser = argparse.ArgumentParser() +- parser.add_argument('-H', '--hmp', action='store_true', +- help='Use HMP interface') +- parser.add_argument('-N', '--skip-negotiation', action='store_true', +- help='Skip negotiate (for qemu-ga)') +- parser.add_argument('-v', '--verbose', action='store_true', +- help='Verbose (echo commands sent and received)') +- parser.add_argument('-p', '--pretty', action='store_true', +- help='Pretty-print JSON') +- +- default_server = os.environ.get('QMP_SOCKET') +- parser.add_argument('qmp_server', action='store', +- default=default_server, +- help='< UNIX socket path | TCP address:port >') +- +- args = parser.parse_args() +- if args.qmp_server is None: +- parser.error("QMP socket or TCP address must be specified") +- +- shell_class = HMPShell if args.hmp else QMPShell +- +- try: +- address = shell_class.parse_address(args.qmp_server) +- except QMPBadPortError: +- parser.error(f"Bad port number: {args.qmp_server}") +- return # pycharm doesn't know error() is noreturn +- +- with shell_class(address, args.pretty, args.verbose) as qemu: +- try: +- qemu.connect(negotiate=not args.skip_negotiation) +- except ConnectError as err: +- if isinstance(err.exc, OSError): +- die(f"Couldn't connect to {args.qmp_server}: {err!s}") +- die(str(err)) +- +- for _ in qemu.repl(): +- pass +- +- +-if __name__ == '__main__': +- main() +diff --git a/python/setup.cfg b/python/setup.cfg +index 91ccef7e8fd85d0d6d3d86adbc8d..0063c757b78638ef651a362af338 100644 +--- a/python/setup.cfg ++++ b/python/setup.cfg +@@ -67,7 +67,7 @@ console_scripts = + qom-tree = qemu.utils.qom:QOMTree.entry_point + qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse] + qemu-ga-client = qemu.utils.qemu_ga_client:main +- qmp-shell = qemu.qmp.qmp_shell:main ++ qmp-shell = qemu.aqmp.qmp_shell:main + aqmp-tui = qemu.aqmp.aqmp_tui:main [tui] + + [flake8] +diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell +index 4a20f97db708b74ebfed715e38ea..31b19d73e22a4d9c91aaf0ce9725 100755 +--- a/scripts/qmp/qmp-shell ++++ b/scripts/qmp/qmp-shell +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp import qmp_shell ++from qemu.aqmp import qmp_shell + + + if __name__ == '__main__': diff --git a/python-move-qmp-utilities-to-python-qemu.patch b/python-move-qmp-utilities-to-python-qemu.patch new file mode 100644 index 00000000..e8f33517 --- /dev/null +++ b/python-move-qmp-utilities-to-python-qemu.patch @@ -0,0 +1,2153 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:54 -0500 +Subject: python: move qmp utilities to python/qemu/utils + +Git-commit: 0347c4c4cfed47e54d9dc275ceb28d35b250749f + +In order to upload a QMP package to PyPI, I want to remove any scripts +that I am not 100% confident I want to support upstream, beyond our +castle walls. + +Move most of our QMP utilities into the utils package so we can split +them out from the PyPI upload. + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Beraldo Leal +Signed-off-by: Li Zhang +--- + python/qemu/qmp/qemu_ga_client.py | 323 ---------------------------- + python/qemu/qmp/qom.py | 272 ----------------------- + python/qemu/qmp/qom_common.py | 178 --------------- + python/qemu/qmp/qom_fuse.py | 206 ------------------ + python/qemu/utils/qemu_ga_client.py | 323 ++++++++++++++++++++++++++++ + python/qemu/utils/qom.py | 272 +++++++++++++++++++++++ + python/qemu/utils/qom_common.py | 178 +++++++++++++++ + python/qemu/utils/qom_fuse.py | 206 ++++++++++++++++++ + python/setup.cfg | 16 +- + scripts/qmp/qemu-ga-client | 2 +- + scripts/qmp/qom-fuse | 2 +- + scripts/qmp/qom-get | 2 +- + scripts/qmp/qom-list | 2 +- + scripts/qmp/qom-set | 2 +- + scripts/qmp/qom-tree | 2 +- + 15 files changed, 993 insertions(+), 993 deletions(-) + +diff --git a/python/qemu/qmp/qemu_ga_client.py b/python/qemu/qmp/qemu_ga_client.py +deleted file mode 100644 +index 67ac0b421129dd03e973886ac4ac1e1e3de3d358..0000000000000000000000000000000000000000 +--- a/python/qemu/qmp/qemu_ga_client.py ++++ /dev/null +@@ -1,323 +0,0 @@ +-""" +-QEMU Guest Agent Client +- +-Usage: +- +-Start QEMU with: +- +-# qemu [...] -chardev socket,path=/tmp/qga.sock,server,wait=off,id=qga0 \ +- -device virtio-serial \ +- -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 +- +-Run the script: +- +-$ qemu-ga-client --address=/tmp/qga.sock [args...] +- +-or +- +-$ export QGA_CLIENT_ADDRESS=/tmp/qga.sock +-$ qemu-ga-client [args...] +- +-For example: +- +-$ qemu-ga-client cat /etc/resolv.conf +-# Generated by NetworkManager +-nameserver 10.0.2.3 +-$ qemu-ga-client fsfreeze status +-thawed +-$ qemu-ga-client fsfreeze freeze +-2 filesystems frozen +- +-See also: https://wiki.qemu.org/Features/QAPI/GuestAgent +-""" +- +-# Copyright (C) 2012 Ryota Ozaki +-# +-# This work is licensed under the terms of the GNU GPL, version 2. See +-# the COPYING file in the top-level directory. +- +-import argparse +-import base64 +-import errno +-import os +-import random +-import sys +-from typing import ( +- Any, +- Callable, +- Dict, +- Optional, +- Sequence, +-) +- +-from qemu import qmp +-from qemu.qmp import SocketAddrT +- +- +-# This script has not seen many patches or careful attention in quite +-# some time. If you would like to improve it, please review the design +-# carefully and add docstrings at that point in time. Until then: +- +-# pylint: disable=missing-docstring +- +- +-class QemuGuestAgent(qmp.QEMUMonitorProtocol): +- def __getattr__(self, name: str) -> Callable[..., Any]: +- def wrapper(**kwds: object) -> object: +- return self.command('guest-' + name.replace('_', '-'), **kwds) +- return wrapper +- +- +-class QemuGuestAgentClient: +- def __init__(self, address: SocketAddrT): +- self.qga = QemuGuestAgent(address) +- self.qga.connect(negotiate=False) +- +- def sync(self, timeout: Optional[float] = 3) -> None: +- # Avoid being blocked forever +- if not self.ping(timeout): +- raise EnvironmentError('Agent seems not alive') +- uid = random.randint(0, (1 << 32) - 1) +- while True: +- ret = self.qga.sync(id=uid) +- if isinstance(ret, int) and int(ret) == uid: +- break +- +- def __file_read_all(self, handle: int) -> bytes: +- eof = False +- data = b'' +- while not eof: +- ret = self.qga.file_read(handle=handle, count=1024) +- _data = base64.b64decode(ret['buf-b64']) +- data += _data +- eof = ret['eof'] +- return data +- +- def read(self, path: str) -> bytes: +- handle = self.qga.file_open(path=path) +- try: +- data = self.__file_read_all(handle) +- finally: +- self.qga.file_close(handle=handle) +- return data +- +- def info(self) -> str: +- info = self.qga.info() +- +- msgs = [] +- msgs.append('version: ' + info['version']) +- msgs.append('supported_commands:') +- enabled = [c['name'] for c in info['supported_commands'] +- if c['enabled']] +- msgs.append('\tenabled: ' + ', '.join(enabled)) +- disabled = [c['name'] for c in info['supported_commands'] +- if not c['enabled']] +- msgs.append('\tdisabled: ' + ', '.join(disabled)) +- +- return '\n'.join(msgs) +- +- @classmethod +- def __gen_ipv4_netmask(cls, prefixlen: int) -> str: +- mask = int('1' * prefixlen + '0' * (32 - prefixlen), 2) +- return '.'.join([str(mask >> 24), +- str((mask >> 16) & 0xff), +- str((mask >> 8) & 0xff), +- str(mask & 0xff)]) +- +- def ifconfig(self) -> str: +- nifs = self.qga.network_get_interfaces() +- +- msgs = [] +- for nif in nifs: +- msgs.append(nif['name'] + ':') +- if 'ip-addresses' in nif: +- for ipaddr in nif['ip-addresses']: +- if ipaddr['ip-address-type'] == 'ipv4': +- addr = ipaddr['ip-address'] +- mask = self.__gen_ipv4_netmask(int(ipaddr['prefix'])) +- msgs.append(f"\tinet {addr} netmask {mask}") +- elif ipaddr['ip-address-type'] == 'ipv6': +- addr = ipaddr['ip-address'] +- prefix = ipaddr['prefix'] +- msgs.append(f"\tinet6 {addr} prefixlen {prefix}") +- if nif['hardware-address'] != '00:00:00:00:00:00': +- msgs.append("\tether " + nif['hardware-address']) +- +- return '\n'.join(msgs) +- +- def ping(self, timeout: Optional[float]) -> bool: +- self.qga.settimeout(timeout) +- try: +- self.qga.ping() +- except TimeoutError: +- return False +- return True +- +- def fsfreeze(self, cmd: str) -> object: +- if cmd not in ['status', 'freeze', 'thaw']: +- raise Exception('Invalid command: ' + cmd) +- # Can be int (freeze, thaw) or GuestFsfreezeStatus (status) +- return getattr(self.qga, 'fsfreeze' + '_' + cmd)() +- +- def fstrim(self, minimum: int) -> Dict[str, object]: +- # returns GuestFilesystemTrimResponse +- ret = getattr(self.qga, 'fstrim')(minimum=minimum) +- assert isinstance(ret, dict) +- return ret +- +- def suspend(self, mode: str) -> None: +- if mode not in ['disk', 'ram', 'hybrid']: +- raise Exception('Invalid mode: ' + mode) +- +- try: +- getattr(self.qga, 'suspend' + '_' + mode)() +- # On error exception will raise +- except TimeoutError: +- # On success command will timed out +- return +- +- def shutdown(self, mode: str = 'powerdown') -> None: +- if mode not in ['powerdown', 'halt', 'reboot']: +- raise Exception('Invalid mode: ' + mode) +- +- try: +- self.qga.shutdown(mode=mode) +- except TimeoutError: +- pass +- +- +-def _cmd_cat(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- if len(args) != 1: +- print('Invalid argument') +- print('Usage: cat ') +- sys.exit(1) +- print(client.read(args[0])) +- +- +-def _cmd_fsfreeze(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- usage = 'Usage: fsfreeze status|freeze|thaw' +- if len(args) != 1: +- print('Invalid argument') +- print(usage) +- sys.exit(1) +- if args[0] not in ['status', 'freeze', 'thaw']: +- print('Invalid command: ' + args[0]) +- print(usage) +- sys.exit(1) +- cmd = args[0] +- ret = client.fsfreeze(cmd) +- if cmd == 'status': +- print(ret) +- return +- +- assert isinstance(ret, int) +- verb = 'frozen' if cmd == 'freeze' else 'thawed' +- print(f"{ret:d} filesystems {verb}") +- +- +-def _cmd_fstrim(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- if len(args) == 0: +- minimum = 0 +- else: +- minimum = int(args[0]) +- print(client.fstrim(minimum)) +- +- +-def _cmd_ifconfig(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- assert not args +- print(client.ifconfig()) +- +- +-def _cmd_info(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- assert not args +- print(client.info()) +- +- +-def _cmd_ping(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- timeout = 3.0 if len(args) == 0 else float(args[0]) +- alive = client.ping(timeout) +- if not alive: +- print("Not responded in %s sec" % args[0]) +- sys.exit(1) +- +- +-def _cmd_suspend(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- usage = 'Usage: suspend disk|ram|hybrid' +- if len(args) != 1: +- print('Less argument') +- print(usage) +- sys.exit(1) +- if args[0] not in ['disk', 'ram', 'hybrid']: +- print('Invalid command: ' + args[0]) +- print(usage) +- sys.exit(1) +- client.suspend(args[0]) +- +- +-def _cmd_shutdown(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- assert not args +- client.shutdown() +- +- +-_cmd_powerdown = _cmd_shutdown +- +- +-def _cmd_halt(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- assert not args +- client.shutdown('halt') +- +- +-def _cmd_reboot(client: QemuGuestAgentClient, args: Sequence[str]) -> None: +- assert not args +- client.shutdown('reboot') +- +- +-commands = [m.replace('_cmd_', '') for m in dir() if '_cmd_' in m] +- +- +-def send_command(address: str, cmd: str, args: Sequence[str]) -> None: +- if not os.path.exists(address): +- print('%s not found' % address) +- sys.exit(1) +- +- if cmd not in commands: +- print('Invalid command: ' + cmd) +- print('Available commands: ' + ', '.join(commands)) +- sys.exit(1) +- +- try: +- client = QemuGuestAgentClient(address) +- except OSError as err: +- print(err) +- if err.errno == errno.ECONNREFUSED: +- print('Hint: qemu is not running?') +- sys.exit(1) +- +- if cmd == 'fsfreeze' and args[0] == 'freeze': +- client.sync(60) +- elif cmd != 'ping': +- client.sync() +- +- globals()['_cmd_' + cmd](client, args) +- +- +-def main() -> None: +- address = os.environ.get('QGA_CLIENT_ADDRESS') +- +- parser = argparse.ArgumentParser() +- parser.add_argument('--address', action='store', +- default=address, +- help='Specify a ip:port pair or a unix socket path') +- parser.add_argument('command', choices=commands) +- parser.add_argument('args', nargs='*') +- +- args = parser.parse_args() +- if args.address is None: +- parser.error('address is not specified') +- sys.exit(1) +- +- send_command(args.address, args.command, args.args) +- +- +-if __name__ == '__main__': +- main() +diff --git a/python/qemu/qmp/qom.py b/python/qemu/qmp/qom.py +deleted file mode 100644 +index 8ff28a83439767ce37db21d8790c50ddb4845f50..0000000000000000000000000000000000000000 +--- a/python/qemu/qmp/qom.py ++++ /dev/null +@@ -1,272 +0,0 @@ +-""" +-QEMU Object Model testing tools. +- +-usage: qom [-h] {set,get,list,tree,fuse} ... +- +-Query and manipulate QOM data +- +-optional arguments: +- -h, --help show this help message and exit +- +-QOM commands: +- {set,get,list,tree,fuse} +- set Set a QOM property value +- get Get a QOM property value +- list List QOM properties at a given path +- tree Show QOM tree from a given path +- fuse Mount a QOM tree as a FUSE filesystem +-""" +-## +-# Copyright John Snow 2020, for Red Hat, Inc. +-# Copyright IBM, Corp. 2011 +-# +-# Authors: +-# John Snow +-# Anthony Liguori +-# +-# This work is licensed under the terms of the GNU GPL, version 2 or later. +-# See the COPYING file in the top-level directory. +-# +-# Based on ./scripts/qmp/qom-[set|get|tree|list] +-## +- +-import argparse +- +-from . import QMPResponseError +-from .qom_common import QOMCommand +- +- +-try: +- from .qom_fuse import QOMFuse +-except ModuleNotFoundError as _err: +- if _err.name != 'fuse': +- raise +-else: +- assert issubclass(QOMFuse, QOMCommand) +- +- +-class QOMSet(QOMCommand): +- """ +- QOM Command - Set a property to a given value. +- +- usage: qom-set [-h] [--socket SOCKET] . +- +- Set a QOM property value +- +- positional arguments: +- . QOM path and property, separated by a period '.' +- new QOM property value +- +- optional arguments: +- -h, --help show this help message and exit +- --socket SOCKET, -s SOCKET +- QMP socket path or address (addr:port). May also be +- set via QMP_SOCKET environment variable. +- """ +- name = 'set' +- help = 'Set a QOM property value' +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- super().configure_parser(parser) +- cls.add_path_prop_arg(parser) +- parser.add_argument( +- 'value', +- metavar='', +- action='store', +- help='new QOM property value' +- ) +- +- def __init__(self, args: argparse.Namespace): +- super().__init__(args) +- self.path, self.prop = args.path_prop.rsplit('.', 1) +- self.value = args.value +- +- def run(self) -> int: +- rsp = self.qmp.command( +- 'qom-set', +- path=self.path, +- property=self.prop, +- value=self.value +- ) +- print(rsp) +- return 0 +- +- +-class QOMGet(QOMCommand): +- """ +- QOM Command - Get a property's current value. +- +- usage: qom-get [-h] [--socket SOCKET] . +- +- Get a QOM property value +- +- positional arguments: +- . QOM path and property, separated by a period '.' +- +- optional arguments: +- -h, --help show this help message and exit +- --socket SOCKET, -s SOCKET +- QMP socket path or address (addr:port). May also be +- set via QMP_SOCKET environment variable. +- """ +- name = 'get' +- help = 'Get a QOM property value' +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- super().configure_parser(parser) +- cls.add_path_prop_arg(parser) +- +- def __init__(self, args: argparse.Namespace): +- super().__init__(args) +- try: +- tmp = args.path_prop.rsplit('.', 1) +- except ValueError as err: +- raise ValueError('Invalid format for .') from err +- self.path = tmp[0] +- self.prop = tmp[1] +- +- def run(self) -> int: +- rsp = self.qmp.command( +- 'qom-get', +- path=self.path, +- property=self.prop +- ) +- if isinstance(rsp, dict): +- for key, value in rsp.items(): +- print(f"{key}: {value}") +- else: +- print(rsp) +- return 0 +- +- +-class QOMList(QOMCommand): +- """ +- QOM Command - List the properties at a given path. +- +- usage: qom-list [-h] [--socket SOCKET] +- +- List QOM properties at a given path +- +- positional arguments: +- QOM path +- +- optional arguments: +- -h, --help show this help message and exit +- --socket SOCKET, -s SOCKET +- QMP socket path or address (addr:port). May also be +- set via QMP_SOCKET environment variable. +- """ +- name = 'list' +- help = 'List QOM properties at a given path' +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- super().configure_parser(parser) +- parser.add_argument( +- 'path', +- metavar='', +- action='store', +- help='QOM path', +- ) +- +- def __init__(self, args: argparse.Namespace): +- super().__init__(args) +- self.path = args.path +- +- def run(self) -> int: +- rsp = self.qom_list(self.path) +- for item in rsp: +- if item.child: +- print(f"{item.name}/") +- elif item.link: +- print(f"@{item.name}/") +- else: +- print(item.name) +- return 0 +- +- +-class QOMTree(QOMCommand): +- """ +- QOM Command - Show the full tree below a given path. +- +- usage: qom-tree [-h] [--socket SOCKET] [] +- +- Show QOM tree from a given path +- +- positional arguments: +- QOM path +- +- optional arguments: +- -h, --help show this help message and exit +- --socket SOCKET, -s SOCKET +- QMP socket path or address (addr:port). May also be +- set via QMP_SOCKET environment variable. +- """ +- name = 'tree' +- help = 'Show QOM tree from a given path' +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- super().configure_parser(parser) +- parser.add_argument( +- 'path', +- metavar='', +- action='store', +- help='QOM path', +- nargs='?', +- default='/' +- ) +- +- def __init__(self, args: argparse.Namespace): +- super().__init__(args) +- self.path = args.path +- +- def _list_node(self, path: str) -> None: +- print(path) +- items = self.qom_list(path) +- for item in items: +- if item.child: +- continue +- try: +- rsp = self.qmp.command('qom-get', path=path, +- property=item.name) +- print(f" {item.name}: {rsp} ({item.type})") +- except QMPResponseError as err: +- print(f" {item.name}: ({item.type})") +- print('') +- for item in items: +- if not item.child: +- continue +- if path == '/': +- path = '' +- self._list_node(f"{path}/{item.name}") +- +- def run(self) -> int: +- self._list_node(self.path) +- return 0 +- +- +-def main() -> int: +- """QOM script main entry point.""" +- parser = argparse.ArgumentParser( +- description='Query and manipulate QOM data' +- ) +- subparsers = parser.add_subparsers( +- title='QOM commands', +- dest='command' +- ) +- +- for command in QOMCommand.__subclasses__(): +- command.register(subparsers) +- +- args = parser.parse_args() +- +- if args.command is None: +- parser.error('Command not specified.') +- return 1 +- +- cmd_class = args.cmd_class +- assert isinstance(cmd_class, type(QOMCommand)) +- return cmd_class.command_runner(args) +diff --git a/python/qemu/qmp/qom_common.py b/python/qemu/qmp/qom_common.py +deleted file mode 100644 +index a59ae1a2a1883cb4d89b0e44507c5001f44357a0..0000000000000000000000000000000000000000 +--- a/python/qemu/qmp/qom_common.py ++++ /dev/null +@@ -1,178 +0,0 @@ +-""" +-QOM Command abstractions. +-""" +-## +-# Copyright John Snow 2020, for Red Hat, Inc. +-# Copyright IBM, Corp. 2011 +-# +-# Authors: +-# John Snow +-# Anthony Liguori +-# +-# This work is licensed under the terms of the GNU GPL, version 2 or later. +-# See the COPYING file in the top-level directory. +-# +-# Based on ./scripts/qmp/qom-[set|get|tree|list] +-## +- +-import argparse +-import os +-import sys +-from typing import ( +- Any, +- Dict, +- List, +- Optional, +- Type, +- TypeVar, +-) +- +-from . import QEMUMonitorProtocol, QMPError +- +- +-# The following is needed only for a type alias. +-Subparsers = argparse._SubParsersAction # pylint: disable=protected-access +- +- +-class ObjectPropertyInfo: +- """ +- Represents the return type from e.g. qom-list. +- """ +- def __init__(self, name: str, type_: str, +- description: Optional[str] = None, +- default_value: Optional[object] = None): +- self.name = name +- self.type = type_ +- self.description = description +- self.default_value = default_value +- +- @classmethod +- def make(cls, value: Dict[str, Any]) -> 'ObjectPropertyInfo': +- """ +- Build an ObjectPropertyInfo from a Dict with an unknown shape. +- """ +- assert value.keys() >= {'name', 'type'} +- assert value.keys() <= {'name', 'type', 'description', 'default-value'} +- return cls(value['name'], value['type'], +- value.get('description'), +- value.get('default-value')) +- +- @property +- def child(self) -> bool: +- """Is this property a child property?""" +- return self.type.startswith('child<') +- +- @property +- def link(self) -> bool: +- """Is this property a link property?""" +- return self.type.startswith('link<') +- +- +-CommandT = TypeVar('CommandT', bound='QOMCommand') +- +- +-class QOMCommand: +- """ +- Represents a QOM sub-command. +- +- :param args: Parsed arguments, as returned from parser.parse_args. +- """ +- name: str +- help: str +- +- def __init__(self, args: argparse.Namespace): +- if args.socket is None: +- raise QMPError("No QMP socket path or address given") +- self.qmp = QEMUMonitorProtocol( +- QEMUMonitorProtocol.parse_address(args.socket) +- ) +- self.qmp.connect() +- +- @classmethod +- def register(cls, subparsers: Subparsers) -> None: +- """ +- Register this command with the argument parser. +- +- :param subparsers: argparse subparsers object, from "add_subparsers". +- """ +- subparser = subparsers.add_parser(cls.name, help=cls.help, +- description=cls.help) +- cls.configure_parser(subparser) +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- """ +- Configure a parser with this command's arguments. +- +- :param parser: argparse parser or subparser object. +- """ +- default_path = os.environ.get('QMP_SOCKET') +- parser.add_argument( +- '--socket', '-s', +- dest='socket', +- action='store', +- help='QMP socket path or address (addr:port).' +- ' May also be set via QMP_SOCKET environment variable.', +- default=default_path +- ) +- parser.set_defaults(cmd_class=cls) +- +- @classmethod +- def add_path_prop_arg(cls, parser: argparse.ArgumentParser) -> None: +- """ +- Add the . positional argument to this command. +- +- :param parser: The parser to add the argument to. +- """ +- parser.add_argument( +- 'path_prop', +- metavar='.', +- action='store', +- help="QOM path and property, separated by a period '.'" +- ) +- +- def run(self) -> int: +- """ +- Run this command. +- +- :return: 0 on success, 1 otherwise. +- """ +- raise NotImplementedError +- +- def qom_list(self, path: str) -> List[ObjectPropertyInfo]: +- """ +- :return: a strongly typed list from the 'qom-list' command. +- """ +- rsp = self.qmp.command('qom-list', path=path) +- # qom-list returns List[ObjectPropertyInfo] +- assert isinstance(rsp, list) +- return [ObjectPropertyInfo.make(x) for x in rsp] +- +- @classmethod +- def command_runner( +- cls: Type[CommandT], +- args: argparse.Namespace +- ) -> int: +- """ +- Run a fully-parsed subcommand, with error-handling for the CLI. +- +- :return: The return code from `run()`. +- """ +- try: +- cmd = cls(args) +- return cmd.run() +- except QMPError as err: +- print(f"{type(err).__name__}: {err!s}", file=sys.stderr) +- return -1 +- +- @classmethod +- def entry_point(cls) -> int: +- """ +- Build this command's parser, parse arguments, and run the command. +- +- :return: `run`'s return code. +- """ +- parser = argparse.ArgumentParser(description=cls.help) +- cls.configure_parser(parser) +- args = parser.parse_args() +- return cls.command_runner(args) +diff --git a/python/qemu/qmp/qom_fuse.py b/python/qemu/qmp/qom_fuse.py +deleted file mode 100644 +index 43f4671fdb18c6aa1b11df9694855167fced8f10..0000000000000000000000000000000000000000 +--- a/python/qemu/qmp/qom_fuse.py ++++ /dev/null +@@ -1,206 +0,0 @@ +-""" +-QEMU Object Model FUSE filesystem tool +- +-This script offers a simple FUSE filesystem within which the QOM tree +-may be browsed, queried and edited using traditional shell tooling. +- +-This script requires the 'fusepy' python package. +- +- +-usage: qom-fuse [-h] [--socket SOCKET] +- +-Mount a QOM tree as a FUSE filesystem +- +-positional arguments: +- Mount point +- +-optional arguments: +- -h, --help show this help message and exit +- --socket SOCKET, -s SOCKET +- QMP socket path or address (addr:port). May also be +- set via QMP_SOCKET environment variable. +-""" +-## +-# Copyright IBM, Corp. 2012 +-# Copyright (C) 2020 Red Hat, Inc. +-# +-# Authors: +-# Anthony Liguori +-# Markus Armbruster +-# +-# This work is licensed under the terms of the GNU GPL, version 2 or later. +-# See the COPYING file in the top-level directory. +-## +- +-import argparse +-from errno import ENOENT, EPERM +-import stat +-import sys +-from typing import ( +- IO, +- Dict, +- Iterator, +- Mapping, +- Optional, +- Union, +-) +- +-import fuse +-from fuse import FUSE, FuseOSError, Operations +- +-from . import QMPResponseError +-from .qom_common import QOMCommand +- +- +-fuse.fuse_python_api = (0, 2) +- +- +-class QOMFuse(QOMCommand, Operations): +- """ +- QOMFuse implements both fuse.Operations and QOMCommand. +- +- Operations implements the FS, and QOMCommand implements the CLI command. +- """ +- name = 'fuse' +- help = 'Mount a QOM tree as a FUSE filesystem' +- fuse: FUSE +- +- @classmethod +- def configure_parser(cls, parser: argparse.ArgumentParser) -> None: +- super().configure_parser(parser) +- parser.add_argument( +- 'mount', +- metavar='', +- action='store', +- help="Mount point", +- ) +- +- def __init__(self, args: argparse.Namespace): +- super().__init__(args) +- self.mount = args.mount +- self.ino_map: Dict[str, int] = {} +- self.ino_count = 1 +- +- def run(self) -> int: +- print(f"Mounting QOMFS to '{self.mount}'", file=sys.stderr) +- self.fuse = FUSE(self, self.mount, foreground=True) +- return 0 +- +- def get_ino(self, path: str) -> int: +- """Get an inode number for a given QOM path.""" +- if path in self.ino_map: +- return self.ino_map[path] +- self.ino_map[path] = self.ino_count +- self.ino_count += 1 +- return self.ino_map[path] +- +- def is_object(self, path: str) -> bool: +- """Is the given QOM path an object?""" +- try: +- self.qom_list(path) +- return True +- except QMPResponseError: +- return False +- +- def is_property(self, path: str) -> bool: +- """Is the given QOM path a property?""" +- path, prop = path.rsplit('/', 1) +- if path == '': +- path = '/' +- try: +- for item in self.qom_list(path): +- if item.name == prop: +- return True +- return False +- except QMPResponseError: +- return False +- +- def is_link(self, path: str) -> bool: +- """Is the given QOM path a link?""" +- path, prop = path.rsplit('/', 1) +- if path == '': +- path = '/' +- try: +- for item in self.qom_list(path): +- if item.name == prop and item.link: +- return True +- return False +- except QMPResponseError: +- return False +- +- def read(self, path: str, size: int, offset: int, fh: IO[bytes]) -> bytes: +- if not self.is_property(path): +- raise FuseOSError(ENOENT) +- +- path, prop = path.rsplit('/', 1) +- if path == '': +- path = '/' +- try: +- data = str(self.qmp.command('qom-get', path=path, property=prop)) +- data += '\n' # make values shell friendly +- except QMPResponseError as err: +- raise FuseOSError(EPERM) from err +- +- if offset > len(data): +- return b'' +- +- return bytes(data[offset:][:size], encoding='utf-8') +- +- def readlink(self, path: str) -> Union[bool, str]: +- if not self.is_link(path): +- return False +- path, prop = path.rsplit('/', 1) +- prefix = '/'.join(['..'] * (len(path.split('/')) - 1)) +- return prefix + str(self.qmp.command('qom-get', path=path, +- property=prop)) +- +- def getattr(self, path: str, +- fh: Optional[IO[bytes]] = None) -> Mapping[str, object]: +- if self.is_link(path): +- value = { +- 'st_mode': 0o755 | stat.S_IFLNK, +- 'st_ino': self.get_ino(path), +- 'st_dev': 0, +- 'st_nlink': 2, +- 'st_uid': 1000, +- 'st_gid': 1000, +- 'st_size': 4096, +- 'st_atime': 0, +- 'st_mtime': 0, +- 'st_ctime': 0 +- } +- elif self.is_object(path): +- value = { +- 'st_mode': 0o755 | stat.S_IFDIR, +- 'st_ino': self.get_ino(path), +- 'st_dev': 0, +- 'st_nlink': 2, +- 'st_uid': 1000, +- 'st_gid': 1000, +- 'st_size': 4096, +- 'st_atime': 0, +- 'st_mtime': 0, +- 'st_ctime': 0 +- } +- elif self.is_property(path): +- value = { +- 'st_mode': 0o644 | stat.S_IFREG, +- 'st_ino': self.get_ino(path), +- 'st_dev': 0, +- 'st_nlink': 1, +- 'st_uid': 1000, +- 'st_gid': 1000, +- 'st_size': 4096, +- 'st_atime': 0, +- 'st_mtime': 0, +- 'st_ctime': 0 +- } +- else: +- raise FuseOSError(ENOENT) +- return value +- +- def readdir(self, path: str, fh: IO[bytes]) -> Iterator[str]: +- yield '.' +- yield '..' +- for item in self.qom_list(path): +- yield item.name +diff --git a/python/qemu/utils/qemu_ga_client.py b/python/qemu/utils/qemu_ga_client.py +new file mode 100644 +index 0000000000000000000000000000000000000000..67ac0b421129dd03e973886ac4ac1e1e3de3d358 +--- /dev/null ++++ b/python/qemu/utils/qemu_ga_client.py +@@ -0,0 +1,323 @@ ++""" ++QEMU Guest Agent Client ++ ++Usage: ++ ++Start QEMU with: ++ ++# qemu [...] -chardev socket,path=/tmp/qga.sock,server,wait=off,id=qga0 \ ++ -device virtio-serial \ ++ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 ++ ++Run the script: ++ ++$ qemu-ga-client --address=/tmp/qga.sock [args...] ++ ++or ++ ++$ export QGA_CLIENT_ADDRESS=/tmp/qga.sock ++$ qemu-ga-client [args...] ++ ++For example: ++ ++$ qemu-ga-client cat /etc/resolv.conf ++# Generated by NetworkManager ++nameserver 10.0.2.3 ++$ qemu-ga-client fsfreeze status ++thawed ++$ qemu-ga-client fsfreeze freeze ++2 filesystems frozen ++ ++See also: https://wiki.qemu.org/Features/QAPI/GuestAgent ++""" ++ ++# Copyright (C) 2012 Ryota Ozaki ++# ++# This work is licensed under the terms of the GNU GPL, version 2. See ++# the COPYING file in the top-level directory. ++ ++import argparse ++import base64 ++import errno ++import os ++import random ++import sys ++from typing import ( ++ Any, ++ Callable, ++ Dict, ++ Optional, ++ Sequence, ++) ++ ++from qemu import qmp ++from qemu.qmp import SocketAddrT ++ ++ ++# This script has not seen many patches or careful attention in quite ++# some time. If you would like to improve it, please review the design ++# carefully and add docstrings at that point in time. Until then: ++ ++# pylint: disable=missing-docstring ++ ++ ++class QemuGuestAgent(qmp.QEMUMonitorProtocol): ++ def __getattr__(self, name: str) -> Callable[..., Any]: ++ def wrapper(**kwds: object) -> object: ++ return self.command('guest-' + name.replace('_', '-'), **kwds) ++ return wrapper ++ ++ ++class QemuGuestAgentClient: ++ def __init__(self, address: SocketAddrT): ++ self.qga = QemuGuestAgent(address) ++ self.qga.connect(negotiate=False) ++ ++ def sync(self, timeout: Optional[float] = 3) -> None: ++ # Avoid being blocked forever ++ if not self.ping(timeout): ++ raise EnvironmentError('Agent seems not alive') ++ uid = random.randint(0, (1 << 32) - 1) ++ while True: ++ ret = self.qga.sync(id=uid) ++ if isinstance(ret, int) and int(ret) == uid: ++ break ++ ++ def __file_read_all(self, handle: int) -> bytes: ++ eof = False ++ data = b'' ++ while not eof: ++ ret = self.qga.file_read(handle=handle, count=1024) ++ _data = base64.b64decode(ret['buf-b64']) ++ data += _data ++ eof = ret['eof'] ++ return data ++ ++ def read(self, path: str) -> bytes: ++ handle = self.qga.file_open(path=path) ++ try: ++ data = self.__file_read_all(handle) ++ finally: ++ self.qga.file_close(handle=handle) ++ return data ++ ++ def info(self) -> str: ++ info = self.qga.info() ++ ++ msgs = [] ++ msgs.append('version: ' + info['version']) ++ msgs.append('supported_commands:') ++ enabled = [c['name'] for c in info['supported_commands'] ++ if c['enabled']] ++ msgs.append('\tenabled: ' + ', '.join(enabled)) ++ disabled = [c['name'] for c in info['supported_commands'] ++ if not c['enabled']] ++ msgs.append('\tdisabled: ' + ', '.join(disabled)) ++ ++ return '\n'.join(msgs) ++ ++ @classmethod ++ def __gen_ipv4_netmask(cls, prefixlen: int) -> str: ++ mask = int('1' * prefixlen + '0' * (32 - prefixlen), 2) ++ return '.'.join([str(mask >> 24), ++ str((mask >> 16) & 0xff), ++ str((mask >> 8) & 0xff), ++ str(mask & 0xff)]) ++ ++ def ifconfig(self) -> str: ++ nifs = self.qga.network_get_interfaces() ++ ++ msgs = [] ++ for nif in nifs: ++ msgs.append(nif['name'] + ':') ++ if 'ip-addresses' in nif: ++ for ipaddr in nif['ip-addresses']: ++ if ipaddr['ip-address-type'] == 'ipv4': ++ addr = ipaddr['ip-address'] ++ mask = self.__gen_ipv4_netmask(int(ipaddr['prefix'])) ++ msgs.append(f"\tinet {addr} netmask {mask}") ++ elif ipaddr['ip-address-type'] == 'ipv6': ++ addr = ipaddr['ip-address'] ++ prefix = ipaddr['prefix'] ++ msgs.append(f"\tinet6 {addr} prefixlen {prefix}") ++ if nif['hardware-address'] != '00:00:00:00:00:00': ++ msgs.append("\tether " + nif['hardware-address']) ++ ++ return '\n'.join(msgs) ++ ++ def ping(self, timeout: Optional[float]) -> bool: ++ self.qga.settimeout(timeout) ++ try: ++ self.qga.ping() ++ except TimeoutError: ++ return False ++ return True ++ ++ def fsfreeze(self, cmd: str) -> object: ++ if cmd not in ['status', 'freeze', 'thaw']: ++ raise Exception('Invalid command: ' + cmd) ++ # Can be int (freeze, thaw) or GuestFsfreezeStatus (status) ++ return getattr(self.qga, 'fsfreeze' + '_' + cmd)() ++ ++ def fstrim(self, minimum: int) -> Dict[str, object]: ++ # returns GuestFilesystemTrimResponse ++ ret = getattr(self.qga, 'fstrim')(minimum=minimum) ++ assert isinstance(ret, dict) ++ return ret ++ ++ def suspend(self, mode: str) -> None: ++ if mode not in ['disk', 'ram', 'hybrid']: ++ raise Exception('Invalid mode: ' + mode) ++ ++ try: ++ getattr(self.qga, 'suspend' + '_' + mode)() ++ # On error exception will raise ++ except TimeoutError: ++ # On success command will timed out ++ return ++ ++ def shutdown(self, mode: str = 'powerdown') -> None: ++ if mode not in ['powerdown', 'halt', 'reboot']: ++ raise Exception('Invalid mode: ' + mode) ++ ++ try: ++ self.qga.shutdown(mode=mode) ++ except TimeoutError: ++ pass ++ ++ ++def _cmd_cat(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ if len(args) != 1: ++ print('Invalid argument') ++ print('Usage: cat ') ++ sys.exit(1) ++ print(client.read(args[0])) ++ ++ ++def _cmd_fsfreeze(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ usage = 'Usage: fsfreeze status|freeze|thaw' ++ if len(args) != 1: ++ print('Invalid argument') ++ print(usage) ++ sys.exit(1) ++ if args[0] not in ['status', 'freeze', 'thaw']: ++ print('Invalid command: ' + args[0]) ++ print(usage) ++ sys.exit(1) ++ cmd = args[0] ++ ret = client.fsfreeze(cmd) ++ if cmd == 'status': ++ print(ret) ++ return ++ ++ assert isinstance(ret, int) ++ verb = 'frozen' if cmd == 'freeze' else 'thawed' ++ print(f"{ret:d} filesystems {verb}") ++ ++ ++def _cmd_fstrim(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ if len(args) == 0: ++ minimum = 0 ++ else: ++ minimum = int(args[0]) ++ print(client.fstrim(minimum)) ++ ++ ++def _cmd_ifconfig(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ assert not args ++ print(client.ifconfig()) ++ ++ ++def _cmd_info(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ assert not args ++ print(client.info()) ++ ++ ++def _cmd_ping(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ timeout = 3.0 if len(args) == 0 else float(args[0]) ++ alive = client.ping(timeout) ++ if not alive: ++ print("Not responded in %s sec" % args[0]) ++ sys.exit(1) ++ ++ ++def _cmd_suspend(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ usage = 'Usage: suspend disk|ram|hybrid' ++ if len(args) != 1: ++ print('Less argument') ++ print(usage) ++ sys.exit(1) ++ if args[0] not in ['disk', 'ram', 'hybrid']: ++ print('Invalid command: ' + args[0]) ++ print(usage) ++ sys.exit(1) ++ client.suspend(args[0]) ++ ++ ++def _cmd_shutdown(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ assert not args ++ client.shutdown() ++ ++ ++_cmd_powerdown = _cmd_shutdown ++ ++ ++def _cmd_halt(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ assert not args ++ client.shutdown('halt') ++ ++ ++def _cmd_reboot(client: QemuGuestAgentClient, args: Sequence[str]) -> None: ++ assert not args ++ client.shutdown('reboot') ++ ++ ++commands = [m.replace('_cmd_', '') for m in dir() if '_cmd_' in m] ++ ++ ++def send_command(address: str, cmd: str, args: Sequence[str]) -> None: ++ if not os.path.exists(address): ++ print('%s not found' % address) ++ sys.exit(1) ++ ++ if cmd not in commands: ++ print('Invalid command: ' + cmd) ++ print('Available commands: ' + ', '.join(commands)) ++ sys.exit(1) ++ ++ try: ++ client = QemuGuestAgentClient(address) ++ except OSError as err: ++ print(err) ++ if err.errno == errno.ECONNREFUSED: ++ print('Hint: qemu is not running?') ++ sys.exit(1) ++ ++ if cmd == 'fsfreeze' and args[0] == 'freeze': ++ client.sync(60) ++ elif cmd != 'ping': ++ client.sync() ++ ++ globals()['_cmd_' + cmd](client, args) ++ ++ ++def main() -> None: ++ address = os.environ.get('QGA_CLIENT_ADDRESS') ++ ++ parser = argparse.ArgumentParser() ++ parser.add_argument('--address', action='store', ++ default=address, ++ help='Specify a ip:port pair or a unix socket path') ++ parser.add_argument('command', choices=commands) ++ parser.add_argument('args', nargs='*') ++ ++ args = parser.parse_args() ++ if args.address is None: ++ parser.error('address is not specified') ++ sys.exit(1) ++ ++ send_command(args.address, args.command, args.args) ++ ++ ++if __name__ == '__main__': ++ main() +diff --git a/python/qemu/utils/qom.py b/python/qemu/utils/qom.py +new file mode 100644 +index 0000000000000000000000000000000000000000..8ff28a83439767ce37db21d8790c50ddb4845f50 +--- /dev/null ++++ b/python/qemu/utils/qom.py +@@ -0,0 +1,272 @@ ++""" ++QEMU Object Model testing tools. ++ ++usage: qom [-h] {set,get,list,tree,fuse} ... ++ ++Query and manipulate QOM data ++ ++optional arguments: ++ -h, --help show this help message and exit ++ ++QOM commands: ++ {set,get,list,tree,fuse} ++ set Set a QOM property value ++ get Get a QOM property value ++ list List QOM properties at a given path ++ tree Show QOM tree from a given path ++ fuse Mount a QOM tree as a FUSE filesystem ++""" ++## ++# Copyright John Snow 2020, for Red Hat, Inc. ++# Copyright IBM, Corp. 2011 ++# ++# Authors: ++# John Snow ++# Anthony Liguori ++# ++# This work is licensed under the terms of the GNU GPL, version 2 or later. ++# See the COPYING file in the top-level directory. ++# ++# Based on ./scripts/qmp/qom-[set|get|tree|list] ++## ++ ++import argparse ++ ++from . import QMPResponseError ++from .qom_common import QOMCommand ++ ++ ++try: ++ from .qom_fuse import QOMFuse ++except ModuleNotFoundError as _err: ++ if _err.name != 'fuse': ++ raise ++else: ++ assert issubclass(QOMFuse, QOMCommand) ++ ++ ++class QOMSet(QOMCommand): ++ """ ++ QOM Command - Set a property to a given value. ++ ++ usage: qom-set [-h] [--socket SOCKET] . ++ ++ Set a QOM property value ++ ++ positional arguments: ++ . QOM path and property, separated by a period '.' ++ new QOM property value ++ ++ optional arguments: ++ -h, --help show this help message and exit ++ --socket SOCKET, -s SOCKET ++ QMP socket path or address (addr:port). May also be ++ set via QMP_SOCKET environment variable. ++ """ ++ name = 'set' ++ help = 'Set a QOM property value' ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ super().configure_parser(parser) ++ cls.add_path_prop_arg(parser) ++ parser.add_argument( ++ 'value', ++ metavar='', ++ action='store', ++ help='new QOM property value' ++ ) ++ ++ def __init__(self, args: argparse.Namespace): ++ super().__init__(args) ++ self.path, self.prop = args.path_prop.rsplit('.', 1) ++ self.value = args.value ++ ++ def run(self) -> int: ++ rsp = self.qmp.command( ++ 'qom-set', ++ path=self.path, ++ property=self.prop, ++ value=self.value ++ ) ++ print(rsp) ++ return 0 ++ ++ ++class QOMGet(QOMCommand): ++ """ ++ QOM Command - Get a property's current value. ++ ++ usage: qom-get [-h] [--socket SOCKET] . ++ ++ Get a QOM property value ++ ++ positional arguments: ++ . QOM path and property, separated by a period '.' ++ ++ optional arguments: ++ -h, --help show this help message and exit ++ --socket SOCKET, -s SOCKET ++ QMP socket path or address (addr:port). May also be ++ set via QMP_SOCKET environment variable. ++ """ ++ name = 'get' ++ help = 'Get a QOM property value' ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ super().configure_parser(parser) ++ cls.add_path_prop_arg(parser) ++ ++ def __init__(self, args: argparse.Namespace): ++ super().__init__(args) ++ try: ++ tmp = args.path_prop.rsplit('.', 1) ++ except ValueError as err: ++ raise ValueError('Invalid format for .') from err ++ self.path = tmp[0] ++ self.prop = tmp[1] ++ ++ def run(self) -> int: ++ rsp = self.qmp.command( ++ 'qom-get', ++ path=self.path, ++ property=self.prop ++ ) ++ if isinstance(rsp, dict): ++ for key, value in rsp.items(): ++ print(f"{key}: {value}") ++ else: ++ print(rsp) ++ return 0 ++ ++ ++class QOMList(QOMCommand): ++ """ ++ QOM Command - List the properties at a given path. ++ ++ usage: qom-list [-h] [--socket SOCKET] ++ ++ List QOM properties at a given path ++ ++ positional arguments: ++ QOM path ++ ++ optional arguments: ++ -h, --help show this help message and exit ++ --socket SOCKET, -s SOCKET ++ QMP socket path or address (addr:port). May also be ++ set via QMP_SOCKET environment variable. ++ """ ++ name = 'list' ++ help = 'List QOM properties at a given path' ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ super().configure_parser(parser) ++ parser.add_argument( ++ 'path', ++ metavar='', ++ action='store', ++ help='QOM path', ++ ) ++ ++ def __init__(self, args: argparse.Namespace): ++ super().__init__(args) ++ self.path = args.path ++ ++ def run(self) -> int: ++ rsp = self.qom_list(self.path) ++ for item in rsp: ++ if item.child: ++ print(f"{item.name}/") ++ elif item.link: ++ print(f"@{item.name}/") ++ else: ++ print(item.name) ++ return 0 ++ ++ ++class QOMTree(QOMCommand): ++ """ ++ QOM Command - Show the full tree below a given path. ++ ++ usage: qom-tree [-h] [--socket SOCKET] [] ++ ++ Show QOM tree from a given path ++ ++ positional arguments: ++ QOM path ++ ++ optional arguments: ++ -h, --help show this help message and exit ++ --socket SOCKET, -s SOCKET ++ QMP socket path or address (addr:port). May also be ++ set via QMP_SOCKET environment variable. ++ """ ++ name = 'tree' ++ help = 'Show QOM tree from a given path' ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ super().configure_parser(parser) ++ parser.add_argument( ++ 'path', ++ metavar='', ++ action='store', ++ help='QOM path', ++ nargs='?', ++ default='/' ++ ) ++ ++ def __init__(self, args: argparse.Namespace): ++ super().__init__(args) ++ self.path = args.path ++ ++ def _list_node(self, path: str) -> None: ++ print(path) ++ items = self.qom_list(path) ++ for item in items: ++ if item.child: ++ continue ++ try: ++ rsp = self.qmp.command('qom-get', path=path, ++ property=item.name) ++ print(f" {item.name}: {rsp} ({item.type})") ++ except QMPResponseError as err: ++ print(f" {item.name}: ({item.type})") ++ print('') ++ for item in items: ++ if not item.child: ++ continue ++ if path == '/': ++ path = '' ++ self._list_node(f"{path}/{item.name}") ++ ++ def run(self) -> int: ++ self._list_node(self.path) ++ return 0 ++ ++ ++def main() -> int: ++ """QOM script main entry point.""" ++ parser = argparse.ArgumentParser( ++ description='Query and manipulate QOM data' ++ ) ++ subparsers = parser.add_subparsers( ++ title='QOM commands', ++ dest='command' ++ ) ++ ++ for command in QOMCommand.__subclasses__(): ++ command.register(subparsers) ++ ++ args = parser.parse_args() ++ ++ if args.command is None: ++ parser.error('Command not specified.') ++ return 1 ++ ++ cmd_class = args.cmd_class ++ assert isinstance(cmd_class, type(QOMCommand)) ++ return cmd_class.command_runner(args) +diff --git a/python/qemu/utils/qom_common.py b/python/qemu/utils/qom_common.py +new file mode 100644 +index 0000000000000000000000000000000000000000..a59ae1a2a1883cb4d89b0e44507c5001f44357a0 +--- /dev/null ++++ b/python/qemu/utils/qom_common.py +@@ -0,0 +1,178 @@ ++""" ++QOM Command abstractions. ++""" ++## ++# Copyright John Snow 2020, for Red Hat, Inc. ++# Copyright IBM, Corp. 2011 ++# ++# Authors: ++# John Snow ++# Anthony Liguori ++# ++# This work is licensed under the terms of the GNU GPL, version 2 or later. ++# See the COPYING file in the top-level directory. ++# ++# Based on ./scripts/qmp/qom-[set|get|tree|list] ++## ++ ++import argparse ++import os ++import sys ++from typing import ( ++ Any, ++ Dict, ++ List, ++ Optional, ++ Type, ++ TypeVar, ++) ++ ++from . import QEMUMonitorProtocol, QMPError ++ ++ ++# The following is needed only for a type alias. ++Subparsers = argparse._SubParsersAction # pylint: disable=protected-access ++ ++ ++class ObjectPropertyInfo: ++ """ ++ Represents the return type from e.g. qom-list. ++ """ ++ def __init__(self, name: str, type_: str, ++ description: Optional[str] = None, ++ default_value: Optional[object] = None): ++ self.name = name ++ self.type = type_ ++ self.description = description ++ self.default_value = default_value ++ ++ @classmethod ++ def make(cls, value: Dict[str, Any]) -> 'ObjectPropertyInfo': ++ """ ++ Build an ObjectPropertyInfo from a Dict with an unknown shape. ++ """ ++ assert value.keys() >= {'name', 'type'} ++ assert value.keys() <= {'name', 'type', 'description', 'default-value'} ++ return cls(value['name'], value['type'], ++ value.get('description'), ++ value.get('default-value')) ++ ++ @property ++ def child(self) -> bool: ++ """Is this property a child property?""" ++ return self.type.startswith('child<') ++ ++ @property ++ def link(self) -> bool: ++ """Is this property a link property?""" ++ return self.type.startswith('link<') ++ ++ ++CommandT = TypeVar('CommandT', bound='QOMCommand') ++ ++ ++class QOMCommand: ++ """ ++ Represents a QOM sub-command. ++ ++ :param args: Parsed arguments, as returned from parser.parse_args. ++ """ ++ name: str ++ help: str ++ ++ def __init__(self, args: argparse.Namespace): ++ if args.socket is None: ++ raise QMPError("No QMP socket path or address given") ++ self.qmp = QEMUMonitorProtocol( ++ QEMUMonitorProtocol.parse_address(args.socket) ++ ) ++ self.qmp.connect() ++ ++ @classmethod ++ def register(cls, subparsers: Subparsers) -> None: ++ """ ++ Register this command with the argument parser. ++ ++ :param subparsers: argparse subparsers object, from "add_subparsers". ++ """ ++ subparser = subparsers.add_parser(cls.name, help=cls.help, ++ description=cls.help) ++ cls.configure_parser(subparser) ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ """ ++ Configure a parser with this command's arguments. ++ ++ :param parser: argparse parser or subparser object. ++ """ ++ default_path = os.environ.get('QMP_SOCKET') ++ parser.add_argument( ++ '--socket', '-s', ++ dest='socket', ++ action='store', ++ help='QMP socket path or address (addr:port).' ++ ' May also be set via QMP_SOCKET environment variable.', ++ default=default_path ++ ) ++ parser.set_defaults(cmd_class=cls) ++ ++ @classmethod ++ def add_path_prop_arg(cls, parser: argparse.ArgumentParser) -> None: ++ """ ++ Add the . positional argument to this command. ++ ++ :param parser: The parser to add the argument to. ++ """ ++ parser.add_argument( ++ 'path_prop', ++ metavar='.', ++ action='store', ++ help="QOM path and property, separated by a period '.'" ++ ) ++ ++ def run(self) -> int: ++ """ ++ Run this command. ++ ++ :return: 0 on success, 1 otherwise. ++ """ ++ raise NotImplementedError ++ ++ def qom_list(self, path: str) -> List[ObjectPropertyInfo]: ++ """ ++ :return: a strongly typed list from the 'qom-list' command. ++ """ ++ rsp = self.qmp.command('qom-list', path=path) ++ # qom-list returns List[ObjectPropertyInfo] ++ assert isinstance(rsp, list) ++ return [ObjectPropertyInfo.make(x) for x in rsp] ++ ++ @classmethod ++ def command_runner( ++ cls: Type[CommandT], ++ args: argparse.Namespace ++ ) -> int: ++ """ ++ Run a fully-parsed subcommand, with error-handling for the CLI. ++ ++ :return: The return code from `run()`. ++ """ ++ try: ++ cmd = cls(args) ++ return cmd.run() ++ except QMPError as err: ++ print(f"{type(err).__name__}: {err!s}", file=sys.stderr) ++ return -1 ++ ++ @classmethod ++ def entry_point(cls) -> int: ++ """ ++ Build this command's parser, parse arguments, and run the command. ++ ++ :return: `run`'s return code. ++ """ ++ parser = argparse.ArgumentParser(description=cls.help) ++ cls.configure_parser(parser) ++ args = parser.parse_args() ++ return cls.command_runner(args) +diff --git a/python/qemu/utils/qom_fuse.py b/python/qemu/utils/qom_fuse.py +new file mode 100644 +index 0000000000000000000000000000000000000000..43f4671fdb18c6aa1b11df9694855167fced8f10 +--- /dev/null ++++ b/python/qemu/utils/qom_fuse.py +@@ -0,0 +1,206 @@ ++""" ++QEMU Object Model FUSE filesystem tool ++ ++This script offers a simple FUSE filesystem within which the QOM tree ++may be browsed, queried and edited using traditional shell tooling. ++ ++This script requires the 'fusepy' python package. ++ ++ ++usage: qom-fuse [-h] [--socket SOCKET] ++ ++Mount a QOM tree as a FUSE filesystem ++ ++positional arguments: ++ Mount point ++ ++optional arguments: ++ -h, --help show this help message and exit ++ --socket SOCKET, -s SOCKET ++ QMP socket path or address (addr:port). May also be ++ set via QMP_SOCKET environment variable. ++""" ++## ++# Copyright IBM, Corp. 2012 ++# Copyright (C) 2020 Red Hat, Inc. ++# ++# Authors: ++# Anthony Liguori ++# Markus Armbruster ++# ++# This work is licensed under the terms of the GNU GPL, version 2 or later. ++# See the COPYING file in the top-level directory. ++## ++ ++import argparse ++from errno import ENOENT, EPERM ++import stat ++import sys ++from typing import ( ++ IO, ++ Dict, ++ Iterator, ++ Mapping, ++ Optional, ++ Union, ++) ++ ++import fuse ++from fuse import FUSE, FuseOSError, Operations ++ ++from . import QMPResponseError ++from .qom_common import QOMCommand ++ ++ ++fuse.fuse_python_api = (0, 2) ++ ++ ++class QOMFuse(QOMCommand, Operations): ++ """ ++ QOMFuse implements both fuse.Operations and QOMCommand. ++ ++ Operations implements the FS, and QOMCommand implements the CLI command. ++ """ ++ name = 'fuse' ++ help = 'Mount a QOM tree as a FUSE filesystem' ++ fuse: FUSE ++ ++ @classmethod ++ def configure_parser(cls, parser: argparse.ArgumentParser) -> None: ++ super().configure_parser(parser) ++ parser.add_argument( ++ 'mount', ++ metavar='', ++ action='store', ++ help="Mount point", ++ ) ++ ++ def __init__(self, args: argparse.Namespace): ++ super().__init__(args) ++ self.mount = args.mount ++ self.ino_map: Dict[str, int] = {} ++ self.ino_count = 1 ++ ++ def run(self) -> int: ++ print(f"Mounting QOMFS to '{self.mount}'", file=sys.stderr) ++ self.fuse = FUSE(self, self.mount, foreground=True) ++ return 0 ++ ++ def get_ino(self, path: str) -> int: ++ """Get an inode number for a given QOM path.""" ++ if path in self.ino_map: ++ return self.ino_map[path] ++ self.ino_map[path] = self.ino_count ++ self.ino_count += 1 ++ return self.ino_map[path] ++ ++ def is_object(self, path: str) -> bool: ++ """Is the given QOM path an object?""" ++ try: ++ self.qom_list(path) ++ return True ++ except QMPResponseError: ++ return False ++ ++ def is_property(self, path: str) -> bool: ++ """Is the given QOM path a property?""" ++ path, prop = path.rsplit('/', 1) ++ if path == '': ++ path = '/' ++ try: ++ for item in self.qom_list(path): ++ if item.name == prop: ++ return True ++ return False ++ except QMPResponseError: ++ return False ++ ++ def is_link(self, path: str) -> bool: ++ """Is the given QOM path a link?""" ++ path, prop = path.rsplit('/', 1) ++ if path == '': ++ path = '/' ++ try: ++ for item in self.qom_list(path): ++ if item.name == prop and item.link: ++ return True ++ return False ++ except QMPResponseError: ++ return False ++ ++ def read(self, path: str, size: int, offset: int, fh: IO[bytes]) -> bytes: ++ if not self.is_property(path): ++ raise FuseOSError(ENOENT) ++ ++ path, prop = path.rsplit('/', 1) ++ if path == '': ++ path = '/' ++ try: ++ data = str(self.qmp.command('qom-get', path=path, property=prop)) ++ data += '\n' # make values shell friendly ++ except QMPResponseError as err: ++ raise FuseOSError(EPERM) from err ++ ++ if offset > len(data): ++ return b'' ++ ++ return bytes(data[offset:][:size], encoding='utf-8') ++ ++ def readlink(self, path: str) -> Union[bool, str]: ++ if not self.is_link(path): ++ return False ++ path, prop = path.rsplit('/', 1) ++ prefix = '/'.join(['..'] * (len(path.split('/')) - 1)) ++ return prefix + str(self.qmp.command('qom-get', path=path, ++ property=prop)) ++ ++ def getattr(self, path: str, ++ fh: Optional[IO[bytes]] = None) -> Mapping[str, object]: ++ if self.is_link(path): ++ value = { ++ 'st_mode': 0o755 | stat.S_IFLNK, ++ 'st_ino': self.get_ino(path), ++ 'st_dev': 0, ++ 'st_nlink': 2, ++ 'st_uid': 1000, ++ 'st_gid': 1000, ++ 'st_size': 4096, ++ 'st_atime': 0, ++ 'st_mtime': 0, ++ 'st_ctime': 0 ++ } ++ elif self.is_object(path): ++ value = { ++ 'st_mode': 0o755 | stat.S_IFDIR, ++ 'st_ino': self.get_ino(path), ++ 'st_dev': 0, ++ 'st_nlink': 2, ++ 'st_uid': 1000, ++ 'st_gid': 1000, ++ 'st_size': 4096, ++ 'st_atime': 0, ++ 'st_mtime': 0, ++ 'st_ctime': 0 ++ } ++ elif self.is_property(path): ++ value = { ++ 'st_mode': 0o644 | stat.S_IFREG, ++ 'st_ino': self.get_ino(path), ++ 'st_dev': 0, ++ 'st_nlink': 1, ++ 'st_uid': 1000, ++ 'st_gid': 1000, ++ 'st_size': 4096, ++ 'st_atime': 0, ++ 'st_mtime': 0, ++ 'st_ctime': 0 ++ } ++ else: ++ raise FuseOSError(ENOENT) ++ return value ++ ++ def readdir(self, path: str, fh: IO[bytes]) -> Iterator[str]: ++ yield '.' ++ yield '..' ++ for item in self.qom_list(path): ++ yield item.name +diff --git a/python/setup.cfg b/python/setup.cfg +index 4f4f20571f304507e20ce16cee66..91ccef7e8fd85d0d6d3d86adbc8d 100644 +--- a/python/setup.cfg ++++ b/python/setup.cfg +@@ -60,13 +60,13 @@ tui = + + [options.entry_points] + console_scripts = +- qom = qemu.qmp.qom:main +- qom-set = qemu.qmp.qom:QOMSet.entry_point +- qom-get = qemu.qmp.qom:QOMGet.entry_point +- qom-list = qemu.qmp.qom:QOMList.entry_point +- qom-tree = qemu.qmp.qom:QOMTree.entry_point +- qom-fuse = qemu.qmp.qom_fuse:QOMFuse.entry_point [fuse] +- qemu-ga-client = qemu.qmp.qemu_ga_client:main ++ qom = qemu.utils.qom:main ++ qom-set = qemu.utils.qom:QOMSet.entry_point ++ qom-get = qemu.utils.qom:QOMGet.entry_point ++ qom-list = qemu.utils.qom:QOMList.entry_point ++ qom-tree = qemu.utils.qom:QOMTree.entry_point ++ qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse] ++ qemu-ga-client = qemu.utils.qemu_ga_client:main + qmp-shell = qemu.qmp.qmp_shell:main + aqmp-tui = qemu.aqmp.aqmp_tui:main [tui] + +@@ -80,7 +80,7 @@ python_version = 3.6 + warn_unused_configs = True + namespace_packages = True + +-[mypy-qemu.qmp.qom_fuse] ++[mypy-qemu.utils.qom_fuse] + # fusepy has no type stubs: + allow_subclassing_any = True + +diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client +index 102fd2cad93fcc76e428e841241a..56edd0234a62cc25a069bb6a65dc 100755 +--- a/scripts/qmp/qemu-ga-client ++++ b/scripts/qmp/qemu-ga-client +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp import qemu_ga_client ++from qemu.utils import qemu_ga_client + + + if __name__ == '__main__': +diff --git a/scripts/qmp/qom-fuse b/scripts/qmp/qom-fuse +index a58c8ef9793b387eeaec6777a5c3..d453807b273f5576c90c8809df2b 100755 +--- a/scripts/qmp/qom-fuse ++++ b/scripts/qmp/qom-fuse +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp.qom_fuse import QOMFuse ++from qemu.utils.qom_fuse import QOMFuse + + + if __name__ == '__main__': +diff --git a/scripts/qmp/qom-get b/scripts/qmp/qom-get +index e4f3e0c01381a0240379016340d4..04ebe052e82a97bbadde1dfc76c4 100755 +--- a/scripts/qmp/qom-get ++++ b/scripts/qmp/qom-get +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp.qom import QOMGet ++from qemu.utils.qom import QOMGet + + + if __name__ == '__main__': +diff --git a/scripts/qmp/qom-list b/scripts/qmp/qom-list +index 7a071a54e1e7785142b2dc262ac4..853b85a8d3fc4e032f40747c3c08 100755 +--- a/scripts/qmp/qom-list ++++ b/scripts/qmp/qom-list +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp.qom import QOMList ++from qemu.utils.qom import QOMList + + + if __name__ == '__main__': +diff --git a/scripts/qmp/qom-set b/scripts/qmp/qom-set +index 9ca9e2ba106b158e0926184533b9..06820feec424ecd30235c6ed9006 100755 +--- a/scripts/qmp/qom-set ++++ b/scripts/qmp/qom-set +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp.qom import QOMSet ++from qemu.utils.qom import QOMSet + + + if __name__ == '__main__': +diff --git a/scripts/qmp/qom-tree b/scripts/qmp/qom-tree +index 7d0ccca3a4dd87edb92b1ac7a6e7..760e172277e9b66ff2a6d770b526 100755 +--- a/scripts/qmp/qom-tree ++++ b/scripts/qmp/qom-tree +@@ -4,7 +4,7 @@ import os + import sys + + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +-from qemu.qmp.qom import QOMTree ++from qemu.utils.qom import QOMTree + + + if __name__ == '__main__': diff --git a/python-qmp-switch-qmp-shell-to-AQMP.patch b/python-qmp-switch-qmp-shell-to-AQMP.patch new file mode 100644 index 00000000..198a3020 --- /dev/null +++ b/python-qmp-switch-qmp-shell-to-AQMP.patch @@ -0,0 +1,121 @@ +From: John Snow +Date: Mon, 10 Jan 2022 18:28:53 -0500 +Subject: python/qmp: switch qmp-shell to AQMP + +Git-commit: f3efd12930f34b9724e15d8fd2ff56a97b67219d + +We have a replacement for async QMP, but it doesn't have feature parity +yet. For now, then, port the old tool onto the new backend. + +Signed-off-by: John Snow +Reviewed-by: Vladimir Sementsov-Ogievskiy +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/legacy.py | 3 +++ + python/qemu/qmp/qmp_shell.py | 31 +++++++++++++++++-------------- + 2 files changed, 20 insertions(+), 14 deletions(-) + +diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py +index 27df22818a76190e872f08c0852e..0890f95b16875ecb815ed4560bc7 100644 +--- a/python/qemu/aqmp/legacy.py ++++ b/python/qemu/aqmp/legacy.py +@@ -22,6 +22,9 @@ from .protocol import Runstate, SocketAddrT + from .qmp_client import QMPClient + + ++# (Temporarily) Re-export QMPBadPortError ++QMPBadPortError = qemu.qmp.QMPBadPortError ++ + #: QMPMessage is an entire QMP message of any kind. + QMPMessage = Dict[str, Any] + +diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/qmp/qmp_shell.py +index e7d7eb18f19cae7ac185b333013e..d11bf54b00e5d56616ae57be0006 100644 +--- a/python/qemu/qmp/qmp_shell.py ++++ b/python/qemu/qmp/qmp_shell.py +@@ -95,8 +95,13 @@ from typing import ( + Sequence, + ) + +-from qemu import qmp +-from qemu.qmp import QMPMessage ++from qemu.aqmp import ConnectError, QMPError, SocketAddrT ++from qemu.aqmp.legacy import ( ++ QEMUMonitorProtocol, ++ QMPBadPortError, ++ QMPMessage, ++ QMPObject, ++) + + + LOG = logging.getLogger(__name__) +@@ -125,7 +130,7 @@ class QMPCompleter: + return None + + +-class QMPShellError(qmp.QMPError): ++class QMPShellError(QMPError): + """ + QMP Shell Base error class. + """ +@@ -153,7 +158,7 @@ class FuzzyJSON(ast.NodeTransformer): + return node + + +-class QMPShell(qmp.QEMUMonitorProtocol): ++class QMPShell(QEMUMonitorProtocol): + """ + QMPShell provides a basic readline-based QMP shell. + +@@ -161,7 +166,7 @@ class QMPShell(qmp.QEMUMonitorProtocol): + :param pretty: Pretty-print QMP messages. + :param verbose: Echo outgoing QMP messages to console. + """ +- def __init__(self, address: qmp.SocketAddrT, ++ def __init__(self, address: SocketAddrT, + pretty: bool = False, verbose: bool = False): + super().__init__(address) + self._greeting: Optional[QMPMessage] = None +@@ -237,7 +242,7 @@ class QMPShell(qmp.QEMUMonitorProtocol): + + def _cli_expr(self, + tokens: Sequence[str], +- parent: qmp.QMPObject) -> None: ++ parent: QMPObject) -> None: + for arg in tokens: + (key, sep, val) = arg.partition('=') + if sep != '=': +@@ -403,7 +408,7 @@ class HMPShell(QMPShell): + :param pretty: Pretty-print QMP messages. + :param verbose: Echo outgoing QMP messages to console. + """ +- def __init__(self, address: qmp.SocketAddrT, ++ def __init__(self, address: SocketAddrT, + pretty: bool = False, verbose: bool = False): + super().__init__(address, pretty, verbose) + self._cpu_index = 0 +@@ -512,19 +517,17 @@ def main() -> None: + + try: + address = shell_class.parse_address(args.qmp_server) +- except qmp.QMPBadPortError: ++ except QMPBadPortError: + parser.error(f"Bad port number: {args.qmp_server}") + return # pycharm doesn't know error() is noreturn + + with shell_class(address, args.pretty, args.verbose) as qemu: + try: + qemu.connect(negotiate=not args.skip_negotiation) +- except qmp.QMPConnectError: +- die("Didn't get QMP greeting message") +- except qmp.QMPCapabilitiesError: +- die("Couldn't negotiate capabilities") +- except OSError as err: +- die(f"Couldn't connect to {args.qmp_server}: {err!s}") ++ except ConnectError as err: ++ if isinstance(err.exc, OSError): ++ die(f"Couldn't connect to {args.qmp_server}: {err!s}") ++ die(str(err)) + + for _ in qemu.repl(): + pass diff --git a/python-support-recording-QMP-session-to-.patch b/python-support-recording-QMP-session-to-.patch new file mode 100644 index 00000000..58088a36 --- /dev/null +++ b/python-support-recording-QMP-session-to-.patch @@ -0,0 +1,184 @@ +From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= +Date: Fri, 28 Jan 2022 16:11:57 +0000 +Subject: python: support recording QMP session to a file +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Git-commit: 5c66d7d8de9a00460199669d26cd83fba135bee5 + +When running QMP commands with very large response payloads, it is often +not easy to spot the info you want. If we can save the response to a +file then tools like 'grep' or 'jq' can be used to extract information. + +For convenience of processing, we merge the QMP command and response +dictionaries together: + + { + "arguments": {}, + "execute": "query-kvm", + "return": { + "enabled": false, + "present": true + } + } + +Example usage + + $ ./scripts/qmp/qmp-shell-wrap -l q.log -p -- ./build/qemu-system-x86_64 -display none + Welcome to the QMP low-level shell! + Connected + (QEMU) query-kvm + { + "return": { + "enabled": false, + "present": true + } + } + (QEMU) query-mice + { + "return": [ + { + "absolute": false, + "current": true, + "index": 2, + "name": "QEMU PS/2 Mouse" + } + ] + } + + $ jq --slurp '. | to_entries[] | select(.value.execute == "query-kvm") | + .value.return.enabled' < q.log + false + +Reviewed-by: Philippe Mathieu-Daudé +Signed-off-by: Daniel P. Berrangé +Message-id: 20220128161157.36261-3-berrange@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/qemu/aqmp/qmp_shell.py | 29 ++++++++++++++++++++++------- + python/setup.cfg | 3 +++ + 2 files changed, 25 insertions(+), 7 deletions(-) + +diff --git a/python/qemu/aqmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py +index c60df787fcd50bf8a0109e5f5cd3..35691494d0a88070bcf1ad691699 100644 +--- a/python/qemu/aqmp/qmp_shell.py ++++ b/python/qemu/aqmp/qmp_shell.py +@@ -89,6 +89,7 @@ import readline + from subprocess import Popen + import sys + from typing import ( ++ IO, + Iterator, + List, + NoReturn, +@@ -170,7 +171,8 @@ class QMPShell(QEMUMonitorProtocol): + def __init__(self, address: SocketAddrT, + pretty: bool = False, + verbose: bool = False, +- server: bool = False): ++ server: bool = False, ++ logfile: Optional[str] = None): + super().__init__(address, server=server) + self._greeting: Optional[QMPMessage] = None + self._completer = QMPCompleter() +@@ -180,6 +182,10 @@ class QMPShell(QEMUMonitorProtocol): + '.qmp-shell_history') + self.pretty = pretty + self.verbose = verbose ++ self.logfile = None ++ ++ if logfile is not None: ++ self.logfile = open(logfile, "w", encoding='utf-8') + + def close(self) -> None: + # Hook into context manager of parent to save shell history. +@@ -320,11 +326,11 @@ class QMPShell(QEMUMonitorProtocol): + self._cli_expr(cmdargs[1:], qmpcmd['arguments']) + return qmpcmd + +- def _print(self, qmp_message: object) -> None: ++ def _print(self, qmp_message: object, fh: IO[str] = sys.stdout) -> None: + jsobj = json.dumps(qmp_message, + indent=4 if self.pretty else None, + sort_keys=self.pretty) +- print(str(jsobj)) ++ print(str(jsobj), file=fh) + + def _execute_cmd(self, cmdline: str) -> bool: + try: +@@ -347,6 +353,9 @@ class QMPShell(QEMUMonitorProtocol): + print('Disconnected') + return False + self._print(resp) ++ if self.logfile is not None: ++ cmd = {**qmpcmd, **resp} ++ self._print(cmd, fh=self.logfile) + return True + + def connect(self, negotiate: bool = True) -> None: +@@ -414,8 +423,9 @@ class HMPShell(QMPShell): + def __init__(self, address: SocketAddrT, + pretty: bool = False, + verbose: bool = False, +- server: bool = False): +- super().__init__(address, pretty, verbose, server) ++ server: bool = False, ++ logfile: Optional[str] = None): ++ super().__init__(address, pretty, verbose, server, logfile) + self._cpu_index = 0 + + def _cmd_completion(self) -> None: +@@ -508,6 +518,8 @@ def main() -> None: + help='Verbose (echo commands sent and received)') + parser.add_argument('-p', '--pretty', action='store_true', + help='Pretty-print JSON') ++ parser.add_argument('-l', '--logfile', ++ help='Save log of all QMP messages to PATH') + + default_server = os.environ.get('QMP_SOCKET') + parser.add_argument('qmp_server', action='store', +@@ -526,7 +538,7 @@ def main() -> None: + parser.error(f"Bad port number: {args.qmp_server}") + return # pycharm doesn't know error() is noreturn + +- with shell_class(address, args.pretty, args.verbose) as qemu: ++ with shell_class(address, args.pretty, args.verbose, args.logfile) as qemu: + try: + qemu.connect(negotiate=not args.skip_negotiation) + except ConnectError as err: +@@ -550,6 +562,8 @@ def main_wrap() -> None: + help='Verbose (echo commands sent and received)') + parser.add_argument('-p', '--pretty', action='store_true', + help='Pretty-print JSON') ++ parser.add_argument('-l', '--logfile', ++ help='Save log of all QMP messages to PATH') + + parser.add_argument('command', nargs=argparse.REMAINDER, + help='QEMU command line to invoke') +@@ -574,7 +588,8 @@ def main_wrap() -> None: + return # pycharm doesn't know error() is noreturn + + try: +- with shell_class(address, args.pretty, args.verbose, True) as qemu: ++ with shell_class(address, args.pretty, args.verbose, ++ True, args.logfile) as qemu: + with Popen(cmd): + + try: +diff --git a/python/setup.cfg b/python/setup.cfg +index bec54e8b0d663191e2b7afbfa350..241f243e8b94417f9b032c41576b 100644 +--- a/python/setup.cfg ++++ b/python/setup.cfg +@@ -114,7 +114,10 @@ ignore_missing_imports = True + # no Warning level messages displayed, use "--disable=all --enable=classes + # --disable=W". + disable=consider-using-f-string, ++ consider-using-with, ++ too-many-arguments, + too-many-function-args, # mypy handles this with less false positives. ++ too-many-instance-attributes, + no-member, # mypy also handles this better. + + [pylint.basic] diff --git a/python-upgrade-mypy-to-0.780.patch b/python-upgrade-mypy-to-0.780.patch new file mode 100644 index 00000000..f7b9618e --- /dev/null +++ b/python-upgrade-mypy-to-0.780.patch @@ -0,0 +1,232 @@ +From: John Snow +Date: Mon, 31 Jan 2022 23:11:33 -0500 +Subject: python: upgrade mypy to 0.780 + +Git-commit: 74a1505d279897d2a448c876820a33cbe1f0f22e + +We need a slightly newer version of mypy in order to use some features +of the asyncio server functions in the next commit. + +(Note: pipenv is not really suited to upgrading individual packages; I +need to replace this tool with something better for the task. For now, +the miscellaneous updates not related to the mypy upgrade are simply +beyond my control. It's on my list to take care of soon.) + +Signed-off-by: John Snow +Reviewed-by: Kevin Wolf +Message-id: 20220201041134.1237016-4-jsnow@redhat.com +Signed-off-by: John Snow +Signed-off-by: Li Zhang +--- + python/Pipfile.lock | 66 ++++++++++++++++++++++++++------------------- + python/setup.cfg | 2 +- + 2 files changed, 40 insertions(+), 28 deletions(-) + +diff --git a/python/Pipfile.lock b/python/Pipfile.lock +index d2a7dbd88be19fd6db0baa083d8a..ce46404ce0840c693d3c982674ac 100644 +--- a/python/Pipfile.lock ++++ b/python/Pipfile.lock +@@ -1,7 +1,7 @@ + { + "_meta": { + "hash": { +- "sha256": "784b327272db32403d5a488507853b5afba850ba26a5948e5b6a90c1baef2d9c" ++ "sha256": "f1a25654d884a5b450e38d78b1f2e3ebb9073e421cc4358d4bbb83ac251a5670" + }, + "pipfile-spec": 6, + "requires": { +@@ -34,7 +34,7 @@ + "sha256:09bdb456e02564731f8b5957cdd0c98a7f01d2db5e90eb1d794c353c28bfd705", + "sha256:6a8a51f64dae307f6e0c9db752b66a7951e282389d8362cc1d39a56f3feeb31d" + ], +- "markers": "python_version ~= '3.6'", ++ "index": "pypi", + "version": "==2.6.0" + }, + "avocado-framework": { +@@ -50,6 +50,7 @@ + "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736", + "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c" + ], ++ "index": "pypi", + "version": "==0.3.2" + }, + "filelock": { +@@ -57,6 +58,7 @@ + "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", + "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" + ], ++ "index": "pypi", + "version": "==3.0.12" + }, + "flake8": { +@@ -88,7 +90,7 @@ + "sha256:54161657e8ffc76596c4ede7080ca68cb02962a2e074a2586b695a93a925d36e", + "sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351" + ], +- "markers": "python_version < '3.7'", ++ "index": "pypi", + "version": "==5.1.4" + }, + "isort": { +@@ -124,7 +126,7 @@ + "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93", + "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b" + ], +- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", ++ "index": "pypi", + "version": "==1.6.0" + }, + "mccabe": { +@@ -136,23 +138,23 @@ + }, + "mypy": { + "hashes": [ +- "sha256:15b948e1302682e3682f11f50208b726a246ab4e6c1b39f9264a8796bb416aa2", +- "sha256:219a3116ecd015f8dca7b5d2c366c973509dfb9a8fc97ef044a36e3da66144a1", +- "sha256:3b1fc683fb204c6b4403a1ef23f0b1fac8e4477091585e0c8c54cbdf7d7bb164", +- "sha256:3beff56b453b6ef94ecb2996bea101a08f1f8a9771d3cbf4988a61e4d9973761", +- "sha256:7687f6455ec3ed7649d1ae574136835a4272b65b3ddcf01ab8704ac65616c5ce", +- "sha256:7ec45a70d40ede1ec7ad7f95b3c94c9cf4c186a32f6bacb1795b60abd2f9ef27", +- "sha256:86c857510a9b7c3104cf4cde1568f4921762c8f9842e987bc03ed4f160925754", +- "sha256:8a627507ef9b307b46a1fea9513d5c98680ba09591253082b4c48697ba05a4ae", +- "sha256:8dfb69fbf9f3aeed18afffb15e319ca7f8da9642336348ddd6cab2713ddcf8f9", +- "sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600", +- "sha256:a8ffcd53cb5dfc131850851cc09f1c44689c2812d0beb954d8138d4f5fc17f65", +- "sha256:b90928f2d9eb2f33162405f32dde9f6dcead63a0971ca8a1b50eb4ca3e35ceb8", +- "sha256:c56ffe22faa2e51054c5f7a3bc70a370939c2ed4de308c690e7949230c995913", +- "sha256:f91c7ae919bbc3f96cd5e5b2e786b2b108343d1d7972ea130f7de27fdd547cf3" ++ "sha256:00cb1964a7476e871d6108341ac9c1a857d6bd20bf5877f4773ac5e9d92cd3cd", ++ "sha256:127de5a9b817a03a98c5ae8a0c46a20dc44442af6dcfa2ae7f96cb519b312efa", ++ "sha256:1f3976a945ad7f0a0727aafdc5651c2d3278e3c88dee94e2bf75cd3386b7b2f4", ++ "sha256:2f8c098f12b402c19b735aec724cc9105cc1a9eea405d08814eb4b14a6fb1a41", ++ "sha256:4ef13b619a289aa025f2273e05e755f8049bb4eaba6d703a425de37d495d178d", ++ "sha256:5d142f219bf8c7894dfa79ebfb7d352c4c63a325e75f10dfb4c3db9417dcd135", ++ "sha256:62eb5dd4ea86bda8ce386f26684f7f26e4bfe6283c9f2b6ca6d17faf704dcfad", ++ "sha256:64c36eb0936d0bfb7d8da49f92c18e312ad2e3ed46e5548ae4ca997b0d33bd59", ++ "sha256:75eed74d2faf2759f79c5f56f17388defd2fc994222312ec54ee921e37b31ad4", ++ "sha256:974bebe3699b9b46278a7f076635d219183da26e1a675c1f8243a69221758273", ++ "sha256:a5e5bb12b7982b179af513dddb06fca12285f0316d74f3964078acbfcf4c68f2", ++ "sha256:d31291df31bafb997952dc0a17ebb2737f802c754aed31dd155a8bfe75112c57", ++ "sha256:d3b4941de44341227ece1caaf5b08b23e42ad4eeb8b603219afb11e9d4cfb437", ++ "sha256:eadb865126da4e3c4c95bdb47fe1bb087a3e3ea14d39a3b13224b8a4d9f9a102" + ], + "index": "pypi", +- "version": "==0.770" ++ "version": "==0.780" + }, + "mypy-extensions": { + "hashes": [ +@@ -166,7 +168,7 @@ + "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", + "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + ], +- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", ++ "index": "pypi", + "version": "==20.9" + }, + "pluggy": { +@@ -174,7 +176,7 @@ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], +- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", ++ "index": "pypi", + "version": "==0.13.1" + }, + "py": { +@@ -182,7 +184,7 @@ + "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", + "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" + ], +- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", ++ "index": "pypi", + "version": "==1.10.0" + }, + "pycodestyle": { +@@ -205,7 +207,7 @@ + "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f", + "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e" + ], +- "markers": "python_version >= '3.5'", ++ "index": "pypi", + "version": "==2.9.0" + }, + "pylint": { +@@ -221,13 +223,21 @@ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], +- "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", ++ "index": "pypi", + "version": "==2.4.7" + }, + "qemu": { + "editable": true, + "path": "." + }, ++ "setuptools": { ++ "hashes": [ ++ "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373", ++ "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e" ++ ], ++ "markers": "python_version >= '3.6'", ++ "version": "==59.6.0" ++ }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", +@@ -294,19 +304,21 @@ + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" + ], +- "markers": "python_version < '3.8'", ++ "index": "pypi", + "version": "==3.10.0.0" + }, + "urwid": { + "hashes": [ + "sha256:588bee9c1cb208d0906a9f73c613d2bd32c3ed3702012f51efe318a3f2127eae" + ], ++ "index": "pypi", + "version": "==2.1.2" + }, + "urwid-readline": { + "hashes": [ + "sha256:018020cbc864bb5ed87be17dc26b069eae2755cb29f3a9c569aac3bded1efaf4" + ], ++ "index": "pypi", + "version": "==0.13" + }, + "virtualenv": { +@@ -314,7 +326,7 @@ + "sha256:14fdf849f80dbb29a4eb6caa9875d476ee2a5cf76a5f5415fa2f1606010ab467", + "sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76" + ], +- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", ++ "index": "pypi", + "version": "==20.4.7" + }, + "wrapt": { +@@ -328,7 +340,7 @@ + "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", + "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" + ], +- "markers": "python_version < '3.10'", ++ "index": "pypi", + "version": "==3.4.1" + } + } +diff --git a/python/setup.cfg b/python/setup.cfg +index 417e937839b85eecd752b29ad7df..4f4f20571f304507e20ce16cee66 100644 +--- a/python/setup.cfg ++++ b/python/setup.cfg +@@ -41,7 +41,7 @@ devel = + flake8 >= 3.6.0 + fusepy >= 2.0.4 + isort >= 5.1.2 +- mypy >= 0.770 ++ mypy >= 0.780 + pylint >= 2.8.0 + tox >= 3.18.0 + urwid >= 2.1.2 diff --git a/qemu.changes b/qemu.changes index e03c8a17..efce0794 100644 --- a/qemu.changes +++ b/qemu.changes @@ -1,3 +1,47 @@ +------------------------------------------------------------------- +Tue Apr 5 08:54:51 UTC 2022 - Li Zhang + +- Backport aqmp patches from upstream which can fix iotest issues +* Patches added: + python-aqmp-add-__del__-method-to-legacy.patch + python-aqmp-add-_session_guard.patch + python-aqmp-add-SocketAddrT-to-package-r.patch + python-aqmp-add-socket-bind-step-to-lega.patch + python-aqmp-add-start_server-and-accept-.patch + python-aqmp-copy-type-definitions-from-q.patch + python-aqmp-drop-_bind_hack.patch + python-aqmp-fix-docstring-typo.patch + python-aqmp-Fix-negotiation-with-pre-oob.patch + python-aqmp-fix-race-condition-in-legacy.patch + Python-aqmp-fix-type-definitions-for-myp.patch + python-aqmp-handle-asyncio.TimeoutError-.patch + python-aqmp-refactor-_do_accept-into-two.patch + python-aqmp-remove-_new_session-and-_est.patch + python-aqmp-rename-accept-to-start_serve.patch + python-aqmp-rename-AQMPError-to-QMPError.patch + python-aqmp-split-_client_connected_cb-o.patch + python-aqmp-squelch-pylint-warning-for-t.patch + python-aqmp-stop-the-server-during-disco.patch + python-introduce-qmp-shell-wrap-convenie.patch + python-machine-raise-VMLaunchFailure-exc.patch + python-move-qmp-shell-under-the-AQMP-pac.patch + python-move-qmp-utilities-to-python-qemu.patch + python-qmp-switch-qmp-shell-to-AQMP.patch + python-support-recording-QMP-session-to-.patch + python-upgrade-mypy-to-0.780.patch + +------------------------------------------------------------------- +Tue Apr 5 08:24:58 UTC 2022 - Li Zhang + +- Drop the patches which are workaround to fix iotest issues +* Patches dropped: + Revert-python-iotests-replace-qmp-with-a.patch + Revert-python-machine-add-instance-disam.patch + Revert-python-machine-add-sock_dir-prope.patch + Revert-python-machine-handle-fast-QEMU-t.patch + Revert-python-machine-move-more-variable.patch + Revert-python-machine-remove-_remove_mon.patch + ------------------------------------------------------------------- Thu Mar 31 10:35:44 UTC 2022 - Li Zhang diff --git a/qemu.spec b/qemu.spec index 9349ab17..a283d829 100644 --- a/qemu.spec +++ b/qemu.spec @@ -220,16 +220,36 @@ Patch00073: tests-qemu-iotests-040-Skip-TestCommitWi.patch Patch00074: tests-qemu-iotests-testrunner-Quote-case.patch Patch00075: Fix-the-module-building-problem-for-s390.patch Patch00076: scsi-generic-check-for-additional-SG_IO-.patch -Patch00077: Revert-python-machine-handle-fast-QEMU-t.patch -Patch00078: Revert-python-machine-move-more-variable.patch -Patch00079: Revert-python-machine-add-instance-disam.patch -Patch00080: Revert-python-machine-remove-_remove_mon.patch -Patch00081: Revert-python-machine-add-sock_dir-prope.patch -Patch00082: Revert-python-iotests-replace-qmp-with-a.patch -Patch00083: hw-nvme-fix-CVE-2021-3929.patch -Patch00084: numa-Enable-numa-for-SGX-EPC-sections.patch -Patch00085: numa-Support-SGX-numa-in-the-monitor-and.patch -Patch00086: doc-Add-the-SGX-numa-description.patch +Patch00077: hw-nvme-fix-CVE-2021-3929.patch +Patch00078: numa-Enable-numa-for-SGX-EPC-sections.patch +Patch00079: numa-Support-SGX-numa-in-the-monitor-and.patch +Patch00080: doc-Add-the-SGX-numa-description.patch +Patch00081: python-aqmp-Fix-negotiation-with-pre-oob.patch +Patch00082: python-machine-raise-VMLaunchFailure-exc.patch +Patch00083: python-upgrade-mypy-to-0.780.patch +Patch00084: Python-aqmp-fix-type-definitions-for-myp.patch +Patch00085: python-aqmp-add-__del__-method-to-legacy.patch +Patch00086: python-aqmp-copy-type-definitions-from-q.patch +Patch00087: python-aqmp-handle-asyncio.TimeoutError-.patch +Patch00088: python-aqmp-add-SocketAddrT-to-package-r.patch +Patch00089: python-aqmp-fix-docstring-typo.patch +Patch00090: python-aqmp-rename-AQMPError-to-QMPError.patch +Patch00091: python-qmp-switch-qmp-shell-to-AQMP.patch +Patch00092: python-aqmp-add-socket-bind-step-to-lega.patch +Patch00093: python-move-qmp-utilities-to-python-qemu.patch +Patch00094: python-move-qmp-shell-under-the-AQMP-pac.patch +Patch00095: python-introduce-qmp-shell-wrap-convenie.patch +Patch00096: python-support-recording-QMP-session-to-.patch +Patch00097: python-aqmp-add-_session_guard.patch +Patch00098: python-aqmp-rename-accept-to-start_serve.patch +Patch00099: python-aqmp-remove-_new_session-and-_est.patch +Patch00100: python-aqmp-split-_client_connected_cb-o.patch +Patch00101: python-aqmp-squelch-pylint-warning-for-t.patch +Patch00102: python-aqmp-refactor-_do_accept-into-two.patch +Patch00103: python-aqmp-stop-the-server-during-disco.patch +Patch00104: python-aqmp-add-start_server-and-accept-.patch +Patch00105: python-aqmp-fix-race-condition-in-legacy.patch +Patch00106: python-aqmp-drop-_bind_hack.patch # Patches applied in roms/seabios/: Patch01000: seabios-use-python2-explicitly-as-needed.patch Patch01001: seabios-switch-to-python3-as-needed.patch @@ -1239,6 +1259,26 @@ This package records qemu testsuite results and represents successful testing. %patch00084 -p1 %patch00085 -p1 %patch00086 -p1 +%patch00087 -p1 +%patch00088 -p1 +%patch00089 -p1 +%patch00090 -p1 +%patch00091 -p1 +%patch00092 -p1 +%patch00093 -p1 +%patch00094 -p1 +%patch00095 -p1 +%patch00096 -p1 +%patch00097 -p1 +%patch00098 -p1 +%patch00099 -p1 +%patch00100 -p1 +%patch00101 -p1 +%patch00102 -p1 +%patch00103 -p1 +%patch00104 -p1 +%patch00105 -p1 +%patch00106 -p1 %patch01000 -p1 %patch01001 -p1 %patch01002 -p1