From 05f33b59a2fefbd51303bde74368c73b2317549a Mon Sep 17 00:00:00 2001 From: Olivier Lambert Date: Tue, 7 Jul 2026 17:46:48 +0200 Subject: [PATCH] Retry the xapi connection instead of latching it broken forever Auth sets masterConnectionBroken when a XenAPI call times out, and OpenSession never attempts another connection once the flag is set. A long-lived xsconsole (e.g. the one agetty spawns on the physical console) that experiences a single 15-second xapi hang therefore never talks to xapi again: the status screen shows 'Pool master is unreachable.' forever, while newly launched xsconsole instances on the same host work fine. This is easy to hit during an HA pool master failover, where xapi on the surviving host can be unresponsive for a while, and was reported after exactly that scenario. Keep the flag as a cooldown rather than a latch: after a timeout, skip connection attempts for CONNECTION_RETRY_SECONDS (so an unresponsive xapi doesn't block the UI for the socket timeout on every update cycle), then try again, and clear the flag on the next successful login. Also log the transitions, since previously the process went quiet with no trace of why. Reproduced and verified on XCP-ng 8.3: freeze xapi (SIGSTOP) for 60s while pressing F5 in xsconsole; before this change the console never recovers, with it the connection recovers within a minute of xapi returning. --- XSConsoleAuth.py | 77 ++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/XSConsoleAuth.py b/XSConsoleAuth.py index 5c65636..b5f0ed9 100644 --- a/XSConsoleAuth.py +++ b/XSConsoleAuth.py @@ -35,6 +35,11 @@ class Auth: instance = None + # Seconds to wait before retrying to connect to xapi after a connection + # timeout, so that a long-lived xsconsole recovers once xapi is responsive + # again, e.g. after an HA pool master failover. + CONNECTION_RETRY_SECONDS = 60 + def __init__(self): self.isAuthenticated = False self.loggedInUsername = '' @@ -43,6 +48,7 @@ def __init__(self): self.testingHost = None self.authTimestampSeconds = None self.masterConnectionBroken = False + self.connectionBrokenTimestamp = None socket.setdefaulttimeout(15) self.testMode = False @@ -107,8 +113,7 @@ def TCPAuthenticate(self, inUsername, inPassword): session.logout() except socket.timeout: session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' + self.ConnectionTimedOut() finally: session.close() @@ -158,43 +163,59 @@ def LogOut(self): self.isAuthenticated = False self.loggedInUsername = None + def ConnectionTimedOut(self): + self.masterConnectionBroken = True + self.connectionBrokenTimestamp = getTimeStamp() + self.error = 'The master connection has timed out.' + XSLog('XenAPI connection timed out - retrying in ' + str(self.CONNECTION_RETRY_SECONDS) + ' seconds at most') + def OpenSession(self): session = None - if not self.masterConnectionBroken: + if self.masterConnectionBroken: + if getTimeStamp() - self.connectionBrokenTimestamp < self.CONNECTION_RETRY_SECONDS: + # Don't hammer an unresponsive xapi with blocking connection attempts + return None + # Restart the cooldown before retrying, so a failure of any kind below + # doesn't lead to a retry on every call + self.connectionBrokenTimestamp = getTimeStamp() + + try: + # Try the local Unix domain socket first + session = XenAPI.xapi_local() + if not session is None: + session.login_with_password('root', '', '', 'XSConsole') + if session._session is None: + session = None + except socket.timeout: + session = None + self.ConnectionTimedOut() + except Exception as e: + session = None + # pylint: disable-next=redefined-variable-type # ToString may handle it + self.error = e + + if session is None and self.testingHost is not None: + # Local session couldn't connect, so try remote. + session = XenAPI.Session("https://" + self.testingHost) try: - # Try the local Unix domain socket first - session = XenAPI.xapi_local() - if not session is None: - session.login_with_password('root','','','XSConsole') - if session._session is None: - session = None + session.login_with_password('root', self.defaultPassword, '', 'XSConsole') + + except XenAPI.Failure as e: + if e.details[0] != 'HOST_IS_SLAVE': # Ignore slave errors when testing + session = None + self.error = e except socket.timeout: session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' + self.ConnectionTimedOut() except Exception as e: session = None - # pylint: disable-next=redefined-variable-type # ToString may handle it self.error = e - if session is None and self.testingHost is not None: - # Local session couldn't connect, so try remote. - session = XenAPI.Session("https://"+self.testingHost) - try: - session.login_with_password('root', self.defaultPassword,'','XSConsole') + if session is not None and self.masterConnectionBroken: + XSLog('XenAPI connection recovered') + self.masterConnectionBroken = False - except XenAPI.Failure as e: - if e.details[0] != 'HOST_IS_SLAVE': # Ignore slave errors when testing - session = None - self.error = e - except socket.timeout: - session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' - except Exception as e: - session = None - self.error = e return session def NewSession(self):