wintest: cope with VMs sometimes not rebooting

my w2k8 image occasionally gets stuck in the early stages of
booting. This adds code to detect a failed reboot, in which case the
VM is reset

Pair-Programmed-With: Andrew Bartlett <abartlet@samba.org>

Autobuild-User: Andrew Tridgell <tridge@samba.org>
Autobuild-Date: Sat Nov 27 02:42:19 CET 2010 on sn-devel-104

4 files changed, 33 insertions, 7 deletions

diff --git a/wintest/conf/abartlet.conf b/wintest/conf/abartlet.conf
index f775c81b3a..00578f437d 100644
--- a/wintest/conf/abartlet.conf
+++ b/wintest/conf/abartlet.conf
@@ -12,6 +12,7 @@ DEBUGLEVEL	      : 1
 # commands to control VMs
 VM_POWEROFF           : virsh destroy ${VMNAME}
 VM_RESTORE            : virsh snapshot-revert ${VMNAME} ${SNAPSHOT}
+VM_RESET              : virsh reboot ${VMNAME}
 
 # interfaces to create
 INTERFACE            : virbr0:0
diff --git a/wintest/conf/tridge.conf b/wintest/conf/tridge.conf
index 1c7ed0bda7..72b54d88da 100644
--- a/wintest/conf/tridge.conf
+++ b/wintest/conf/tridge.conf
@@ -12,6 +12,7 @@ DEBUGLEVEL	      : 1
 # commands to control VMs
 VM_POWEROFF           : su tridge -c "VBoxManage controlvm ${VMNAME} poweroff"
 VM_RESTORE            : su tridge -c "VBoxManage snapshot ${VMNAME} restore ${SNAPSHOT} && VBoxManage startvm ${VMNAME}"
+VM_RESET              : su tridge -c "VBoxManage controlvm ${VMNAME} reset"
 
 # interfaces to listen on
 INTERFACE             : virbr0:0
diff --git a/wintest/test-s4-howto.py b/wintest/test-s4-howto.py
index 3d4b5e0511..ae0dc1437a 100755
--- a/wintest/test-s4-howto.py
+++ b/wintest/test-s4-howto.py
@@ -261,8 +261,7 @@ def run_winjoin(t, vm):
     child.expect("The command completed successfully")
     child.expect("C:")
     child.sendline("shutdown /r -t 0")
-    t.port_wait("${WIN_IP}", 139, wait_for_fail=True)
-    t.port_wait("${WIN_IP}", 139)
+    t.wait_reboot()
     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_USER}", "${WIN_PASS}", set_time=True, set_ip=True)
     child.sendline("ipconfig /registerdns")
     child.expect("Registration of the DNS resource records for all adapters of this computer has been initiated. Any errors will be reported in the Event Viewer")
@@ -318,8 +317,7 @@ SafeModeAdminPassword=${PASSWORD1}
     i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:"], timeout=120)
     if i == 1 or i == 2:
         raise Exception("dcpromo failed")
-    t.port_wait("${WIN_IP}", 139, wait_for_fail=True)
-    t.port_wait("${WIN_IP}", 139)
+    t.wait_reboot()
 
 
 def test_dcpromo(t, vm):
@@ -462,8 +460,7 @@ RebootOnCompletion=No
     if i != 0:
         raise Exception("dcpromo failed")
     child.sendline("shutdown -r -t 0")
-    t.port_wait("${WIN_IP}", 139, wait_for_fail=True)
-    t.port_wait("${WIN_IP}", 139)
+    t.wait_reboot()
 
 
 
diff --git a/wintest/wintest.py b/wintest/wintest.py
index 1c5b087aa1..effc77e95e 100644
--- a/wintest/wintest.py
+++ b/wintest/wintest.py
@@ -200,7 +200,8 @@ class wintest():
                 return
             except:
                 time.sleep(delay)
-                retries = retries - 1
+                retries -= 1
+                self.info("retrying (retries=%u delay=%u)" % (retries, delay))
         raise RuntimeError("Failed to find %s" % contains)
 
     def pexpect_spawn(self, cmd, timeout=60, crlf=True):
@@ -240,6 +241,11 @@ class wintest():
         self.setvar('VMNAME', vmname)
         self.run_cmd("${VM_POWEROFF}", checkfail=checkfail)
 
+    def vm_reset(self, vmname):
+        '''reset a VM'''
+        self.setvar('VMNAME', vmname)
+        self.run_cmd("${VM_RESET}")
+
     def vm_restore(self, vmname, snapshot):
         '''restore a VM'''
         self.setvar('VMNAME', vmname)
@@ -368,6 +374,7 @@ class wintest():
                 return child.after
             retries -= 1
             time.sleep(delay)
+            self.info("retrying (retries=%u delay=%u)" % (retries, delay))
         raise RuntimeError("Failed to resolve IP of %s" % hostname)
 
 
@@ -394,6 +401,7 @@ class wintest():
                 child.close()
                 time.sleep(delay)
                 retries -= 1
+                self.info("retrying (retries=%u delay=%u)" % (retries, delay))
                 continue
             child.expect("password:")
             child.sendline(password)
@@ -408,6 +416,7 @@ class wintest():
                 child.close()
                 time.sleep(delay)
                 retries -= 1
+                self.info("retrying (retries=%u delay=%u)" % (retries, delay))
                 continue
             if set_dns:
                 set_dns = False
@@ -459,3 +468,21 @@ class wintest():
                 if base + '_IP' in self.vars:
                     ret[self.vars[base + '_REALM']] = self.vars[base + '_IP']
         return ret
+
+    def wait_reboot(self, retries=3):
+        '''wait for a VM to reboot'''
+
+        # first wait for it to shutdown
+        self.port_wait("${WIN_IP}", 139, wait_for_fail=True, delay=6)
+
+        # now wait for it to come back. If it fails to come back
+        # then try resetting it
+        while retries > 0:
+            try:
+                self.port_wait("${WIN_IP}", 139)
+                return
+            except:
+                retries -= 1
+                self.vm_reset("${WIN_VM}")
+                self.info("retrying reboot (retries=%u)" % retries)
+        raise RuntimeError(self.substitute("VM ${WIN_VM} failed to reboot"))