summaryrefslogtreecommitdiff
path: root/source3/stf/comfychair.py
blob: d7e23efc10f15acae7a67a517361bd495effc0d6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
#! /usr/bin/env python

# Copyright (C) 2002, 2003 by Martin Pool <mbp@samba.org>
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

"""comfychair: a Python-based instrument of software torture.

Copyright (C) 2002, 2003 by Martin Pool <mbp@samba.org>

This is a test framework designed for testing programs written in
Python, or (through a fork/exec interface) any other language.

For more information, see the file README.comfychair.

To run a test suite based on ComfyChair, just run it as a program.
"""

# TODO: Put everything into a temporary directory?

# TODO: Have a means for tests to customize the display of their
# failure messages.  In particular, if a shell command failed, then
# give its stderr.

import sys, re

class TestCase:
    """A base class for tests.  This class defines required functions which
    can optionally be overridden by subclasses.  It also provides some
    utility functions for"""

    def __init__(self):
        self.test_log = ""
        self.background_pids = []
    
    def setup(self):
        """Set up test fixture."""
        pass

    def teardown(self):
        """Tear down test fixture."""
        pass

    def runtest(self):
        """Run the test."""
        pass

    def fail(self, reason = ""):
        """Say the test failed."""
        raise AssertionError(reason)


    #############################################################
    # Requisition methods

    def require(self, predicate, message):
        """Check a predicate for running this test.

If the predicate value is not true, the test is skipped with a message explaining
why."""
        if not predicate:
            raise NotRunError, message

    def require_root(self):
        """Skip this test unless run by root."""
        import os
        self.require(os.getuid() == 0,
                     "must be root to run this test")

    #############################################################
    # Assertion methods

    def assert_(self, expr, reason = ""):
        if not expr:
            raise AssertionError(reason)

    def assert_equal(self, a, b):
        if not a == b:
            raise AssertionError("assertEquals failed: %s" % `(a, b)`)
            
    def assert_notequal(self, a, b):
        if a == b:
            raise AssertionError("assertNotEqual failed: %s" % `(a, b)`)

    def assert_re_match(self, pattern, s):
        """Assert that a string matches a particular pattern

        Inputs:
          pattern      string: regular expression
          s            string: to be matched

        Raises:
          AssertionError if not matched
          """
        if not re.match(pattern, s):
            raise AssertionError("string does not match regexp\n"
                                 "    string: %s\n"
                                 "    re: %s" % (`s`, `pattern`))

    def assert_re_search(self, pattern, s):
        """Assert that a string *contains* a particular pattern

        Inputs:
          pattern      string: regular expression
          s            string: to be searched

        Raises:
          AssertionError if not matched
          """
        if not re.search(pattern, s):
            raise AssertionError("string does not contain regexp\n"
                                 "    string: %s\n"
                                 "    re: %s" % (`s`, `pattern`))


    def assert_no_file(self, filename):
        import os.path
        assert not os.path.exists(filename), ("file exists but should not: %s" % filename)


    #############################################################
    # Methods for running programs

    def runcmd_background(self, cmd):
        import os
        name = cmd[0]
        self.test_log = self.test_log + "Run in background:\n" + `cmd` + "\n"
        pid = os.spawnvp(os.P_NOWAIT, name, cmd)
        self.test_log = self.test_log + "pid: %d\n" % pid
        return pid


    def runcmd(self, cmd, expectedResult = 0):
        """Run a command, fail if the command returns an unexpected exit
        code.  Return the output produced."""
        rc, output = self.runcmd_unchecked(cmd)
        if rc != expectedResult:
            raise AssertionError("command returned %d; expected %s: \"%s\"" %
                                 (rc, expectedResult, cmd))

        return output

    def runcmd_unchecked(self, cmd, skip_on_noexec = 0):
        """Invoke a command; return (exitcode, stdout)"""
        import os, popen2
        pobj = popen2.Popen4(cmd)
        output = pobj.fromchild.read()
        waitstatus = pobj.wait()
        assert not os.WIFSIGNALED(waitstatus), \
               ("%s terminated with signal %d", cmd, os.WTERMSIG(waitstatus))
        rc = os.WEXITSTATUS(waitstatus)
        self.test_log = self.test_log + ("""Run command: %s
Wait status: %#x
Output:
%s""" % (cmd, waitstatus, output))
        if skip_on_noexec and rc == 127:
            # Either we could not execute the command or the command
            # returned exit code 127.  According to system(3) we can't
            # tell the difference.
            raise NotRunError, "could not execute %s" % `cmd`
        return rc, output

    def explain_failure(self, exc_info = None):
        import traceback
        # Move along, nothing to see here
        if not exc_info and self.test_log == "":
            return
        print "-----------------------------------------------------------------"
        if exc_info:
            traceback.print_exc(file=sys.stdout)
        print self.test_log
        print "-----------------------------------------------------------------"


    def log(self, msg):
        """Log a message to the test log.  This message is displayed if
        the test fails, or when the runtests function is invoked with
        the verbose option."""
        self.test_log = self.test_log + msg + "\n"


class NotRunError(Exception):
    """Raised if a test must be skipped because of missing resources"""
    def __init__(self, value = None):
        self.value = value


def runtests(test_list, verbose = 0):
    """Run a series of tests.

    Eventually, this routine will also examine sys.argv[] to handle
    extra options.

    Inputs:
      test_list    sequence of callable test objects

    Returns:
      unix return code: 0 for success, 1 for failures, 2 for test failure
    """
    import traceback
    ret = 0
    for test_class in test_list:
        print "%-30s" % _test_name(test_class),
        # flush now so that long running tests are easier to follow
        sys.stdout.flush()

        try:
            try: # run test and show result
                obj = test_class()
                if hasattr(obj, "setup"):
                    obj.setup()
                obj.runtest()
                print "OK"
            except KeyboardInterrupt:
                print "INTERRUPT"
                obj.explain_failure(sys.exc_info())
                ret = 2
                break
            except NotRunError, msg:
                print "NOTRUN, %s" % msg.value
            except:
                print "FAIL"
                obj.explain_failure(sys.exc_info())
                ret = 1
        finally:
            try:
                if hasattr(obj, "teardown"):
                    obj.teardown()
            except KeyboardInterrupt:
                print "interrupted during teardown"
                obj.explain_failure(sys.exc_info())
                ret = 2
                break
            except:
                print "error during teardown"
                obj.explain_failure(sys.exc_info())
                ret = 1
        # Display log file if we're verbose
        if ret == 0 and verbose:
            obj.explain_failure()
            
    return ret


def _test_name(test_class):
    """Return a human-readable name for a test class.
    """
    try:
        return test_class.__name__
    except:
        return `test_class`


def print_help():
    """Help for people running tests"""
    import sys
    print """%s: software test suite based on ComfyChair

usage:
    To run all tests, just run this program.  To run particular tests,
    list them on the command line.

options:
    --help           show usage message
    --list           list available tests
    --verbose        show more information while running tests
""" % sys.argv[0]


def print_list(test_list):
    """Show list of available tests"""
    for test_class in test_list:
        print "    %s" % _test_name(test_class)


def main(tests):
    """Main entry point for test suites based on ComfyChair.

Test suites should contain this boilerplate:

    if __name__ == '__main__':
        comfychair.main(tests)

This function handles standard options such as --help and --list, and
by default runs all tests in the suggested order.

Calls sys.exit() on completion.
"""
    from sys import argv
    import getopt, sys

    verbose = 0

    opts, args = getopt.getopt(argv[1:], '', ['help', 'list', 'verbose'])
    if ('--help', '') in opts:
        print_help()
        return
    elif ('--list', '') in opts:
        print_list(tests)
        return 

    if ('--verbose', '') in opts:
        verbose = 1

    if args:
        by_name = {}
        for t in tests:
            by_name[_test_name(t)] = t
        which_tests = [by_name[name] for name in args]
    else:
        which_tests = tests

    sys.exit(runtests(which_tests, verbose))


if __name__ == '__main__':
    print __doc__