diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/subunit/Apache-2.0 | 202 | ||||
| -rw-r--r-- | lib/subunit/BSD | 26 | ||||
| -rw-r--r-- | lib/subunit/COPYING | 36 | ||||
| -rw-r--r-- | lib/subunit/INSTALL | 25 | ||||
| -rw-r--r-- | lib/subunit/Makefile.am | 136 | ||||
| -rw-r--r-- | lib/subunit/NEWS | 174 | ||||
| -rw-r--r-- | lib/subunit/README | 217 | ||||
| -rw-r--r-- | lib/subunit/c++/README | 50 | ||||
| -rw-r--r-- | lib/subunit/c++/SubunitTestProgressListener.cpp | 63 | ||||
| -rw-r--r-- | lib/subunit/c++/SubunitTestProgressListener.h | 56 | ||||
| -rw-r--r-- | lib/subunit/c/README | 68 | ||||
| -rw-r--r-- | lib/subunit/c/include/subunit/child.h | 79 | ||||
| -rw-r--r-- | lib/subunit/c/lib/child.c | 82 | ||||
| -rw-r--r-- | lib/subunit/c/tests/test_child.c | 192 | ||||
| -rw-r--r-- | lib/subunit/configure.ac | 75 | ||||
| -rw-r--r-- | lib/subunit/libcppunit_subunit.pc.in | 11 | ||||
| -rw-r--r-- | lib/subunit/libsubunit.pc.in | 11 | ||||
| -rwxr-xr-x | lib/subunit/runtests.py | 138 | ||||
| -rw-r--r-- | lib/subunit/shell/README | 62 | ||||
| -rw-r--r-- | lib/subunit/shell/share/subunit.sh | 56 | ||||
| -rwxr-xr-x | lib/subunit/shell/tests/test_function_output.sh | 97 | ||||
| -rwxr-xr-x | lib/subunit/shell/tests/test_source_library.sh | 108 | ||||
| -rwxr-xr-x | lib/update-external.sh | 5 |
23 files changed, 1959 insertions, 10 deletions
diff --git a/lib/subunit/Apache-2.0 b/lib/subunit/Apache-2.0 new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/lib/subunit/Apache-2.0 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/subunit/BSD b/lib/subunit/BSD new file mode 100644 index 0000000000..fa130cd529 --- /dev/null +++ b/lib/subunit/BSD @@ -0,0 +1,26 @@ +Copyright (c) Robert Collins and Subunit contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of Robert Collins nor the names of Subunit contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY ROBERT COLLINS AND SUBUNIT CONTRIBUTORS ``AS IS'' +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/lib/subunit/COPYING b/lib/subunit/COPYING new file mode 100644 index 0000000000..3ba50f8e08 --- /dev/null +++ b/lib/subunit/COPYING @@ -0,0 +1,36 @@ +Subunit is licensed under two licenses, the Apache License, Version 2.0 or the +3-clause BSD License. You may use this project under either of these licenses +- choose the one that works best for you. + +We require contributions to be licensed under both licenses. The primary +difference between them is that the Apache license takes care of potential +issues with Patents and other intellectual property concerns. This is +important to Subunit as Subunit wants to be license compatible in a very +broad manner to allow reuse and incorporation into other projects. + +Generally every source file in Subunit needs a license grant under both these +licenses. As the code is shipped as a single unit, a brief form is used: +---- +Copyright (c) [yyyy][,yyyy]* [name or 'Subunit Contributors'] + +Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +license at the users choice. A copy of both licenses are available in the +project source as Apache-2.0 and BSD. You may not use this file except in +compliance with one of these two licences. + +Unless required by applicable law or agreed to in writing, software +distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +license you chose for the specific language governing permissions and +limitations under that license. +---- + +Code that has been incorporated into Subunit from other projects will +naturally be under its own license, and will retain that license. + +A known list of such code is maintained here: +* The python/iso8601 module by Michael Twomey, distributed under an MIT style + licence - see python/iso8601/LICENSE for details. +* The runtests.py and python/subunit/tests/TestUtil.py module are GPL test + support modules. There are not installed by Subunit - they are only ever + used on the build machine. Copyright 2004 Canonical Limited. diff --git a/lib/subunit/INSTALL b/lib/subunit/INSTALL new file mode 100644 index 0000000000..79cf7c18d0 --- /dev/null +++ b/lib/subunit/INSTALL @@ -0,0 +1,25 @@ +To install subunit +------------------ + +Bootstrap:: + autoreconf -vi +Configure:: + ./configure +Install:: + make install + +Dependencies +------------ + +* Python for the filters +* 'testtools' (On Debian and Ubuntu systems the 'python-testtools' package, + the testtools package on pypi, or https://launchpad.net/testtools) for + the extended test API which permits attachments. Version 0.9.2 or newer is + required. Of particular note, http://testtools.python-hosting.com/ is not + the testtools you want. +* A C compiler for the C bindings +* Perl for the Perl tools (including subunit-diff) +* Check to run the subunit test suite. +* python-gtk2 if you wish to use subunit2gtk +* python-junitxml if you wish to use subunit2junitxml +* pkg-config for configure detection of supporting libraries. diff --git a/lib/subunit/Makefile.am b/lib/subunit/Makefile.am new file mode 100644 index 0000000000..716fa0fe21 --- /dev/null +++ b/lib/subunit/Makefile.am @@ -0,0 +1,136 @@ +EXTRA_DIST = \ + .bzrignore \ + Apache-2.0 \ + BSD \ + INSTALL \ + Makefile.am \ + NEWS \ + README \ + c++/README \ + c/README \ + c/check-subunit-0.9.3.patch \ + c/check-subunit-0.9.5.patch \ + c/check-subunit-0.9.6.patch \ + perl/Makefile.PL.in \ + perl/lib/Subunit.pm \ + perl/lib/Subunit/Diff.pm \ + perl/subunit-diff \ + python/iso8601/LICENSE \ + python/iso8601/README \ + python/iso8601/README.subunit \ + python/iso8601/setup.py \ + python/iso8601/test_iso8601.py \ + python/subunit/tests/TestUtil.py \ + python/subunit/tests/__init__.py \ + python/subunit/tests/sample-script.py \ + python/subunit/tests/sample-two-script.py \ + python/subunit/tests/test_chunked.py \ + python/subunit/tests/test_details.py \ + python/subunit/tests/test_progress_model.py \ + python/subunit/tests/test_subunit_filter.py \ + python/subunit/tests/test_subunit_stats.py \ + python/subunit/tests/test_subunit_tags.py \ + python/subunit/tests/test_tap2subunit.py \ + python/subunit/tests/test_test_protocol.py \ + python/subunit/tests/test_test_results.py \ + runtests.py \ + shell/README \ + shell/share/subunit.sh \ + shell/subunit-ui.patch \ + shell/tests/test_function_output.sh \ + shell/tests/test_source_library.sh + +ACLOCAL_AMFLAGS = -I m4 + +include_subunitdir = $(includedir)/subunit + +dist_bin_SCRIPTS = \ + filters/subunit-filter \ + filters/subunit-ls \ + filters/subunit-stats \ + filters/subunit-tags \ + filters/subunit2gtk \ + filters/subunit2junitxml \ + filters/subunit2pyunit \ + filters/tap2subunit + +TESTS_ENVIRONMENT = SHELL_SHARE='$(top_srcdir)/shell/share/' PYTHONPATH='$(abs_top_srcdir)/python':${PYTHONPATH} +TESTS = runtests.py $(check_PROGRAMS) + +## install libsubunit.pc +pcdatadir = $(libdir)/pkgconfig +pcdata_DATA = \ + libsubunit.pc \ + libcppunit_subunit.pc + +pkgpython_PYTHON = \ + python/subunit/__init__.py \ + python/subunit/chunked.py \ + python/subunit/details.py \ + python/subunit/iso8601.py \ + python/subunit/progress_model.py \ + python/subunit/run.py \ + python/subunit/test_results.py + +lib_LTLIBRARIES = libsubunit.la +lib_LTLIBRARIES += libcppunit_subunit.la + +include_subunit_HEADERS = \ + c/include/subunit/child.h \ + c++/SubunitTestProgressListener.h + +check_PROGRAMS = \ + c/tests/test_child + +check_SCRIPTS = \ + runtests.py + +libsubunit_la_SOURCES = \ + c/lib/child.c \ + c/include/subunit/child.h + +libcppunit_subunit_la_SOURCES = \ + c++/SubunitTestProgressListener.cpp \ + c++/SubunitTestProgressListener.h + +tests_LDADD = @CHECK_LIBS@ $(top_builddir)/libsubunit.la +c_tests_test_child_CFLAGS = -I$(top_srcdir)/c/include $(SUBUNIT_CFLAGS) @CHECK_CFLAGS@ +c_tests_test_child_LDADD = $(tests_LDADD) + + +all-local: perl/Makefile + $(MAKE) -C perl all + +check-local: perl/Makefile + $(MAKE) -C perl check + +clean-local: + find . -type f -name "*.pyc" -exec rm {} ';' + rm -f perl/Makefile + +# Remove perl dir for VPATH builds. +distclean-local: + -rmdir perl > /dev/null + -rm perl/Makefile.PL > /dev/null + +install-exec-local: perl/Makefile + $(MAKE) -C perl install + +mostlyclean-local: + rm -rf perl/blib + rm -rf perl/pm_to_blib + +# 'uninstall' perl files during distcheck +uninstall-local: + if [ "_inst" = `basename ${prefix}` ]; then \ + $(MAKE) -C perl uninstall_distcheck; \ + rm -f "$(DESTDIR)$(bindir)"/subunit-diff; \ + fi + +# The default for MakeMaker; can be overridden by exporting +INSTALLDIRS ?= site + +perl/Makefile: perl/Makefile.PL + mkdir -p perl + cd perl && perl Makefile.PL INSTALLDIRS=${INSTALLDIRS} + -rm perl/Makefile.old > /dev/null diff --git a/lib/subunit/NEWS b/lib/subunit/NEWS new file mode 100644 index 0000000000..7c933c8f6e --- /dev/null +++ b/lib/subunit/NEWS @@ -0,0 +1,174 @@ +--------------------- +subunit release notes +--------------------- + +NEXT (In development) +--------------------- + +BUG FIXES +~~~~~~~~~ + +* Fix incorrect reference to subunit_test_failf in c/README. + (Brad Hards, #524341) + +* Fix incorrect ordering of tags method parameters in TestResultDecorator. This + is purely cosmetic as the parameters are passed down with no interpretation. + (Robert Collins, #537611) + +0.0.5 +----- + +BUG FIXES +~~~~~~~~~ + +* make check was failing if subunit wasn't installed due to a missing include + path for the test program test_child. + +* make distcheck was failing due to a missing $(top_srcdir) rune. + +IMPROVEMENTS +~~~~~~~~~~~~ + +* New filter `subunit-notify` that will show a notification window with test + statistics when the test run finishes. + +* subunit.run will now pipe its output to the command in the + SUBUNIT_FORMATTER environment variable, if set. + +0.0.4 +----- + +BUG FIXES +~~~~~~~~~ + +* subunit2junitxml -f required a value, this is now fixed and -f acts as a + boolean switch with no parameter. + +* Building with autoconf 2.65 is now supported. + + +0.0.3 +----- + + CHANGES: + + * License change, by unanimous agreement of contributors to BSD/Apache + License Version 2.0. This makes Subunit compatible with more testing + frameworks. + + IMPROVEMENTS: + + * CPPUnit is now directly supported: subunit builds a cppunit listener + ``libcppunit-subunit``. + + * In the python API ``addExpectedFailure`` and ``addUnexpectedSuccess`` + from python 2.7/3.1 are now supported. ``addExpectedFailure`` is + serialised as ``xfail``, and ``addUnexpectedSuccess`` as ``success``. + The ``ProtocolTestCase`` parser now calls outcomes using an extended + API that permits attaching arbitrary MIME resources such as text files + log entries and so on. This extended API is being developed with the + Python testing community, and is in flux. ``TestResult`` objects that + do not support the API will be detected and transparently downgraded + back to the regular Python unittest API. + + * INSTALLDIRS can be set to control the perl MakeMaker 'INSTALLDIRS' + viarable when installing. + + * Multipart test outcomes are tentatively supported; the exact protocol + for them, both serialiser and object is not yet finalised. Testers and + early adopters are sought. As part of this and also in an attempt to + provider a more precise focus on the wire protocol and toolchain, + Subunit now depends on testtools (http://launchpad.net/testtools) + release 0.9.0 or newer. + + * subunit2junitxml supports a new option, --forward which causes it + to forward the raw subunit stream in a similar manner to tee. This + is used with the -o option to both write a xml report and get some + other subunit filter to process the stream. + + * The C library now has ``subunit_test_skip``. + + BUG FIXES: + + * Install progress_model.py correctly. + + * Non-gcc builds will no longer try to use gcc specific flags. + (Thanks trondn-norbye) + + API CHANGES: + + INTERNALS: + +0.0.2 +----- + + CHANGES: + + IMPROVEMENTS: + + * A number of filters now support ``--no-passthrough`` to cause all + non-subunit content to be discarded. This is useful when precise control + over what is output is required - such as with subunit2junitxml. + + * A small perl parser is now included, and a new ``subunit-diff`` tool + using that is included. (Jelmer Vernooij) + + * Subunit streams can now include optional, incremental lookahead + information about progress. This allows reporters to make estimates + about completion, when such information is available. See the README + under ``progress`` for more details. + + * ``subunit-filter`` now supports regex filtering via ``--with`` and + ``without`` options. (Martin Pool) + + * ``subunit2gtk`` has been added, a filter that shows a GTK summary of a + test stream. + + * ``subunit2pyunit`` has a --progress flag which will cause the bzrlib + test reporter to be used, which has a textual progress bar. This requires + a recent bzrlib as a minor bugfix was required in bzrlib to support this. + + * ``subunit2junitxml`` has been added. This filter converts a subunit + stream to a single JUnit style XML stream using the pyjunitxml + python library. + + * The shell functions support skipping via ``subunit_skip_test`` now. + + BUG FIXES: + + * ``xfail`` outcomes are now passed to python TestResult's via + addExpectedFailure if it is present on the TestResult. Python 2.6 and + earlier which do not have this function will have ``xfail`` outcomes + passed through as success outcomes as earlier versions of subunit did. + + API CHANGES: + + * tags are no longer passed around in python via the ``TestCase.tags`` + attribute. Instead ``TestResult.tags(new_tags, gone_tags)`` is called, + and like in the protocol, if called while a test is active only applies + to that test. (Robert Collins) + + * ``TestResultFilter`` takes a new optional constructor parameter + ``filter_predicate``. (Martin Pool) + + * When a progress: directive is encountered in a subunit stream, the + python bindings now call the ``progress(offset, whence)`` method on + ``TestResult``. + + * When a time: directive is encountered in a subunit stream, the python + bindings now call the ``time(seconds)`` method on ``TestResult``. + + INTERNALS: + + * (python) Added ``subunit.test_results.AutoTimingTestResultDecorator``. Most + users of subunit will want to wrap their ``TestProtocolClient`` objects + in this decorator to get test timing data for performance analysis. + + * (python) ExecTestCase supports passing arguments to test scripts. + + * (python) New helper ``subunit.test_results.HookedTestResultDecorator`` + which can be used to call some code on every event, without having to + implement all the event methods. + + * (python) ``TestProtocolClient.time(a_datetime)`` has been added which + causes a timestamp to be output to the stream. diff --git a/lib/subunit/README b/lib/subunit/README index c657992c7a..9740d013a5 100644 --- a/lib/subunit/README +++ b/lib/subunit/README @@ -1,7 +1,212 @@ -This directory contains some helper code for the Subunit protocol. It is -a partial import of the code from the upstream subunit project, which can -be found at https://launchpad.net/subunit. -To update the snapshot, run update.sh in this directory. When making changes -here, please also submit them upstream - otherwise they'll be gone by the -next time we import subunit. + subunit: A streaming protocol for test results + Copyright (C) 2005-2009 Robert Collins <robertc@robertcollins.net> + + Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + license at the users choice. A copy of both licenses are available in the + project source as Apache-2.0 and BSD. You may not use this file except in + compliance with one of these two licences. + + Unless required by applicable law or agreed to in writing, software + distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + license you chose for the specific language governing permissions and + limitations under that license. + + See the COPYING file for full details on the licensing of Subunit. + + subunit reuses iso8601 by Michael Twomey, distributed under an MIT style + licence - see python/iso8601/LICENSE for details. + +Subunit +------- + +Subunit is a streaming protocol for test results. The protocol is human +readable and easily generated and parsed. By design all the components of +the protocol conceptually fit into the xUnit TestCase->TestResult interaction. + +Subunit comes with command line filters to process a subunit stream and +language bindings for python, C, C++ and shell. Bindings are easy to write +for other languages. + +A number of useful things can be done easily with subunit: + * Test aggregation: Tests run separately can be combined and then + reported/displayed together. For instance, tests from different languages + can be shown as a seamless whole. + * Test archiving: A test run may be recorded and replayed later. + * Test isolation: Tests that may crash or otherwise interact badly with each + other can be run seperately and then aggregated, rather than interfering + with each other. + * Grid testing: subunit can act as the necessary serialisation and + deserialiation to get test runs on distributed machines to be reported in + real time. + +Subunit supplies the following filters: + * tap2subunit - convert perl's TestAnythingProtocol to subunit. + * subunit2pyunit - convert a subunit stream to pyunit test results. + * subunit2gtk - show a subunit stream in GTK. + * subunit2junitxml - convert a subunit stream to JUnit's XML format. + * subunit-diff - compare two subunit streams. + * subunit-filter - filter out tests from a subunit stream. + * subunit-ls - list info about tests present in a subunit stream. + * subunit-stats - generate a summary of a subunit stream. + * subunit-tags - add or remove tags from a stream. + +Integration with other tools +---------------------------- + +Subunit's language bindings act as integration with various test runners like +'check', 'cppunit', Python's 'unittest'. Beyond that a small amount of glue +(typically a few lines) will allow Subunit to be used in more sophisticated +ways. + +Python +====== + +Subunit has excellent Python support: most of the filters and tools are written +in python and there are facilities for using Subunit to increase test isolation +seamlessly within a test suite. + +One simple way to run an existing python test suite and have it output subunit +is the module ``subunit.run``:: + + $ python -m subunit.run mypackage.tests.test_suite + +For more information on the Python support Subunit offers , please see +``pydoc subunit``, or the source in ``python/subunit/__init__.py`` + +C += + +Subunit has C bindings to emit the protocol, and comes with a patch for 'check' +which has been nominally accepted by the 'check' developers. See 'c/README' for +more details. + +C++ +=== + +The C library is includable and usable directly from C++. A TestListener for +CPPUnit is included in the Subunit distribution. See 'c++/README' for details. + +shell +===== + +Similar to C, the shell bindings consist of simple functions to output protocol +elements, and a patch for adding subunit output to the 'ShUnit' shell test +runner. See 'shell/README' for details. + +Filter recipes +-------------- + +To ignore some failing tests whose root cause is already known:: + + subunit-filter --without 'AttributeError.*flavor' + + +The protocol +------------ + +Sample subunit wire contents +---------------------------- + +The following:: + test: test foo works + success: test foo works. + test: tar a file. + failure: tar a file. [ + .. + ].. space is eaten. + foo.c:34 WARNING foo is not defined. + ] + a writeln to stdout + +When run through subunit2pyunit:: + .F + a writeln to stdout + + ======================== + FAILURE: tar a file. + ------------------- + .. + ].. space is eaten. + foo.c:34 WARNING foo is not defined. + + +Subunit protocol description +============================ + +This description is being ported to an EBNF style. Currently its only partly in +that style, but should be fairly clear all the same. When in doubt, refer the +source (and ideally help fix up the description!). Generally the protocol is +line orientated and consists of either directives and their parameters, or +when outside a DETAILS region unexpected lines which are not interpreted by +the parser - they should be forwarded unaltered. + +test|testing|test:|testing: test label +success|success:|successful|successful: test label +success|success:|successful|successful: test label DETAILS +failure: test label +failure: test label DETAILS +error: test label +error: test label DETAILS +skip[:] test label +skip[:] test label DETAILS +xfail[:] test label +xfail[:] test label DETAILS +progress: [+|-]X +progress: push +progress: pop +tags: [-]TAG ... +time: YYYY-MM-DD HH:MM:SSZ + +DETAILS ::= BRACKETED | MULTIPART +BRACKETED ::= '[' CR lines ']' CR +MULTIPART ::= '[ multipart' CR PART* ']' CR +PART ::= PART_TYPE CR NAME CR PART_BYTES CR +PART_TYPE ::= Content-Type: type/sub-type(;parameter=value,parameter=value) +PART_BYTES ::= (DIGITS CR LF BYTE{DIGITS})* '0' CR LF + +unexpected output on stdout -> stdout. +exit w/0 or last test completing -> error + +Tags given outside a test are applied to all following tests +Tags given after a test: line and before the result line for the same test +apply only to that test, and inherit the current global tags. +A '-' before a tag is used to remove tags - e.g. to prevent a global tag +applying to a single test, or to cancel a global tag. + +The progress directive is used to provide progress information about a stream +so that stream consumer can provide completion estimates, progress bars and so +on. Stream generators that know how many tests will be present in the stream +should output "progress: COUNT". Stream filters that add tests should output +"progress: +COUNT", and those that remove tests should output +"progress: -COUNT". An absolute count should reset the progress indicators in +use - it indicates that two separate streams from different generators have +been trivially concatenated together, and there is no knowledge of how many +more complete streams are incoming. Smart concatenation could scan each stream +for their count and sum them, or alternatively translate absolute counts into +relative counts inline. It is recommended that outputters avoid absolute counts +unless necessary. The push and pop directives are used to provide local regions +for progress reporting. This fits with hierarchically operating test +environments - such as those that organise tests into suites - the top-most +runner can report on the number of suites, and each suite surround its output +with a (push, pop) pair. Interpreters should interpret a pop as also advancing +the progress of the restored level by one step. Encountering progress +directives between the start and end of a test pair indicates that a previous +test was interrupted and did not cleanly terminate: it should be implicitly +closed with an error (the same as when a stream ends with no closing test +directive for the most recently started test). + +The time directive acts as a clock event - it sets the time for all future +events. The value should be a valid ISO8601 time. + +The skip result is used to indicate a test that was found by the runner but not +fully executed due to some policy or dependency issue. This is represented in +python using the addSkip interface that testtools +(https://edge.launchpad.net/testtools) defines. When communicating with a non +skip aware test result, the test is reported as an error. +The xfail result is used to indicate a test that was expected to fail failing +in the expected manner. As this is a normal condition for such tests it is +represented as a successful test in Python. +In future, skip and xfail results will be represented semantically in Python, +but some discussion is underway on the right way to do this. diff --git a/lib/subunit/c++/README b/lib/subunit/c++/README new file mode 100644 index 0000000000..7b8184400e --- /dev/null +++ b/lib/subunit/c++/README @@ -0,0 +1,50 @@ +# +# subunit C++ bindings. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +Currently there are no native C++ bindings for subunit. However the C library +can be used from C++ safely. A CPPUnit listener is built as part of Subunit to +allow CPPUnit users to simply get Subunit output. + +To use the listener, use pkg-config (or your preferred replacement) to get the +cflags and link settings from libcppunit_subunit.pc. + +In your test driver main, use SubunitTestProgressListener, as shown in this +example main:: + + { + // Create the event manager and test controller + CPPUNIT_NS::TestResult controller; + + // Add a listener that collects test result + // so we can get the overall status. + // note this isn't needed for subunit... + CPPUNIT_NS::TestResultCollector result; + controller.addListener( &result ); + + // Add a listener that print test activity in subunit format. + CPPUNIT_NS::SubunitTestProgressListener progress; + controller.addListener( &progress ); + + // Add the top suite to the test runner + CPPUNIT_NS::TestRunner runner; + runner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest() ); + runner.run( controller ); + + return result.wasSuccessful() ? 0 : 1; + } diff --git a/lib/subunit/c++/SubunitTestProgressListener.cpp b/lib/subunit/c++/SubunitTestProgressListener.cpp new file mode 100644 index 0000000000..76cd9e1194 --- /dev/null +++ b/lib/subunit/c++/SubunitTestProgressListener.cpp @@ -0,0 +1,63 @@ +/* Subunit test listener for cppunit (http://cppunit.sourceforge.net). + * Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> + * + * Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + * license at the users choice. A copy of both licenses are available in the + * project source as Apache-2.0 and BSD. You may not use this file except in + * compliance with one of these two licences. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under these licenses is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license you chose for the specific language governing permissions + * and limitations under that license. + */ + +#include <cppunit/Exception.h> +#include <cppunit/Test.h> +#include <cppunit/TestFailure.h> +#include <cppunit/TextOutputter.h> +#include <iostream> + +// Have to be able to import the public interface without config.h. +#include "SubunitTestProgressListener.h" +#include "config.h" +#include "subunit/child.h" + + +CPPUNIT_NS_BEGIN + + +void +SubunitTestProgressListener::startTest( Test *test ) +{ + subunit_test_start(test->getName().c_str()); + last_test_failed = false; +} + +void +SubunitTestProgressListener::addFailure( const TestFailure &failure ) +{ + std::ostringstream capture_stream; + TextOutputter outputter(NULL, capture_stream); + outputter.printFailureLocation(failure.sourceLine()); + outputter.printFailureDetail(failure.thrownException()); + + if (failure.isError()) + subunit_test_error(failure.failedTestName().c_str(), + capture_stream.str().c_str()); + else + subunit_test_fail(failure.failedTestName().c_str(), + capture_stream.str().c_str()); + last_test_failed = true; +} + +void +SubunitTestProgressListener::endTest( Test *test) +{ + if (!last_test_failed) + subunit_test_pass(test->getName().c_str()); +} + + +CPPUNIT_NS_END diff --git a/lib/subunit/c++/SubunitTestProgressListener.h b/lib/subunit/c++/SubunitTestProgressListener.h new file mode 100644 index 0000000000..5206d833c7 --- /dev/null +++ b/lib/subunit/c++/SubunitTestProgressListener.h @@ -0,0 +1,56 @@ +/* Subunit test listener for cppunit (http://cppunit.sourceforge.net). + * Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> + * + * Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + * license at the users choice. A copy of both licenses are available in the + * project source as Apache-2.0 and BSD. You may not use this file except in + * compliance with one of these two licences. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under these licenses is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license you chose for the specific language governing permissions + * and limitations under that license. + */ +#ifndef CPPUNIT_SUBUNITTESTPROGRESSLISTENER_H +#define CPPUNIT_SUBUNITTESTPROGRESSLISTENER_H + +#include <cppunit/TestListener.h> + + +CPPUNIT_NS_BEGIN + + +/*! + * \brief TestListener that outputs subunit + * (http://www.robertcollins.net/unittest/subunit) compatible output. + * \ingroup TrackingTestExecution + */ +class CPPUNIT_API SubunitTestProgressListener : public TestListener +{ +public: + + SubunitTestProgressListener() {} + + void startTest( Test *test ); + + void addFailure( const TestFailure &failure ); + + void endTest( Test *test ); + +private: + /// Prevents the use of the copy constructor. + SubunitTestProgressListener( const SubunitTestProgressListener © ); + + /// Prevents the use of the copy operator. + void operator =( const SubunitTestProgressListener © ); + +private: + int last_test_failed; +}; + + +CPPUNIT_NS_END + +#endif // CPPUNIT_SUBUNITTESTPROGRESSLISTENER_H + diff --git a/lib/subunit/c/README b/lib/subunit/c/README new file mode 100644 index 0000000000..b62fd45395 --- /dev/null +++ b/lib/subunit/c/README @@ -0,0 +1,68 @@ +# +# subunit C bindings. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +This subtree contains an implementation of the subunit child protocol. +Currently I have no plans to write a test runner in C, so I have not written +an implementation of the parent protocol. [but will happily accept patches]. +This implementation is built using SCons and tested via 'check'. +See the tests/ directory for the test programs. +You can use `make check` or `scons check` to run the tests. + +The C protocol consists of four functions which you can use to output test +metadata trivially. See lib/subunit_child.[ch] for details. + +However, this is not a test runner - subunit provides no support for [for +instance] managing assertions, cleaning up on errors etc. You can look at +'check' (http://check.sourceforge.net/) or +'gunit' (https://garage.maemo.org/projects/gunit) for C unit test +frameworks. +There is a patch for 'check' (check-subunit-*.patch) in this source tree. +Its also available as request ID #1470750 in the sourceforge request tracker +http://sourceforge.net/tracker/index.php. The 'check' developers have indicated +they will merge this during the current release cycle. + +If you are a test environment maintainer - either homegrown, or 'check' or +'gunit' or some other, you will to know how the subunit calls should be used. +Here is what a manually written test using the bindings might look like: + + +void +a_test(void) { + int result; + subunit_test_start("test name"); + # determine if test passes or fails + result = SOME_VALUE; + if (!result) { + subunit_test_pass("test name"); + } else { + subunit_test_fail("test name", + "Something went wrong running something:\n" + "exited with result: '%s'", result); + } +} + +Which when run with a subunit test runner will generate something like: +test name ... ok + +on success, and: + +test name ... FAIL + +====================================================================== +FAIL: test name +---------------------------------------------------------------------- +RemoteError: +Something went wrong running something: +exited with result: '1' diff --git a/lib/subunit/c/include/subunit/child.h b/lib/subunit/c/include/subunit/child.h new file mode 100644 index 0000000000..0a4e60127b --- /dev/null +++ b/lib/subunit/c/include/subunit/child.h @@ -0,0 +1,79 @@ +/** + * + * subunit C bindings. + * Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> + * + * Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + * license at the users choice. A copy of both licenses are available in the + * project source as Apache-2.0 and BSD. You may not use this file except in + * compliance with one of these two licences. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under these licenses is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license you chose for the specific language governing permissions + * and limitations under that license. + **/ + + +#ifdef __cplusplus +extern "C" { +#endif + + +/** + * subunit_test_start: + * + * Report that a test is starting. + * @name: test case name + */ +extern void subunit_test_start(char const * const name); + + +/** + * subunit_test_pass: + * + * Report that a test has passed. + * + * @name: test case name + */ +extern void subunit_test_pass(char const * const name); + + +/** + * subunit_test_fail: + * + * Report that a test has failed. + * @name: test case name + * @error: a string describing the error. + */ +extern void subunit_test_fail(char const * const name, char const * const error); + + +/** + * subunit_test_error: + * + * Report that a test has errored. An error is an unintentional failure - i.e. + * a segfault rather than a failed assertion. + * @name: test case name + * @error: a string describing the error. + */ +extern void subunit_test_error(char const * const name, + char const * const error); + + +/** + * subunit_test_skip: + * + * Report that a test has been skipped. An skip is a test that has not run to + * conclusion but hasn't given an error either - its result is unknown. + * @name: test case name + * @reason: a string describing the reason for the skip. + */ +extern void subunit_test_skip(char const * const name, + char const * const reason); + + +#ifdef __cplusplus +} +#endif diff --git a/lib/subunit/c/lib/child.c b/lib/subunit/c/lib/child.c new file mode 100644 index 0000000000..2b59747c0e --- /dev/null +++ b/lib/subunit/c/lib/child.c @@ -0,0 +1,82 @@ +/** + * + * subunit C child-side bindings: report on tests being run. + * Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> + * + * Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + * license at the users choice. A copy of both licenses are available in the + * project source as Apache-2.0 and BSD. You may not use this file except in + * compliance with one of these two licences. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under these licenses is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license you chose for the specific language governing permissions + * and limitations under that license. + **/ + +#include <stdio.h> +#include <string.h> +#include "subunit/child.h" + +/* Write details about a test event. It is the callers responsibility to ensure + * that details are only provided for events the protocol expects details on. + * @event: The event - e.g. 'skip' + * @name: The test name/id. + * @details: The details of the event, may be NULL if no details are present. + */ +static void +subunit_send_event(char const * const event, char const * const name, + char const * const details) +{ + if (NULL == details) { + fprintf(stdout, "%s: %s\n", event, name); + } else { + fprintf(stdout, "%s: %s [\n", event, name); + fprintf(stdout, "%s", details); + if (details[strlen(details) - 1] != '\n') + fprintf(stdout, "\n"); + fprintf(stdout, "]\n"); + } + fflush(stdout); +} + +/* these functions all flush to ensure that the test runner knows the action + * that has been taken even if the subsequent test etc takes a long time or + * never completes (i.e. a segfault). + */ + +void +subunit_test_start(char const * const name) +{ + subunit_send_event("test", name, NULL); +} + + +void +subunit_test_pass(char const * const name) +{ + /* TODO: add success details as an option */ + subunit_send_event("success", name, NULL); +} + + +void +subunit_test_fail(char const * const name, char const * const error) +{ + subunit_send_event("failure", name, error); +} + + +void +subunit_test_error(char const * const name, char const * const error) +{ + subunit_send_event("error", name, error); +} + + +void +subunit_test_skip(char const * const name, char const * const reason) +{ + subunit_send_event("skip", name, reason); +} diff --git a/lib/subunit/c/tests/test_child.c b/lib/subunit/c/tests/test_child.c new file mode 100644 index 0000000000..6399eeb645 --- /dev/null +++ b/lib/subunit/c/tests/test_child.c @@ -0,0 +1,192 @@ +/** + * + * subunit C bindings. + * Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> + * + * Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + * license at the users choice. A copy of both licenses are available in the + * project source as Apache-2.0 and BSD. You may not use this file except in + * compliance with one of these two licences. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under these licenses is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license you chose for the specific language governing permissions + * and limitations under that license. + **/ + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <check.h> + +#include "subunit/child.h" + +/** + * Helper function to capture stdout, run some call, and check what + * was written. + * @expected the expected stdout content + * @function the function to call. + **/ +static void +test_stdout_function(char const * expected, + void (*function)(void)) +{ + /* test that the start function emits a correct test: line. */ + int bytecount; + int old_stdout; + int new_stdout[2]; + char buffer[100]; + /* we need a socketpair to capture stdout in */ + fail_if(pipe(new_stdout), "Failed to create a socketpair."); + /* backup stdout so we can replace it */ + old_stdout = dup(1); + if (old_stdout == -1) { + close(new_stdout[0]); + close(new_stdout[1]); + fail("Failed to backup stdout before replacing."); + } + /* redirect stdout so we can analyse it */ + if (dup2(new_stdout[1], 1) != 1) { + close(old_stdout); + close(new_stdout[0]); + close(new_stdout[1]); + fail("Failed to redirect stdout"); + } + /* yes this can block. Its a test case with < 100 bytes of output. + * DEAL. + */ + function(); + /* restore stdout now */ + if (dup2(old_stdout, 1) != 1) { + close(old_stdout); + close(new_stdout[0]); + close(new_stdout[1]); + fail("Failed to restore stdout"); + } + /* and we dont need the write side any more */ + if (close(new_stdout[1])) { + close(new_stdout[0]); + fail("Failed to close write side of socketpair."); + } + /* get the output */ + bytecount = read(new_stdout[0], buffer, 100); + if (0 > bytecount) { + close(new_stdout[0]); + fail("Failed to read captured output."); + } + buffer[bytecount]='\0'; + /* and we dont need the read side any more */ + fail_if(close(new_stdout[0]), "Failed to close write side of socketpair."); + /* compare with expected outcome */ + fail_if(strcmp(expected, buffer), "Did not get expected output [%s], got [%s]", expected, buffer); +} + + +static void +call_test_start(void) +{ + subunit_test_start("test case"); +} + + +START_TEST (test_start) +{ + test_stdout_function("test: test case\n", call_test_start); +} +END_TEST + + +static void +call_test_pass(void) +{ + subunit_test_pass("test case"); +} + + +START_TEST (test_pass) +{ + test_stdout_function("success: test case\n", call_test_pass); +} +END_TEST + + +static void +call_test_fail(void) +{ + subunit_test_fail("test case", "Multiple lines\n of error\n"); +} + + +START_TEST (test_fail) +{ + test_stdout_function("failure: test case [\n" + "Multiple lines\n" + " of error\n" + "]\n", + call_test_fail); +} +END_TEST + + +static void +call_test_error(void) +{ + subunit_test_error("test case", "Multiple lines\n of output\n"); +} + + +START_TEST (test_error) +{ + test_stdout_function("error: test case [\n" + "Multiple lines\n" + " of output\n" + "]\n", + call_test_error); +} +END_TEST + + +static void +call_test_skip(void) +{ + subunit_test_skip("test case", "Multiple lines\n of output\n"); +} + + +START_TEST (test_skip) +{ + test_stdout_function("skip: test case [\n" + "Multiple lines\n" + " of output\n" + "]\n", + call_test_skip); +} +END_TEST + +static Suite * +child_suite(void) +{ + Suite *s = suite_create("subunit_child"); + TCase *tc_core = tcase_create("Core"); + suite_add_tcase (s, tc_core); + tcase_add_test (tc_core, test_start); + tcase_add_test (tc_core, test_pass); + tcase_add_test (tc_core, test_fail); + tcase_add_test (tc_core, test_error); + tcase_add_test (tc_core, test_skip); + return s; +} + + +int +main(void) +{ + int nf; + Suite *s = child_suite(); + SRunner *sr = srunner_create(s); + srunner_run_all(sr, CK_NORMAL); + nf = srunner_ntests_failed(sr); + srunner_free(sr); + return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/lib/subunit/configure.ac b/lib/subunit/configure.ac new file mode 100644 index 0000000000..496aea5719 --- /dev/null +++ b/lib/subunit/configure.ac @@ -0,0 +1,75 @@ +m4_define([SUBUNIT_MAJOR_VERSION], [0]) +m4_define([SUBUNIT_MINOR_VERSION], [0]) +m4_define([SUBUNIT_MICRO_VERSION], [5]) +m4_define([SUBUNIT_VERSION], +m4_defn([SUBUNIT_MAJOR_VERSION]).m4_defn([SUBUNIT_MINOR_VERSION]).m4_defn([SUBUNIT_MICRO_VERSION])) +AC_PREREQ([2.59]) +AC_INIT([subunit], [SUBUNIT_VERSION], [subunit-dev@lists.launchpad.net]) +AC_CONFIG_SRCDIR([c/lib/child.c]) +AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) +AC_CONFIG_MACRO_DIR([m4]) +[SUBUNIT_MAJOR_VERSION]=SUBUNIT_MAJOR_VERSION +[SUBUNIT_MINOR_VERSION]=SUBUNIT_MINOR_VERSION +[SUBUNIT_MICRO_VERSION]=SUBUNIT_MICRO_VERSION +[SUBUNIT_VERSION]=SUBUNIT_VERSION +AC_SUBST([SUBUNIT_MAJOR_VERSION]) +AC_SUBST([SUBUNIT_MINOR_VERSION]) +AC_SUBST([SUBUNIT_MICRO_VERSION]) +AC_SUBST([SUBUNIT_VERSION]) +AC_USE_SYSTEM_EXTENSIONS +AC_PROG_CC +AC_PROG_CXX +AM_PROG_CC_C_O +AC_PROG_INSTALL +AC_PROG_LN_S +AC_PROG_LIBTOOL +AM_PATH_PYTHON + +AS_IF([test "$GCC" = "yes"], + [ + SUBUNIT_CFLAGS="-Wall -Werror -Wextra -Wstrict-prototypes " + SUBUNIT_CFLAGS="$SUBUNIT_CFLAGS -Wmissing-prototypes -Wwrite-strings " + SUBUNIT_CFLAGS="$SUBUNIT_CFLAGS -Wno-variadic-macros " + SUBUNIT_CXXFLAGS="-Wall -Werror -Wextra -Wwrite-strings -Wno-variadic-macros" + ]) + +AM_CFLAGS="$SUBUNIT_CFLAGS -I\$(top_srcdir)/c/include" +AM_CXXFLAGS="$SUBUNIT_CXXFLAGS -I\$(top_srcdir)/c/include" +AC_SUBST(AM_CFLAGS) +AC_SUBST(AM_CXXFLAGS) + +# Checks for libraries. + +# Checks for header files. +AC_CHECK_HEADERS([stdlib.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST +AC_TYPE_PID_T +AC_TYPE_SIZE_T +AC_HEADER_TIME +AC_STRUCT_TM + +AC_CHECK_SIZEOF(int, 4) +AC_CHECK_SIZEOF(short, 2) +AC_CHECK_SIZEOF(long, 4) + +# Checks for library functions. +AC_FUNC_MALLOC +AC_FUNC_REALLOC + +# Easier memory management. +# C unit testing. +PKG_CHECK_MODULES([CHECK], [check >= 0.9.4]) +# C++ unit testing. +PKG_CHECK_MODULES([CPPUNIT], [cppunit]) + +# Output files +AC_CONFIG_HEADERS([config.h]) + +AC_CONFIG_FILES([libsubunit.pc + libcppunit_subunit.pc + Makefile + perl/Makefile.PL + ]) +AC_OUTPUT diff --git a/lib/subunit/libcppunit_subunit.pc.in b/lib/subunit/libcppunit_subunit.pc.in new file mode 100644 index 0000000000..98982c78ae --- /dev/null +++ b/lib/subunit/libcppunit_subunit.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: cppunit subunit listener +Description: Subunit output listener for the CPPUnit test library. +URL: http://launchpad.net/subunit +Version: @VERSION@ +Libs: -L${libdir} -lsubunit +Cflags: -I${includedir} diff --git a/lib/subunit/libsubunit.pc.in b/lib/subunit/libsubunit.pc.in new file mode 100644 index 0000000000..67564148e8 --- /dev/null +++ b/lib/subunit/libsubunit.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: subunit +Description: Subunit test protocol library. +URL: http://launchpad.net/subunit +Version: @VERSION@ +Libs: -L${libdir} -lsubunit +Cflags: -I${includedir} diff --git a/lib/subunit/runtests.py b/lib/subunit/runtests.py new file mode 100755 index 0000000000..8ecc6cd3fb --- /dev/null +++ b/lib/subunit/runtests.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# -*- Mode: python -*- +# +# Copyright (C) 2004 Canonical.com +# Author: Robert Collins <robert.collins@canonical.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +import unittest +from subunit.tests.TestUtil import TestVisitor, TestSuite +import subunit +import sys +import os +import shutil +import logging + +class ParameterisableTextTestRunner(unittest.TextTestRunner): + """I am a TextTestRunner whose result class is + parameterisable without further subclassing""" + def __init__(self, **args): + unittest.TextTestRunner.__init__(self, **args) + self._resultFactory=None + def resultFactory(self, *args): + """set or retrieve the result factory""" + if args: + self._resultFactory=args[0] + return self + if self._resultFactory is None: + self._resultFactory=unittest._TextTestResult + return self._resultFactory + + def _makeResult(self): + return self.resultFactory()(self.stream, self.descriptions, self.verbosity) + + +class EarlyStoppingTextTestResult(unittest._TextTestResult): + """I am a TextTestResult that can optionally stop at the first failure + or error""" + + def addError(self, test, err): + unittest._TextTestResult.addError(self, test, err) + if self.stopOnError(): + self.stop() + + def addFailure(self, test, err): + unittest._TextTestResult.addError(self, test, err) + if self.stopOnFailure(): + self.stop() + + def stopOnError(self, *args): + """should this result indicate an abort when an error occurs? + TODO parameterise this""" + return True + + def stopOnFailure(self, *args): + """should this result indicate an abort when a failure error occurs? + TODO parameterise this""" + return True + + +def earlyStopFactory(*args, **kwargs): + """return a an early stopping text test result""" + result=EarlyStoppingTextTestResult(*args, **kwargs) + return result + + +class ShellTests(subunit.ExecTestCase): + + def test_sourcing(self): + """./shell/tests/test_source_library.sh""" + + def test_functions(self): + """./shell/tests/test_function_output.sh""" + + +def test_suite(): + result = TestSuite() + result.addTest(subunit.test_suite()) + result.addTest(ShellTests('test_sourcing')) + result.addTest(ShellTests('test_functions')) + return result + + +class filteringVisitor(TestVisitor): + """I accrue all the testCases I visit that pass a regexp filter on id + into my suite + """ + + def __init__(self, filter): + import re + TestVisitor.__init__(self) + self._suite=None + self.filter=re.compile(filter) + + def suite(self): + """answer the suite we are building""" + if self._suite is None: + self._suite=TestSuite() + return self._suite + + def visitCase(self, aCase): + if self.filter.match(aCase.id()): + self.suite().addTest(aCase) + + +def main(argv): + """To parameterise what tests are run, run this script like so: + python test_all.py REGEX + i.e. + python test_all.py .*Protocol.* + to run all tests with Protocol in their id.""" + if len(argv) > 1: + pattern = argv[1] + else: + pattern = ".*" + visitor = filteringVisitor(pattern) + test_suite().visit(visitor) + runner = ParameterisableTextTestRunner(verbosity=2) + runner.resultFactory(unittest._TextTestResult) + if not runner.run(visitor.suite()).wasSuccessful(): + return 1 + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/lib/subunit/shell/README b/lib/subunit/shell/README new file mode 100644 index 0000000000..af894a2bd3 --- /dev/null +++ b/lib/subunit/shell/README @@ -0,0 +1,62 @@ +# +# subunit shell bindings. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + +This tree contains shell bindings to the subunit protocol. They are written +entirely in shell, and unit tested in shell. See the tests/ directory for the +test scripts. You can use `make check` to run the tests. There is a trivial +python test_shell.py which uses the pyunit gui to expose the test results in a +compact form. + +The shell bindings consist of four functions which you can use to output test +metadata trivially. See share/subunit.sh for the functions and comments. + +However, this is not a full test environment, its support code for reporting to +subunit. You can look at ShUnit (http://shunit.sourceforge.net) for 'proper' +shell based xUnit functionality. There is a patch for ShUnit 1.3 +(subunit-ui.patch) in the subunit source tree. I hope to have that integrated +upstream in the near future. I will delete the copy of the patch in the subunit +tree a release or two later. + +If you are a test environment maintainer - either homegrown, or ShUnit or some +such, you will need to see how the subunit calls should be used. Here is what +a manually written test using the bindings might look like: + + +subunit_start_test "test name" +# determine if test passes or fails +result=$(something) +if [ $result == 0 ]; then + subunit_pass_test "test name" +else + subunit_fail_test "test name" <<END +Something went wrong running something: +exited with result: '$func_status' +END +fi + +Which when run with a subunit test runner will generate something like: +test name ... ok + +on success, and: + +test name ... FAIL + +====================================================================== +FAIL: test name +---------------------------------------------------------------------- +RemoteError: +Something went wrong running something: +exited with result: '1' diff --git a/lib/subunit/shell/share/subunit.sh b/lib/subunit/shell/share/subunit.sh new file mode 100644 index 0000000000..82737276b8 --- /dev/null +++ b/lib/subunit/shell/share/subunit.sh @@ -0,0 +1,56 @@ +# +# subunit.sh: shell functions to report test status via the subunit protocol. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + +subunit_start_test () { + # emit the current protocol start-marker for test $1 + echo "test: $1" +} + + +subunit_pass_test () { + # emit the current protocol test passed marker for test $1 + echo "success: $1" +} + + +subunit_fail_test () { + # emit the current protocol fail-marker for test $1, and emit stdin as + # the error text. + # we use stdin because the failure message can be arbitrarily long, and this + # makes it convenient to write in scripts (using <<END syntax. + echo "failure: $1 [" + cat - + echo "]" +} + + +subunit_error_test () { + # emit the current protocol error-marker for test $1, and emit stdin as + # the error text. + # we use stdin because the failure message can be arbitrarily long, and this + # makes it convenient to write in scripts (using <<END syntax. + echo "error: $1 [" + cat - + echo "]" +} + + +subunit_skip_test () { + # emit the current protocol test skipped marker for test $1 + echo "skip: $1" +} + + diff --git a/lib/subunit/shell/tests/test_function_output.sh b/lib/subunit/shell/tests/test_function_output.sh new file mode 100755 index 0000000000..b78eee6946 --- /dev/null +++ b/lib/subunit/shell/tests/test_function_output.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# subunit shell bindings. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + + +# this script tests the output of the methods. As each is tested we start using +# it. +# So the first test manually implements the entire protocol, the next uses the +# start method and so on. +# it is assumed that we are running from the 'shell' tree root in the source +# of subunit, and that the library sourcing tests have all passed - if they +# have not, this test script may well fail strangely. + +# import the library. +. ${SHELL_SHARE}subunit.sh + +echo 'test: subunit_start_test output' +func_output=$(subunit_start_test "foo bar") +func_status=$? +if [ $func_status == 0 -a "x$func_output" = "xtest: foo bar" ]; then + echo 'success: subunit_start_test output' +else + echo 'failure: subunit_start_test output [' + echo 'got an error code or incorrect output:' + echo "exit: $func_status" + echo "output: '$func_output'" + echo ']' ; +fi + +subunit_start_test "subunit_pass_test output" +func_output=$(subunit_pass_test "foo bar") +func_status=$? +if [ $func_status == 0 -a "x$func_output" = "xsuccess: foo bar" ]; then + subunit_pass_test "subunit_pass_test output" +else + echo 'failure: subunit_pass_test output [' + echo 'got an error code or incorrect output:' + echo "exit: $func_status" + echo "output: '$func_output'" + echo ']' ; +fi + +subunit_start_test "subunit_fail_test output" +func_output=$(subunit_fail_test "foo bar" <<END +something + wrong +here +END +) +func_status=$? +if [ $func_status == 0 -a "x$func_output" = "xfailure: foo bar [ +something + wrong +here +]" ]; then + subunit_pass_test "subunit_fail_test output" +else + echo 'failure: subunit_fail_test output [' + echo 'got an error code or incorrect output:' + echo "exit: $func_status" + echo "output: '$func_output'" + echo ']' ; +fi + +subunit_start_test "subunit_error_test output" +func_output=$(subunit_error_test "foo bar" <<END +something + died +here +END +) +func_status=$? +if [ $func_status == 0 -a "x$func_output" = "xerror: foo bar [ +something + died +here +]" ]; then + subunit_pass_test "subunit_error_test output" +else + subunit_fail_test "subunit_error_test output" <<END +got an error code or incorrect output: +exit: $func_status +output: '$func_output' +END +fi diff --git a/lib/subunit/shell/tests/test_source_library.sh b/lib/subunit/shell/tests/test_source_library.sh new file mode 100755 index 0000000000..699f1281bc --- /dev/null +++ b/lib/subunit/shell/tests/test_source_library.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# subunit shell bindings. +# Copyright (C) 2006 Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + + +# this script tests that we can source the subunit shell bindings successfully. +# It manually implements the control protocol so that it des not depend on the +# bindings being complete yet. + +# we expect to be run from the tree root. + +echo 'test: shell bindings can be sourced' +# if any output occurs, this has failed to source cleanly +source_output=$(. ${SHELL_SHARE}subunit.sh 2>&1) +if [ $? == 0 -a "x$source_output" = "x" ]; then + echo 'success: shell bindings can be sourced' +else + echo 'failure: shell bindings can be sourced [' + echo 'got an error code or output during sourcing.:' + echo $source_output + echo ']' ; +fi + +# now source it for real +. ${SHELL_SHARE}subunit.sh + +# we should have a start_test function +echo 'test: subunit_start_test exists' +found_type=$(type -t subunit_start_test) +status=$? +if [ $status == 0 -a "x$found_type" = "xfunction" ]; then + echo 'success: subunit_start_test exists' +else + echo 'failure: subunit_start_test exists [' + echo 'subunit_start_test is not a function:' + echo "type -t status: $status" + echo "output: $found_type" + echo ']' ; +fi + +# we should have a pass_test function +echo 'test: subunit_pass_test exists' +found_type=$(type -t subunit_pass_test) +status=$? +if [ $status == 0 -a "x$found_type" = "xfunction" ]; then + echo 'success: subunit_pass_test exists' +else + echo 'failure: subunit_pass_test exists [' + echo 'subunit_pass_test is not a function:' + echo "type -t status: $status" + echo "output: $found_type" + echo ']' ; +fi + +# we should have a fail_test function +echo 'test: subunit_fail_test exists' +found_type=$(type -t subunit_fail_test) +status=$? +if [ $status == 0 -a "x$found_type" = "xfunction" ]; then + echo 'success: subunit_fail_test exists' +else + echo 'failure: subunit_fail_test exists [' + echo 'subunit_fail_test is not a function:' + echo "type -t status: $status" + echo "output: $found_type" + echo ']' ; +fi + +# we should have a error_test function +echo 'test: subunit_error_test exists' +found_type=$(type -t subunit_error_test) +status=$? +if [ $status == 0 -a "x$found_type" = "xfunction" ]; then + echo 'success: subunit_error_test exists' +else + echo 'failure: subunit_error_test exists [' + echo 'subunit_error_test is not a function:' + echo "type -t status: $status" + echo "output: $found_type" + echo ']' ; +fi + +# we should have a skip_test function +echo 'test: subunit_skip_test exists' +found_type=$(type -t subunit_skip_test) +status=$? +if [ $status == 0 -a "x$found_type" = "xfunction" ]; then + echo 'success: subunit_skip_test exists' +else + echo 'failure: subunit_skip_test exists [' + echo 'subunit_skip_test is not a function:' + echo "type -t status: $status" + echo "output: $found_type" + echo ']' ; +fi + diff --git a/lib/update-external.sh b/lib/update-external.sh index ebb7bd0013..53748d8810 100755 --- a/lib/update-external.sh +++ b/lib/update-external.sh @@ -7,10 +7,7 @@ WORKDIR="`mktemp -d`" echo "Updating subunit..." bzr export "$WORKDIR/subunit" lp:subunit -for p in python/ filters/ perl/ -do - rsync -avz --delete "$WORKDIR/subunit/$p" "$TARGETDIR/subunit/$p" -done +rsync -avz --delete "$WORKDIR/subunit/" "$TARGETDIR/subunit/" echo "Updating testtools..." bzr export "$WORKDIR/testtools" lp:testtools |