diff options
| -rw-r--r-- | misc/testsuite/catalog.xml | 5 | ||||
| -rwxr-xr-x | misc/testsuite/harness.py | 30 | ||||
| -rw-r--r-- | misc/testsuite/lib/Documentation.py | 81 | ||||
| -rw-r--r-- | misc/testsuite/lib/TestCase.py | 8 | ||||
| -rw-r--r-- | misc/testsuite/templates/index.html | 170 | ||||
| -rw-r--r-- | misc/testsuite/templates/list.html | 3 | ||||
| -rw-r--r-- | misc/testsuite/templates/test.html | 1 |
7 files changed, 285 insertions, 13 deletions
diff --git a/misc/testsuite/catalog.xml b/misc/testsuite/catalog.xml index f49c9ec..c05dc12 100644 --- a/misc/testsuite/catalog.xml +++ b/misc/testsuite/catalog.xml @@ -858,7 +858,9 @@ </test> </collection> -<collection id="encodings"> +<collection id="encoding"> + <dc:title>Document Encoding Tests</dc:title> + <collection id="encodings"> <dc:title>Various Encodings</dc:title> <test> <uri>http://www.w3.org/Press/1998/XSL-WD.html.ja</uri> @@ -937,6 +939,7 @@ </expect> </test> </collection> +</collection> <collection id="errors"> <dc:title>Error conditions & strange cases</dc:title> diff --git a/misc/testsuite/harness.py b/misc/testsuite/harness.py index 0422132..399fe41 100755 --- a/misc/testsuite/harness.py +++ b/misc/testsuite/harness.py @@ -20,7 +20,7 @@ basedir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(basedir, "lib")) from ValidatorsAPIs import W3CValidatorHTTP, W3CValidatorHTTP_test from TestCase import ValidatorTestCase, ValidatorTestSuite, ValidatorTestCase_UT, ValidatorTestSuite_UT -#from Documentation import Documentation +from Documentation import Documentation help_message = ''' Run or Generate test suite for markup validator(s) @@ -41,7 +41,9 @@ Usage: harness.py [options] [run|sanity|doc] run: run the test suite sanity: check that this code is still working useful after using test cases or modifying code - @@ TBD @@ doc: generate an HTML index of the test cases + list: list the available test collections + doc: generate an HTML index of the test cases + (to be saved in ../htdocs/dev/tests/index.html) ''' class Usage(Exception): @@ -112,6 +114,10 @@ def main(argv=None): suite4 = unittest.TestLoader().loadTestsFromTestCase(W3CValidatorHTTP_test) suite = unittest.TestSuite([suite1, suite2, suite3, suite4]) unittest.TextTestRunner(verbosity=verbose).run(suite) + + elif args[0] == "list": + listCollections() + elif args[0] == "doc": generateIndex() @@ -143,14 +149,18 @@ def runTestSuite(testsuite, verbose): if collection.countTestCases() > 0: unittest.TextTestRunner(verbosity=verbose).run(collection) -# def generateIndex(): -# index = Documentation('index') -# for testcollection_file in (glob.glob(os.path.join(basedir, 'testcases', '**', '*.collection'))): -# colldir = os.path.dirname(os.path.abspath(testcollection_file)) -# colldir = os.path.split(colldir)[-1] -# testcollection = readCollectionMeta(testcollection_file) -# index.addCollection(testcollection) -# print index.generate(template_path=os.path.join(basedir, "templates")).encode('utf-8') +def listCollections(): + index = Documentation('list') + test_suite = buildTestSuite() + index.addTestSuite(test_suite) + print index.generate(template_path=os.path.join(basedir, "templates")).encode('utf-8') + + +def generateIndex(): + index = Documentation('index') + test_suite = buildTestSuite() + index.addTestSuite(test_suite) + print index.generate(template_path=os.path.join(basedir, "templates")).encode('utf-8') diff --git a/misc/testsuite/lib/Documentation.py b/misc/testsuite/lib/Documentation.py new file mode 100644 index 0000000..35e3f64 --- /dev/null +++ b/misc/testsuite/lib/Documentation.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +Documentation.py +Generate Link Test Suite index and documentation + +Created by olivier Thereaux on 2008-01-30. +Copyright (c) 2008 W3C. Licensed under the W3C Software License +http://www.w3.org/Consortium/Legal/copyright-software +""" + +import sys +import os +import unittest +import TestCase +import datetime + +class Documentation: + def __init__(self, type=None): + ok_types = ['test', 'list', 'index'] + if type in ok_types: + self.type = type + else: + self.type = 'index' + self.test_suite = list() + + def addTestSuite(self, test_suite): + if isinstance(test_suite, TestCase.ValidatorTestSuite) == 0: + raise TypeError("not a proper test suite") + else: + self.test_suite = test_suite + + def generate(self, template_path=None): + """pass through template engine""" + import jinja2,datetime + if template_path == None: + template_path= os.path.abspath('..') + template_path=os.path.join(template_path, "templates") + template_engine = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path)) + template = template_engine.get_template(os.path.join(self.type+'.html')) + return template.render(test_suite=self.test_suite, + year=str(datetime.date.today().year)) + +class DocumentationTests(unittest.TestCase): + def test_has_jinja2(self): + try: + import jinja2 + except ImportError: + self.fail("you need to install the jinja2 templating system -- http://jinja.pocoo.org/2/") + + def test_init(self): + """initialize a Documentation Generator""" + generator = Documentation() + self.assertEquals(generator.type, 'index') + + def test_init_default_fallback(self): + """initialize a Documentation Generator with a bogus type""" + generator = Documentation(type="grut") + self.assertEquals(generator.type, 'index') + + def test_addTestSuite(self): + """add empty test suite""" + generator = Documentation() + test_suite = TestCase.ValidatorTestSuite() + generator.addTestSuite(test_suite) + self.assertEquals(len(generator.test_suite.collections), 0) + + def test_bogusSuite(self): + """add bogus test collection and raise error""" + generator = Documentation() + test_suite = 5 + self.assertRaises(TypeError, Documentation.addTestSuite, generator, test_suite) + + def test_generate_test(self): + """generate collections index without any collection""" + generator = Documentation('test') + self.assertEqual(generator.generate(), str(datetime.date.today().year)) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/misc/testsuite/lib/TestCase.py b/misc/testsuite/lib/TestCase.py index 4170251..15af48e 100644 --- a/misc/testsuite/lib/TestCase.py +++ b/misc/testsuite/lib/TestCase.py @@ -87,10 +87,13 @@ class ValidatorTestSuite: raise v return ts_node - def readTestCollection(self, collection_node): + def readTestCollection(self, collection_node, level=None): """read a test collection from a given ElementTree collection node The collection can have test cases as children (which will be read and returned)""" + if level == None: + level = 0 testcollection = ValidatorTestCollection() + testcollection.level = level if collection_node.attrib.has_key("id"): testcollection.collection_id = collection_node.attrib["id"] else: @@ -110,7 +113,7 @@ class ValidatorTestSuite: self.collections.append(testcollection) for child_node in collection_node.getchildren(): if child_node.tag == "collection": - self.readTestCollection(child_node) + self.readTestCollection(child_node, level=level+1) def readTestCase(self, testcase_node): @@ -137,6 +140,7 @@ class ValidatorTestCollection(unittest.TestSuite): def __init__(self): super(ValidatorTestCollection, self).__init__() self.collection_id = None + self.level = None self.title = None diff --git a/misc/testsuite/templates/index.html b/misc/testsuite/templates/index.html new file mode 100644 index 0000000..2df9ee2 --- /dev/null +++ b/misc/testsuite/templates/index.html @@ -0,0 +1,170 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html lang="en"> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>Link Check Test Suite Index</title> + <meta name="author" content="olivier Thereaux"> + <style type="text/css" media="all"> + @import "http://validator.w3.org/style/base.css"; + </style> + +</head> +<body> +<div id="banner"> + <h1 id="title"> + <a href="http://www.w3.org/"><img alt="W3C" width="110" height="61" id="logo" src="http://validator.w3.org/images/w3c.png" /></a> + <a href="./"><span>Markup Validator Test Suite</span></a> + </h1> + <p id="tagline">Collection of test cases for W3C Markup Validator</p> +</div> +<div class="doc"> + <h2>Test Suite Index</h2> + <p id="skip"> + This page lists a few test documents and test scenarios for the W3C Markup Validator + make sure everything works fine after making changes to + <a href="../../">the validator</a>. If you can think of interesting + test cases that aren't on this page, + <a href="../../feedback.html">let us know</a>. + </p> +<p> + This test suite is work in progress. Please submit any issue or idea to the + public mailing-list <a href="http://lists.w3.org/Archives/Public/public-qa-dev/" title="public-qa-dev@w3.org Mail Archives">public-qa-dev@w3.org</a> +</p> + +<div id="toc"> +<ol> +{% for collection in test_suite.collections %} +{% if collection.level is sameas 1 %} +<li><a href="#{{collection.collection_id}}">{{collection.title}}</a></li> +{% endif %} +{% endfor %} +<li><a href="#nonauto">Non-Automated tests</a></li> +</ol> +</div> + +{% for collection in test_suite.collections %} +<h{{collection.level +2}} id="{{collection.collection_id}}">{{collection.title}}</h{{collection.level +2}}> +<p>{{collection.description}}</p> + {% for case in collection._tests %} + <h{{collection.level +3}}>{{case.title}}</h{{collection.level +3}}> + <p> + <a href="../../check?uri={{case.docURI}};ss">validate</a> - + <a href="http://validator.w3.org/check?uri={{case.docURI}};ss">with v.w.o</a> - + <a href="{{case.docURI}}">view</a>: + {{case.description}}</p> + + {% endfor %} +{% endfor %} + + <h3 id="nonauto">Non-Automated tests</h3> + <p>The following test collections are not managed by the automated test suite, and should be + tested by hand on a regular basis</p> + <h4 id="ui">User Interface tests</h4> + <p>The following tests do not test the binary "validity" results, but the proper handling of options, errors, various outputs, etc. + "validate" links use this instance, with validator.w3.org links added for reference comparison</p> +<ul> + <li><a href="../../check?uri=file%3A///etc/passwd">validate</a> - + <a href="http://validator.w3.org/check?uri=file%3A///etc/passwd">with v.w.o</a> - + a file: URL </li> + <li> + <a href="../../check/referer">validate</a> - + <a href="http://validator.w3.org/check/referer">with v.w.o</a> - + /check/referer (deprecated) + </li> + <li> + <a href="../../check/referer;verbose=1">validate</a> - + <a href="http://validator.w3.org/check/referer;verbose=1">with v.w.o</a> - + /check/referer;verbose=1 (unsupported) + </li> + <li> + <a href="../../check/referer;verbose=1;No200=1">validate</a> - + <a href="http://validator.w3.org/check/referer;verbose=1;No200=1">with v.w.o</a> - + /check/referer;verbose=1;No200=1 (unsupported) + </li> + <li> + <a href="../../check?uri=referer">validate</a> - + <a href="http://validator.w3.org/check?uri=referer">with v.w.o</a> - + /check?uri=referer (supported, should validate this test collection) + </li> + <li> + <a href="../../check?uri=referer;verbose=1">validate</a> - + <a href="http://validator.w3.org/check?uri=referer;verbose=1">with v.w.o</a> - + /check?uri=referer;verbose=1 (supported, should validate this test collection) + </li> + <li> + <a href="../../check?uri=referer&verbose=1">validate</a> - + <a href="http://validator.w3.org/check?uri=refererr&verbose=1">with v.w.o</a> - + /check?uri=refererr&verbose=1 (supported, should validate this test collection) + </li> + <li> + <a href="../../check?uri=referer;verbose=1;No200=1">validate</a> - + <a href="http://validator.w3.org/check?uri=referer;verbose=1;No200=1">with v.w.o</a> - + /check?uri=referer;verbose=1;No200=1 (supported, should validate this test collection) + </li> + <li> + <a href="../../check?uri=http://www.w3.org/MarkUp">validate</a> - + <a href="http://validator.w3.org/check?uri=http://www.w3.org/MarkUp">with v.w.o</a> - + Directory Redirect (should mention the redirect and validate the redirection target + </li> + <li> + <a href="../../check">validate</a> - + <a href="http://validator.w3.org/check">with v.w.o</a> - + script invoked with no parameters (gives an error) + </li> + <li> + <a href="../../check?uri=">validate</a> - + <a href="http://validator.w3.org/check?uri=">with v.w.o</a> - + uri parameter is empty + </li> + <li> + <a href="../../check?uri=www.w3.org/">validate</a> - + <a href="http://validator.w3.org/check?uri=www.w3.org/">with v.w.o</a> - + uri is missing http:// (silent fix) + </li> + <li> + <a href="../../check?uri=http://www.w3.org/bogus">validate</a> - + <a href="http://validator.w3.org/check?uri=http://www.w3.org/bogus">with v.w.o</a> - + 404 not found + </li> + <li> + <a href="../../check?uri=http://www.w3.org/bogus;No200=1">validate</a> - + <a href="http://validator.w3.org/check?uri=http://www.w3.org/bogus;no200=1">with v.w.o</a> - + 404 not found, with "validate error pages" ON + </li> + <li> + <a href="../../check?uri=http://validator.w3.org/sgml-lib/sgml.soc">validate</a> - + <a href="http://validator.w3.org/check?uri=http://validator.w3.org/sgml-lib/sgml.soc">with v.w.o</a> - + text/plain resource (refuses to validate) + </li> + <li> + <a href="../../check?uri=http://validator.w3.org:8000/">validate</a> - + <a href="http://validator.w3.org/check?uri=http://validator.w3.org:8000/">with v.w.o</a> - + httpd on a non-standard port (should be working transparently if :8000 server is running) + </li> + <li><a href="../../check?uri=http://www.oreillynet.com/pub/feed/20">validate</a> - + a resource served as application/atom+xml, should be passed on to the Feed Validator.</li> + <li><a href="../../check?uri=http://www.w3.org/StyleSheets/Core/Swiss">validate</a> - + a resource served as text/css, should be passed on to the CSS Validator.</li> +</ul> + + <h4 id="regress">Bugs and Regression Tests</h4> + <p>These should be tied to a bug report on either mailing-list and/or bugzilla whenever possible.</p> + + <ul> + + <li><a href="../../check?uri=http://qa-dev.w3.org/wmvs/HEAD/dev/tests/long-lines.html">Long Lines</a> (cause a core)</li> + <li><a href="../../check?uri=http://qa-dev.w3.org/wmvs/HEAD/dev/tests/trailing-nul.html">A trailing ASCII NUL character</a> (trips up HTML::Parser (not really))</li> + <li><a href="../../check?uri=http://qa-dev.w3.org/wmvs/HEAD/dev/tests/colons.html">Undeclared namespace prefixes</a> (colons ":" in element names (trips up parsing of onsgmls output))</li> + <li> + <a href="../../check?uri=http://qa-dev.w3.org/wmvs/HEAD/dev/tests/857-svg_nodoctype.svg;doctype=SVG+1.1;ss=1">validate</a> - + <a href="http://validator.w3.org/check?uri=qa-dev.w3.org/wmvs/HEAD/dev/tests/857-svg_nodoctype.svg;doctype=SVG+1.1;ss=1">with v.w.o</a> - <a href="857-svg_nodoctype.svg">view</a>: + Test of doctype override for SVG document with no doctype, but an XML decl. The doctype should sneak between + the XML decl and the root svg start tag. (<a href="http://www.w3.org/Bugs/Public/show_bug.cgi?id=857" title="Bug 857 - DOCTYPE Override should take XML Decl into account."> 857 - DOCTYPE Override</a>) + </li> + + + </ul> +</div><!-- doc --> +<!--#include virtual="../../footer.html" --> + </body> +</html> diff --git a/misc/testsuite/templates/list.html b/misc/testsuite/templates/list.html new file mode 100644 index 0000000..1785fc0 --- /dev/null +++ b/misc/testsuite/templates/list.html @@ -0,0 +1,3 @@ +List of test collection (id: description) +{% for collection in test_suite.collections %} +#{{collection.collection_id}}: {{collection.title}} {% endfor %} diff --git a/misc/testsuite/templates/test.html b/misc/testsuite/templates/test.html new file mode 100644 index 0000000..7685f56 --- /dev/null +++ b/misc/testsuite/templates/test.html @@ -0,0 +1 @@ +{{year}} |