simple web/html testing

[Santi]

For static html testing, I'd avoid using Selenium. Even though Selenium is *the* tool for RIA and javascript intensive environments, feels like bringing up a browser with all the coordination and resources that it takes just to crawl the website and find 404s is an overkill.

What we implemented for doing that is just a simple crawler based on urllib:

class LinkTest(MechanizeTestCase):

    def __init__(self, *args, **kwargs):
        super(LinkTest, self).__init__(*args, **kwargs)
        self.pages = ['/?']
        self.exceptions = ['/forums',
                           '/blog']

    def _page_test(self, url):
        try:
            self.get(url[1:], local=True)
        except Exception, e:
            raise Exception("Couldn't test %s - %s (%s)" % (url, e, self.exceptions))

        try:
            links = self.mbrowser.links()
        except mechanize.BrowserStateError, e:
            return

        for l in links:
            if not l.url.startswith('/'):
                continue
            if l.url in self.exceptions:
                continue

            self.pages.append(l.url)
            self.pages = list(set(self.pages))

            try:
                mechanize.urlopen(l.absolute_url)
                #Apparently this will raise with the HTTP Error code
            except Exception, e:
                raise Exception("Error with link '%s' on page '%s'" % (l.url, url))

    def test_all_links(self):
        while self.pages:
            x = self.pages.pop()
            if x not in self.exceptions:
                print "Trying %s" % x
                self._page_test(x)
                self.exceptions.append(x)
                self.exceptions = list(set(self.exceptions))


And basically, MechanizeTestCase is a couple of handy assertions as well as instantiating a mechanize instance:

class MechanizeTestCase(TestCase):
    def setUp(self, extra=None):
        self.config = load_config() or self.fail('Failed to load config.')

    def __init__(self, arg):
        super(MechanizeTestCase, self).__init__(arg)
        self.mbrowser = mechanize.Browser()

    def get(self, url, local=True):
        self.last_url = url
        if local:
            url = self.config['base-url'] + url
        self._page = self.mbrowser.open(url)
        self._content = self._page.read()

    def submitForm(self):
        self._page = self.mbrowser.submit()
        self._content = self._page.read()

    def assertWantedContent(self, content):
        self.assertTrue(content in self._content,
                        "couldn't find %s in /%s" % (content, self.last_url))

    def assertUrl(self, url):
        self.assertEqual(self.config['base-url'] + url, self._page.geturl(),
                "wrong url expected: %s, received: %s, content: %s" % (url, self._page.geturl(), self._content))



Hope this helps