1##############################################################################
2#
3# Copyright (c) 2005 Zope Foundation and Contributors.
4# All Rights Reserved.
5#
6# This software is subject to the provisions of the Zope Public License,
7# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
8# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11# FOR A PARTICULAR PURPOSE.
12#
13##############################################################################
14"""Webtest-based Functional Doctest interfaces
15"""
16
17import re
18import time
19import io
20from contextlib import contextmanager
21
22from six.moves import urllib_robotparser
23from six import string_types
24
25from zope.interface import implementer
26from zope.cachedescriptors.property import Lazy
27from wsgiproxy.proxies import TransparentProxy
28from bs4 import BeautifulSoup
29from soupsieve import escape as css_escape
30
31from zope.testbrowser import interfaces
32from zope.testbrowser._compat import httpclient, PYTHON2
33from zope.testbrowser._compat import urllib_request, urlparse
34import zope.testbrowser.cookies
35
36import webtest
37
38__docformat__ = "reStructuredText"
39
40HTTPError = urllib_request.HTTPError
41RegexType = type(re.compile(''))
42_compress_re = re.compile(r"\s+")
43
44
45class HostNotAllowed(Exception):
46    pass
47
48
49class RobotExclusionError(HTTPError):
50    def __init__(self, *args):
51        super(RobotExclusionError, self).__init__(*args)
52
53
54# RFC 2606
55_allowed_2nd_level = set(['example.com', 'example.net', 'example.org'])
56
57_allowed = set(['localhost', '127.0.0.1'])
58_allowed.update(_allowed_2nd_level)
59
60REDIRECTS = (301, 302, 303, 307)
61
62
63class TestbrowserApp(webtest.TestApp):
64    _last_fragment = ""
65    restricted = False
66
67    def _assertAllowed(self, url):
68        parsed = urlparse.urlparse(url)
69        if self.restricted:
70            # We are in restricted mode, check host part only
71            host = parsed.netloc.partition(':')[0]
72            if host in _allowed:
73                return
74            for dom in _allowed_2nd_level:
75                if host.endswith('.%s' % dom):
76                    return
77
78            raise HostNotAllowed(url)
79        else:
80            # Unrestricted mode: retrieve robots.txt and check against it
81            robotsurl = urlparse.urlunsplit((parsed.scheme, parsed.netloc,
82                                             '/robots.txt', '', ''))
83            rp = urllib_robotparser.RobotFileParser()
84            rp.set_url(robotsurl)
85            rp.read()
86            if not rp.can_fetch("*", url):
87                msg = "request disallowed by robots.txt"
88                raise RobotExclusionError(url, 403, msg, [], None)
89
90    def do_request(self, req, status, expect_errors):
91        self._assertAllowed(req.url)
92
93        response = super(TestbrowserApp, self).do_request(req, status,
94                                                          expect_errors)
95        # Store _last_fragment in response to preserve fragment for history
96        # (goBack() will not lose fragment).
97        response._last_fragment = self._last_fragment
98        return response
99
100    def _remove_fragment(self, url):
101        # HACK: we need to preserve fragment part of url, but webtest strips it
102        # from url on every request. So we override this protected method,
103        # assuming it is called on every request and therefore _last_fragment
104        # will not get outdated. ``getRequestUrlWithFragment()`` will
105        # reconstruct url with fragment for the last request.
106        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
107        self._last_fragment = fragment
108        return super(TestbrowserApp, self)._remove_fragment(url)
109
110    def getRequestUrlWithFragment(self, response):
111        url = response.request.url
112        if not self._last_fragment:
113            return url
114        return "%s#%s" % (url, response._last_fragment)
115
116
117class SetattrErrorsMixin(object):
118    _enable_setattr_errors = False
119
120    def __setattr__(self, name, value):
121        if self._enable_setattr_errors:
122            # cause an attribute error if the attribute doesn't already exist
123            getattr(self, name)
124
125        # set the value
126        object.__setattr__(self, name, value)
127
128
129@implementer(interfaces.IBrowser)
130class Browser(SetattrErrorsMixin):
131    """A web user agent."""
132
133    _contents = None
134    _controls = None
135    _counter = 0
136    _response = None
137    _req_headers = None
138    _req_content_type = None
139    _req_referrer = None
140    _history = None
141    __html = None
142
143    def __init__(self, url=None, wsgi_app=None):
144        self.timer = Timer()
145        self.raiseHttpErrors = True
146        self.handleErrors = True
147        self.followRedirects = True
148
149        if wsgi_app is None:
150            self.testapp = TestbrowserApp(TransparentProxy())
151        else:
152            self.testapp = TestbrowserApp(wsgi_app)
153            self.testapp.restricted = True
154
155        self._req_headers = {}
156        self._history = History()
157        self._enable_setattr_errors = True
158        self._controls = {}
159
160        if url is not None:
161            self.open(url)
162
163    @property
164    def url(self):
165        """See zope.testbrowser.interfaces.IBrowser"""
166        if self._response is None:
167            return None
168        return self.testapp.getRequestUrlWithFragment(self._response)
169
170    @property
171    def isHtml(self):
172        """See zope.testbrowser.interfaces.IBrowser"""
173        return self._response and 'html' in self._response.content_type
174
175    @property
176    def lastRequestSeconds(self):
177        """See zope.testbrowser.interfaces.IBrowser"""
178        return self.timer.elapsedSeconds
179
180    @property
181    def title(self):
182        """See zope.testbrowser.interfaces.IBrowser"""
183        if not self.isHtml:
184            raise BrowserStateError('not viewing HTML')
185
186        titles = self._html.find_all('title')
187        if not titles:
188            return None
189        return self.toStr(titles[0].text)
190
191    def reload(self):
192        """See zope.testbrowser.interfaces.IBrowser"""
193        if self._response is None:
194            raise BrowserStateError("no URL has yet been .open()ed")
195
196        def make_request(args):
197            return self.testapp.request(self._response.request)
198
199        # _req_referrer is left intact, so will be the referrer (if any) of
200        # the request being reloaded.
201        self._processRequest(self.url, make_request)
202
203    def goBack(self, count=1):
204        """See zope.testbrowser.interfaces.IBrowser"""
205        resp = self._history.back(count, self._response)
206        self._setResponse(resp)
207
208    @property
209    def contents(self):
210        """See zope.testbrowser.interfaces.IBrowser"""
211        if self._response is not None:
212            return self.toStr(self._response.body)
213        else:
214            return None
215
216    @property
217    def headers(self):
218        """See zope.testbrowser.interfaces.IBrowser"""
219        resptxt = []
220        resptxt.append('Status: %s' % self._response.status)
221        for h, v in sorted(self._response.headers.items()):
222            resptxt.append(str("%s: %s" % (h, v)))
223
224        inp = '\n'.join(resptxt)
225        stream = io.BytesIO(inp.encode('latin1'))
226        if PYTHON2:
227            return httpclient.HTTPMessage(stream)
228        else:
229            return httpclient.parse_headers(stream)
230
231    @property
232    def cookies(self):
233        if self.url is None:
234            raise RuntimeError("no request found")
235        return zope.testbrowser.cookies.Cookies(self.testapp, self.url,
236                                                self._req_headers)
237
238    def addHeader(self, key, value):
239        """See zope.testbrowser.interfaces.IBrowser"""
240        if (self.url and key.lower() in ('cookie', 'cookie2') and
241                self.cookies.header):
242            raise ValueError('cookies are already set in `cookies` attribute')
243        self._req_headers[key] = value
244
245    def open(self, url, data=None, referrer=None):
246        """See zope.testbrowser.interfaces.IBrowser"""
247        url = self._absoluteUrl(url)
248        if data is not None:
249            def make_request(args):
250                return self.testapp.post(url, data, **args)
251        else:
252            def make_request(args):
253                return self.testapp.get(url, **args)
254
255        self._req_referrer = referrer
256        self._processRequest(url, make_request)
257
258    def post(self, url, data, content_type=None, referrer=None):
259        if content_type is not None:
260            self._req_content_type = content_type
261        self._req_referrer = referrer
262        return self.open(url, data)
263
264    def _clickSubmit(self, form, control=None, coord=None):
265        # find index of given control in the form
266        url = self._absoluteUrl(form.action)
267        if control:
268            def make_request(args):
269                index = form.fields[control.name].index(control)
270                return self._submit(
271                    form, control.name, index, coord=coord, **args)
272        else:
273            def make_request(args):
274                return self._submit(form, coord=coord, **args)
275
276        self._req_referrer = self.url
277        self._processRequest(url, make_request)
278
279    def _processRequest(self, url, make_request):
280        with self._preparedRequest(url) as reqargs:
281            self._history.add(self._response)
282            resp = make_request(reqargs)
283            if self.followRedirects:
284                remaining_redirects = 100  # infinite loops protection
285                while resp.status_int in REDIRECTS and remaining_redirects:
286                    remaining_redirects -= 1
287                    self._req_referrer = url
288                    url = urlparse.urljoin(url, resp.headers['location'])
289                    with self._preparedRequest(url) as reqargs:
290                        resp = self.testapp.get(url, **reqargs)
291                assert remaining_redirects > 0, (
292                    "redirects chain looks infinite")
293            self._setResponse(resp)
294            self._checkStatus()
295
296    def _checkStatus(self):
297        # if the headers don't have a status, I suppose there can't be an error
298        if 'Status' in self.headers:
299            code, msg = self.headers['Status'].split(' ', 1)
300            code = int(code)
301            if self.raiseHttpErrors and code >= 400:
302                raise HTTPError(self.url, code, msg, [], None)
303
304    def _submit(self, form, name=None, index=None, coord=None, **args):
305        # A reimplementation of webtest.forms.Form.submit() to allow to insert
306        # coords into the request
307        fields = form.submit_fields(name, index=index)
308        if coord is not None:
309            fields.extend([('%s.x' % name, coord[0]),
310                           ('%s.y' % name, coord[1])])
311
312        url = self._absoluteUrl(form.action)
313        if form.method.upper() != "GET":
314            args.setdefault("content_type", form.enctype)
315        else:
316            parsed = urlparse.urlparse(url)._replace(query='', fragment='')
317            url = urlparse.urlunparse(parsed)
318        return form.response.goto(url, method=form.method,
319                                  params=fields, **args)
320
321    def _setResponse(self, response):
322        self._response = response
323        self._changed()
324
325    def getLink(self, text=None, url=None, id=None, index=0):
326        """See zope.testbrowser.interfaces.IBrowser"""
327        qa = 'a' if id is None else 'a#%s' % css_escape(id)
328        qarea = 'area' if id is None else 'area#%s' % css_escape(id)
329        html = self._html
330        links = html.select(qa)
331        links.extend(html.select(qarea))
332
333        matching = []
334        for elem in links:
335            matches = (isMatching(elem.text, text) and
336                       isMatching(elem.get('href', ''), url))
337
338            if matches:
339                matching.append(elem)
340
341        if index >= len(matching):
342            raise LinkNotFoundError()
343        elem = matching[index]
344
345        baseurl = self._getBaseUrl()
346
347        return Link(elem, self, baseurl)
348
349    def follow(self, *args, **kw):
350        """Select a link and follow it."""
351        self.getLink(*args, **kw).click()
352
353    def _getBaseUrl(self):
354        # Look for <base href> tag and use it as base, if it exists
355        url = self._response.request.url
356        if b"<base" not in self._response.body:
357            return url
358
359        # we suspect there is a base tag in body, try to find href there
360        html = self._html
361        if not html.head:
362            return url
363        base = html.head.base
364        if not base:
365            return url
366        return base['href'] or url
367
368    def getForm(self, id=None, name=None, action=None, index=None):
369        """See zope.testbrowser.interfaces.IBrowser"""
370        zeroOrOne([id, name, action], '"id", "name", and "action"')
371        matching_forms = []
372        allforms = self._getAllResponseForms()
373        for form in allforms:
374            if ((id is not None and form.id == id) or
375                (name is not None and form.html.form.get('name') == name) or
376                (action is not None and re.search(action, form.action)) or
377                    id == name == action is None):
378                matching_forms.append(form)
379
380        if index is None and not any([id, name, action]):
381            if len(matching_forms) == 1:
382                index = 0
383            else:
384                raise ValueError(
385                    'if no other arguments are given, index is required.')
386
387        form = disambiguate(matching_forms, '', index)
388        return Form(self, form)
389
390    def getControl(self, label=None, name=None, index=None):
391        """See zope.testbrowser.interfaces.IBrowser"""
392        intermediate, msg, available = self._getAllControls(
393            label, name, self._getAllResponseForms(),
394            include_subcontrols=True)
395        control = disambiguate(intermediate, msg, index,
396                               controlFormTupleRepr,
397                               available)
398        return control
399
400    def _getAllResponseForms(self):
401        """ Return set of response forms in the order they appear in
402        ``self._response.form``."""
403        respforms = self._response.forms
404        idxkeys = [k for k in respforms.keys() if isinstance(k, int)]
405        return [respforms[k] for k in sorted(idxkeys)]
406
407    def _getAllControls(self, label, name, forms, include_subcontrols=False):
408        onlyOne([label, name], '"label" and "name"')
409
410        # might be an iterator, and we need to iterate twice
411        forms = list(forms)
412
413        available = None
414        if label is not None:
415            res = self._findByLabel(label, forms, include_subcontrols)
416            msg = 'label %r' % label
417        elif name is not None:
418            include_subcontrols = False
419            res = self._findByName(name, forms)
420            msg = 'name %r' % name
421        if not res:
422            available = list(self._findAllControls(forms, include_subcontrols))
423        return res, msg, available
424
425    def _findByLabel(self, label, forms, include_subcontrols=False):
426        # forms are iterable of mech_forms
427        matches = re.compile(r'(^|\b|\W)%s(\b|\W|$)'
428                             % re.escape(normalizeWhitespace(label))).search
429        found = []
430        for wtcontrol in self._findAllControls(forms, include_subcontrols):
431            control = getattr(wtcontrol, 'control', wtcontrol)
432            if control.type == 'hidden':
433                continue
434            for l in wtcontrol.labels:
435                if matches(l):
436                    found.append(wtcontrol)
437                    break
438        return found
439
440    def _indexControls(self, form):
441        # Unfortunately, webtest will remove all 'name' attributes from
442        # form.html after parsing. But we need them (at least to locate labels
443        # for radio buttons). So we are forced to reparse part of html, to
444        # extract elements.
445        html = BeautifulSoup(form.text, 'html.parser')
446        tags = ('input', 'select', 'textarea', 'button')
447        return html.find_all(tags)
448
449    def _findByName(self, name, forms):
450        return [c for c in self._findAllControls(forms) if c.name == name]
451
452    def _findAllControls(self, forms, include_subcontrols=False):
453        res = []
454        for f in forms:
455            if f not in self._controls:
456                fc = []
457                allelems = self._indexControls(f)
458                already_processed = set()
459                for cname, wtcontrol in f.field_order:
460                    # we need to group checkboxes by name, but leave
461                    # the other controls in the original order,
462                    # even if the name repeats
463                    if isinstance(wtcontrol, webtest.forms.Checkbox):
464                        if cname in already_processed:
465                            continue
466                        already_processed.add(cname)
467                        wtcontrols = f.fields[cname]
468                    else:
469                        wtcontrols = [wtcontrol]
470                    for c in controlFactory(cname, wtcontrols, allelems, self):
471                        fc.append((c, False))
472
473                        for subcontrol in c.controls:
474                            fc.append((subcontrol, True))
475
476                self._controls[f] = fc
477
478            controls = [c for c, subcontrol in self._controls[f]
479                        if not subcontrol or include_subcontrols]
480            res.extend(controls)
481
482        return res
483
484    def _changed(self):
485        self._counter += 1
486        self._contents = None
487        self._controls = {}
488        self.__html = None
489
490    @contextmanager
491    def _preparedRequest(self, url):
492        self.timer.start()
493
494        headers = {}
495        if self._req_referrer is not None:
496            headers['Referer'] = self._req_referrer
497
498        if self._req_content_type:
499            headers['Content-Type'] = self._req_content_type
500
501        headers['Connection'] = 'close'
502        headers['Host'] = urlparse.urlparse(url).netloc
503        headers['User-Agent'] = 'Python-urllib/2.4'
504
505        headers.update(self._req_headers)
506
507        extra_environ = {}
508        if self.handleErrors:
509            extra_environ['paste.throw_errors'] = None
510            headers['X-zope-handle-errors'] = 'True'
511        else:
512            extra_environ['wsgi.handleErrors'] = False
513            extra_environ['paste.throw_errors'] = True
514            extra_environ['x-wsgiorg.throw_errors'] = True
515            headers.pop('X-zope-handle-errors', None)
516
517        kwargs = {'headers': sorted(headers.items()),
518                  'extra_environ': extra_environ,
519                  'expect_errors': True}
520
521        yield kwargs
522
523        self._req_content_type = None
524        self.timer.stop()
525
526    def _absoluteUrl(self, url):
527        absolute = url.startswith('http://') or url.startswith('https://')
528        if absolute:
529            return str(url)
530
531        if self._response is None:
532            raise BrowserStateError(
533                "can't fetch relative reference: not viewing any document")
534
535        return str(urlparse.urljoin(self._getBaseUrl(), url))
536
537    def toStr(self, s):
538        """Convert possibly unicode object to native string using response
539        charset"""
540        if not self._response.charset:
541            return s
542        if s is None:
543            return None
544        # Might be an iterable, especially the 'class' attribute.
545        if isinstance(s, (list, tuple)):
546            subs = [self.toStr(sub) for sub in s]
547            if isinstance(s, tuple):
548                return tuple(subs)
549            return subs
550        if PYTHON2 and not isinstance(s, bytes):
551            return s.encode(self._response.charset)
552        if not PYTHON2 and isinstance(s, bytes):
553            return s.decode(self._response.charset)
554        return s
555
556    @property
557    def _html(self):
558        if self.__html is None:
559            self.__html = self._response.html
560        return self.__html
561
562
563def controlFactory(name, wtcontrols, elemindex, browser):
564    assert len(wtcontrols) > 0
565
566    first_wtc = wtcontrols[0]
567    checkbox = isinstance(first_wtc, webtest.forms.Checkbox)
568
569    # Create control list
570    if checkbox:
571        ctrlelems = [(wtc, elemindex[wtc.pos]) for wtc in wtcontrols]
572        controls = [CheckboxListControl(name, ctrlelems, browser)]
573
574    else:
575        controls = []
576        for wtc in wtcontrols:
577            controls.append(simpleControlFactory(
578                wtc, wtc.form, elemindex, browser))
579
580    return controls
581
582
583def simpleControlFactory(wtcontrol, form, elemindex, browser):
584    if isinstance(wtcontrol, webtest.forms.Radio):
585        elems = [e for e in elemindex
586                 if e.attrs.get('name') == wtcontrol.name]
587        return RadioListControl(wtcontrol, form, elems, browser)
588
589    elem = elemindex[wtcontrol.pos]
590    if isinstance(wtcontrol, (webtest.forms.Select,
591                              webtest.forms.MultipleSelect)):
592        return ListControl(wtcontrol, form, elem, browser)
593
594    elif isinstance(wtcontrol, webtest.forms.Submit):
595        if wtcontrol.attrs.get('type', 'submit') == 'image':
596            return ImageControl(wtcontrol, form, elem, browser)
597        else:
598            return SubmitControl(wtcontrol, form, elem, browser)
599    else:
600        return Control(wtcontrol, form, elem, browser)
601
602
603@implementer(interfaces.ILink)
604class Link(SetattrErrorsMixin):
605
606    def __init__(self, link, browser, baseurl=""):
607        self._link = link
608        self.browser = browser
609        self._baseurl = baseurl
610        self._browser_counter = self.browser._counter
611        self._enable_setattr_errors = True
612
613    def click(self):
614        if self._browser_counter != self.browser._counter:
615            raise interfaces.ExpiredError
616        self.browser.open(self.url, referrer=self.browser.url)
617
618    @property
619    def url(self):
620        relurl = self._link['href']
621        return self.browser._absoluteUrl(relurl)
622
623    @property
624    def text(self):
625        txt = normalizeWhitespace(self._link.text)
626        return self.browser.toStr(txt)
627
628    @property
629    def tag(self):
630        return str(self._link.name)
631
632    @property
633    def attrs(self):
634        toStr = self.browser.toStr
635        return dict((toStr(k), toStr(v)) for k, v in self._link.attrs.items())
636
637    def __repr__(self):
638        return "<%s text='%s' url='%s'>" % (
639            self.__class__.__name__, normalizeWhitespace(self.text), self.url)
640
641
642def controlFormTupleRepr(wtcontrol):
643    return wtcontrol.mechRepr()
644
645
646@implementer(interfaces.IControl)
647class Control(SetattrErrorsMixin):
648
649    _enable_setattr_errors = False
650
651    def __init__(self, control, form, elem, browser):
652        self._control = control
653        self._form = form
654        self._elem = elem
655        self.browser = browser
656        self._browser_counter = self.browser._counter
657
658        # disable addition of further attributes
659        self._enable_setattr_errors = True
660
661    @property
662    def disabled(self):
663        return 'disabled' in self._control.attrs
664
665    @property
666    def readonly(self):
667        return 'readonly' in self._control.attrs
668
669    @property
670    def type(self):
671        typeattr = self._control.attrs.get('type', None)
672        if typeattr is None:
673            # try to figure out type by tag
674            if self._control.tag == 'textarea':
675                return 'textarea'
676            else:
677                # By default, inputs are of 'text' type
678                return 'text'
679        return self.browser.toStr(typeattr)
680
681    @property
682    def name(self):
683        if self._control.name is None:
684            return None
685        return self.browser.toStr(self._control.name)
686
687    @property
688    def multiple(self):
689        return 'multiple' in self._control.attrs
690
691    @property
692    def value(self):
693        if self.type == 'file':
694            if not self._control.value:
695                return None
696
697        if self.type == 'image':
698            if not self._control.value:
699                return ''
700
701        if isinstance(self._control, webtest.forms.Submit):
702            return self.browser.toStr(self._control.value_if_submitted())
703
704        val = self._control.value
705        if val is None:
706            return None
707
708        return self.browser.toStr(val)
709
710    @value.setter
711    def value(self, value):
712        if self._browser_counter != self.browser._counter:
713            raise interfaces.ExpiredError
714        if self.readonly:
715            raise AttributeError("Trying to set value of readonly control")
716        if self.type == 'file':
717            self.add_file(value, content_type=None, filename=None)
718        else:
719            self._control.value = value
720
721    def add_file(self, file, content_type, filename):
722        if self.type != 'file':
723            raise TypeError("Can't call add_file on %s controls"
724                            % self.type)
725
726        if hasattr(file, 'read'):
727            contents = file.read()
728        else:
729            contents = file
730
731        self._form[self.name] = webtest.forms.Upload(filename or '', contents,
732                                                     content_type)
733
734    def clear(self):
735        if self._browser_counter != self.browser._counter:
736            raise zope.testbrowser.interfaces.ExpiredError
737        self.value = None
738
739    def __repr__(self):
740        return "<%s name='%s' type='%s'>" % (
741            self.__class__.__name__, self.name, self.type)
742
743    @Lazy
744    def labels(self):
745        return [self.browser.toStr(l)
746                for l in getControlLabels(self._elem, self._form.html)]
747
748    @property
749    def controls(self):
750        return []
751
752    def mechRepr(self):
753        # emulate mechanize control representation
754        toStr = self.browser.toStr
755        ctrl = self._control
756        if isinstance(ctrl, (webtest.forms.Text, webtest.forms.Email)):
757            tp = ctrl.attrs.get('type')
758            infos = []
759            if 'readonly' in ctrl.attrs or tp == 'hidden':
760                infos.append('readonly')
761            if 'disabled' in ctrl.attrs:
762                infos.append('disabled')
763
764            classnames = {'password': "PasswordControl",
765                          'hidden': "HiddenControl",
766                          'email': "EMailControl",
767                          }
768            clname = classnames.get(tp, "TextControl")
769            return "<%s(%s=%s)%s>" % (
770                clname, toStr(ctrl.name), toStr(ctrl.value),
771                ' (%s)' % (', '.join(infos)) if infos else '')
772
773        if isinstance(ctrl, webtest.forms.File):
774            return repr(ctrl) + "<-- unknown"
775        raise NotImplementedError(str((self, ctrl)))
776
777
778@implementer(interfaces.ISubmitControl)
779class SubmitControl(Control):
780
781    def click(self):
782        if self._browser_counter != self.browser._counter:
783            raise interfaces.ExpiredError
784        self.browser._clickSubmit(self._form, self._control)
785
786    @Lazy
787    def labels(self):
788        labels = super(SubmitControl, self).labels
789        labels.append(self._control.value_if_submitted())
790        if self._elem.text:
791            labels.append(normalizeWhitespace(self._elem.text))
792        return [l for l in labels if l]
793
794    def mechRepr(self):
795        name = self.name if self.name is not None else "<None>"
796        value = self.value if self.value is not None else "<None>"
797        extra = ' (disabled)' if self.disabled else ''
798        # Mechanize explicitly told us submit controls were readonly, as
799        # if they could be any other way.... *sigh*  Let's take this
800        # opportunity and strip that off.
801        return "<SubmitControl(%s=%s)%s>" % (name, value, extra)
802
803
804@implementer(interfaces.IListControl)
805class ListControl(Control):
806
807    def __init__(self, control, form, elem, browser):
808        super(ListControl, self).__init__(control, form, elem, browser)
809        # HACK: set default value of a list control and then forget about
810        # initial default values. Otherwise webtest will not allow to set None
811        # as a value of select and radio controls.
812        v = control.value
813        if v:
814            control.value = v
815            # Uncheck all the options   Carefully: WebTest used to have
816            # 2-tuples here before commit 1031d82e, and 3-tuples since then.
817            control.options = [option[:1] + (False,) + option[2:]
818                               for option in control.options]
819
820    @property
821    def type(self):
822        return 'select'
823
824    @property
825    def value(self):
826        val = self._control.value
827        if val is None:
828            return []
829
830        if self.multiple and isinstance(val, (list, tuple)):
831            return [self.browser.toStr(v) for v in val]
832        else:
833            return [self.browser.toStr(val)]
834
835    @value.setter
836    def value(self, value):
837        if not value:
838            self._set_falsy_value(value)
839        else:
840            if not self.multiple and isinstance(value, (list, tuple)):
841                value = value[0]
842            self._control.value = value
843
844    @property
845    def _selectedIndex(self):
846        return self._control.selectedIndex
847
848    @_selectedIndex.setter
849    def _selectedIndex(self, index):
850        self._control.force_value(webtest.forms.NoValue)
851        self._control.selectedIndex = index
852
853    def _set_falsy_value(self, value):
854        self._control.force_value(value)
855
856    @property
857    def displayValue(self):
858        """See zope.testbrowser.interfaces.IListControl"""
859        # not implemented for anything other than select;
860        cvalue = self._control.value
861        if cvalue is None:
862            return []
863
864        if not isinstance(cvalue, list):
865            cvalue = [cvalue]
866
867        alltitles = []
868        for key, titles in self._getOptions():
869            if key in cvalue:
870                alltitles.append(titles[0])
871        return alltitles
872
873    @displayValue.setter
874    def displayValue(self, value):
875        if self._browser_counter != self.browser._counter:
876            raise interfaces.ExpiredError
877
878        if isinstance(value, string_types):
879            value = [value]
880        if not self.multiple and len(value) > 1:
881            raise ItemCountError(
882                "single selection list, must set sequence of length 0 or 1")
883        values = []
884        found = set()
885        for key, titles in self._getOptions():
886            matches = set(v for t in titles for v in value if v in t)
887            if matches:
888                values.append(key)
889                found.update(matches)
890        for v in value:
891            if v not in found:
892                raise ItemNotFoundError(v)
893        self.value = values
894
895    @property
896    def displayOptions(self):
897        """See zope.testbrowser.interfaces.IListControl"""
898        return [titles[0] for key, titles in self._getOptions()]
899
900    @property
901    def options(self):
902        """See zope.testbrowser.interfaces.IListControl"""
903        return [key for key, title in self._getOptions()]
904
905    def getControl(self, label=None, value=None, index=None):
906        if self._browser_counter != self.browser._counter:
907            raise interfaces.ExpiredError
908
909        return getControl(self.controls, label, value, index)
910
911    @property
912    def controls(self):
913        if self._browser_counter != self.browser._counter:
914            raise interfaces.ExpiredError
915        ctrls = []
916        for idx, elem in enumerate(self._elem.select('option')):
917            ctrls.append(ItemControl(self, elem, self._form, self.browser,
918                                     idx))
919
920        return ctrls
921
922    def _getOptions(self):
923        return [(c.optionValue, c.labels) for c in self.controls]
924
925    def mechRepr(self):
926        # TODO: figure out what is replacement for "[*, ambiguous])"
927        return "<SelectControl(%s=[*, ambiguous])>" % self.name
928
929
930@implementer(interfaces.IListControl)
931class RadioListControl(ListControl):
932
933    _elems = None
934
935    def __init__(self, control, form, elems, browser):
936        super(RadioListControl, self).__init__(
937            control, form, elems[0], browser)
938        self._elems = elems
939
940    @property
941    def type(self):
942        return 'radio'
943
944    def __repr__(self):
945        # Return backwards compatible representation
946        return "<ListControl name='%s' type='radio'>" % self.name
947
948    @property
949    def controls(self):
950        if self._browser_counter != self.browser._counter:
951            raise interfaces.ExpiredError
952        for idx, opt in enumerate(self._elems):
953            yield RadioItemControl(self, opt, self._form, self.browser, idx)
954
955    @Lazy
956    def labels(self):
957        # Parent radio button control has no labels. Children are labeled.
958        return []
959
960    def _set_falsy_value(self, value):
961        # HACK: Force unsetting selected value, by avoiding validity check.
962        # Note, that force_value will not work for webtest.forms.Radio
963        # controls.
964        self._control.selectedIndex = None
965
966
967@implementer(interfaces.IListControl)
968class CheckboxListControl(SetattrErrorsMixin):
969    def __init__(self, name, ctrlelems, browser):
970        self.name = name
971        self.browser = browser
972        self._browser_counter = self.browser._counter
973        self._ctrlelems = ctrlelems
974        self._enable_setattr_errors = True
975
976    @property
977    def options(self):
978        opts = [self._trValue(c.optionValue) for c in self.controls]
979        return opts
980
981    @property
982    def displayOptions(self):
983        return [c.labels[0] for c in self.controls]
984
985    @property
986    def value(self):
987        ctrls = self.controls
988        val = [self._trValue(c.optionValue) for c in ctrls if c.selected]
989
990        if len(self._ctrlelems) == 1 and val == [True]:
991            return True
992        return val
993
994    @value.setter
995    def value(self, value):
996        ctrls = self.controls
997        if isinstance(value, (list, tuple)):
998            for c in ctrls:
999                c.selected = c.optionValue in value
1000        else:
1001            ctrls[0].selected = value
1002
1003    @property
1004    def displayValue(self):
1005        return [c.labels[0] for c in self.controls if c.selected]
1006
1007    @displayValue.setter
1008    def displayValue(self, value):
1009        found = set()
1010        for c in self.controls:
1011            matches = set(v for v in value if v in c.labels)
1012            c.selected = bool(matches)
1013            found.update(matches)
1014        for v in value:
1015            if v not in found:
1016                raise ItemNotFoundError(v)
1017
1018    @property
1019    def multiple(self):
1020        return True
1021
1022    @property
1023    def disabled(self):
1024        return all('disabled' in e.attrs for c, e in self._ctrlelems)
1025
1026    @property
1027    def type(self):
1028        return 'checkbox'
1029
1030    def getControl(self, label=None, value=None, index=None):
1031        if self._browser_counter != self.browser._counter:
1032            raise interfaces.ExpiredError
1033
1034        return getControl(self.controls, label, value, index)
1035
1036    @property
1037    def controls(self):
1038        return [CheckboxItemControl(self, c, e, c.form, self.browser, i)
1039                for i, (c, e) in enumerate(self._ctrlelems)]
1040
1041    def clear(self):
1042        if self._browser_counter != self.browser._counter:
1043            raise zope.testbrowser.interfaces.ExpiredError
1044        self.value = []
1045
1046    def mechRepr(self):
1047        return "<SelectControl(%s=[*, ambiguous])>" % self.browser.toStr(
1048            self.name)
1049
1050    @Lazy
1051    def labels(self):
1052        return []
1053
1054    def __repr__(self):
1055        # Return backwards compatible representation
1056        return "<ListControl name='%s' type='checkbox'>" % self.name
1057
1058    def _trValue(self, chbval):
1059        return True if chbval == 'on' else chbval
1060
1061
1062@implementer(interfaces.IImageSubmitControl)
1063class ImageControl(Control):
1064
1065    def click(self, coord=(1, 1)):
1066        if self._browser_counter != self.browser._counter:
1067            raise interfaces.ExpiredError
1068        self.browser._clickSubmit(self._form, self._control, coord)
1069
1070    def mechRepr(self):
1071        return "ImageControl???"  # TODO
1072
1073
1074@implementer(interfaces.IItemControl)
1075class ItemControl(SetattrErrorsMixin):
1076
1077    def __init__(self, parent, elem, form, browser, index):
1078        self._parent = parent
1079        self._elem = elem
1080        self._index = index
1081        self._form = form
1082        self.browser = browser
1083        self._browser_counter = self.browser._counter
1084        self._enable_setattr_errors = True
1085
1086    @property
1087    def control(self):
1088        if self._browser_counter != self.browser._counter:
1089            raise interfaces.ExpiredError
1090        return self._parent
1091
1092    @property
1093    def _value(self):
1094        return self._elem.attrs.get('value', self._elem.text)
1095
1096    @property
1097    def disabled(self):
1098        return 'disabled' in self._elem.attrs
1099
1100    @property
1101    def selected(self):
1102        """See zope.testbrowser.interfaces.IControl"""
1103        if self._parent.multiple:
1104            return self._value in self._parent.value
1105        else:
1106            return self._parent._selectedIndex == self._index
1107
1108    @selected.setter
1109    def selected(self, value):
1110        if self._browser_counter != self.browser._counter:
1111            raise interfaces.ExpiredError
1112        if self._parent.multiple:
1113            values = list(self._parent.value)
1114            if value:
1115                values.append(self._value)
1116            else:
1117                values = [v for v in values if v != self._value]
1118            self._parent.value = values
1119        else:
1120            if value:
1121                self._parent._selectedIndex = self._index
1122            else:
1123                self._parent.value = None
1124
1125    @property
1126    def optionValue(self):
1127        return self.browser.toStr(self._value)
1128
1129    @property
1130    def value(self):
1131        # internal alias for convenience implementing getControl()
1132        return self.optionValue
1133
1134    def click(self):
1135        if self._browser_counter != self.browser._counter:
1136            raise interfaces.ExpiredError
1137        self.selected = not self.selected
1138
1139    def __repr__(self):
1140        return (
1141            "<ItemControl name='%s' type='select' optionValue=%r selected=%r>"
1142        ) % (self._parent.name, self.optionValue, self.selected)
1143
1144    @Lazy
1145    def labels(self):
1146        labels = [self._elem.attrs.get('label'), self._elem.text]
1147        return [self.browser.toStr(normalizeWhitespace(lbl))
1148                for lbl in labels if lbl]
1149
1150    def mechRepr(self):
1151        toStr = self.browser.toStr
1152        contents = toStr(normalizeWhitespace(self._elem.text))
1153        id = toStr(self._elem.attrs.get('id'))
1154        label = toStr(self._elem.attrs.get('label', contents))
1155        value = toStr(self._value)
1156        name = toStr(self._elem.attrs.get('name', value))  # XXX wha????
1157        return (
1158            "<Item name='%s' id=%s contents='%s' value='%s' label='%s'>"
1159        ) % (name, id, contents, value, label)
1160
1161
1162class RadioItemControl(ItemControl):
1163    @property
1164    def optionValue(self):
1165        return self.browser.toStr(self._elem.attrs.get('value'))
1166
1167    @Lazy
1168    def labels(self):
1169        return [self.browser.toStr(l)
1170                for l in getControlLabels(self._elem, self._form.html)]
1171
1172    def __repr__(self):
1173        return (
1174            "<ItemControl name='%s' type='radio' optionValue=%r selected=%r>"
1175        ) % (self._parent.name, self.optionValue, self.selected)
1176
1177    def click(self):
1178        # Radio buttons cannot be unselected by clicking on them, see
1179        # https://github.com/zopefoundation/zope.testbrowser/issues/68
1180        if not self.selected:
1181            super(RadioItemControl, self).click()
1182
1183    def mechRepr(self):
1184        toStr = self.browser.toStr
1185        id = toStr(self._elem.attrs.get('id'))
1186        value = toStr(self._elem.attrs.get('value'))
1187        name = toStr(self._elem.attrs.get('name'))
1188
1189        props = []
1190        if self._elem.parent.name == 'label':
1191            props.append((
1192                '__label', {'__text': toStr(self._elem.parent.text)}))
1193        if self.selected:
1194            props.append(('checked', 'checked'))
1195        props.append(('type', 'radio'))
1196        props.append(('name', name))
1197        props.append(('value', value))
1198        props.append(('id', id))
1199
1200        propstr = ' '.join('%s=%r' % (pk, pv) for pk, pv in props)
1201        return "<Item name='%s' id='%s' %s>" % (value, id, propstr)
1202
1203
1204class CheckboxItemControl(ItemControl):
1205    _control = None
1206
1207    def __init__(self, parent, wtcontrol, elem, form, browser, index):
1208        super(CheckboxItemControl, self).__init__(parent, elem, form, browser,
1209                                                  index)
1210        self._control = wtcontrol
1211
1212    @property
1213    def selected(self):
1214        """See zope.testbrowser.interfaces.IControl"""
1215        return self._control.checked
1216
1217    @selected.setter
1218    def selected(self, value):
1219        if self._browser_counter != self.browser._counter:
1220            raise interfaces.ExpiredError
1221        self._control.checked = value
1222
1223    @property
1224    def optionValue(self):
1225        return self.browser.toStr(self._control._value or 'on')
1226
1227    @Lazy
1228    def labels(self):
1229        return [self.browser.toStr(l)
1230                for l in getControlLabels(self._elem, self._form.html)]
1231
1232    def __repr__(self):
1233        return (
1234            "<ItemControl name='%s' type='checkbox' "
1235            "optionValue=%r selected=%r>"
1236        ) % (self._control.name, self.optionValue, self.selected)
1237
1238    def mechRepr(self):
1239        id = self.browser.toStr(self._elem.attrs.get('id'))
1240        value = self.browser.toStr(self._elem.attrs.get('value'))
1241        name = self.browser.toStr(self._elem.attrs.get('name'))
1242
1243        props = []
1244        if self._elem.parent.name == 'label':
1245            props.append(('__label', {'__text': self.browser.toStr(
1246                self._elem.parent.text)}))
1247        if self.selected:
1248            props.append(('checked', 'checked'))
1249        props.append(('name', name))
1250        props.append(('type', 'checkbox'))
1251        props.append(('id', id))
1252        props.append(('value', value))
1253
1254        propstr = ' '.join('%s=%r' % (pk, pv) for pk, pv in props)
1255        return "<Item name='%s' id='%s' %s>" % (value, id, propstr)
1256
1257
1258@implementer(interfaces.IForm)
1259class Form(SetattrErrorsMixin):
1260    """HTML Form"""
1261
1262    def __init__(self, browser, form):
1263        """Initialize the Form
1264
1265        browser - a Browser instance
1266        form - a webtest.Form instance
1267        """
1268        self.browser = browser
1269        self._form = form
1270        self._browser_counter = self.browser._counter
1271        self._enable_setattr_errors = True
1272
1273    @property
1274    def action(self):
1275        return self.browser._absoluteUrl(self._form.action)
1276
1277    @property
1278    def method(self):
1279        return str(self._form.method)
1280
1281    @property
1282    def enctype(self):
1283        return str(self._form.enctype)
1284
1285    @property
1286    def name(self):
1287        return str(self._form.html.form.get('name'))
1288
1289    @property
1290    def id(self):
1291        """See zope.testbrowser.interfaces.IForm"""
1292        return str(self._form.id)
1293
1294    def submit(self, label=None, name=None, index=None, coord=None):
1295        """See zope.testbrowser.interfaces.IForm"""
1296        if self._browser_counter != self.browser._counter:
1297            raise interfaces.ExpiredError
1298
1299        form = self._form
1300        if label is not None or name is not None:
1301            controls, msg, available = self.browser._getAllControls(
1302                label, name, [form])
1303            controls = [c for c in controls
1304                        if isinstance(c, (ImageControl, SubmitControl))]
1305            control = disambiguate(
1306                controls, msg, index, controlFormTupleRepr, available)
1307            self.browser._clickSubmit(form, control._control, coord)
1308        else:  # JavaScript sort of submit
1309            if index is not None or coord is not None:
1310                raise ValueError(
1311                    'May not use index or coord without a control')
1312            self.browser._clickSubmit(form)
1313
1314    def getControl(self, label=None, name=None, index=None):
1315        """See zope.testbrowser.interfaces.IBrowser"""
1316        if self._browser_counter != self.browser._counter:
1317            raise interfaces.ExpiredError
1318        intermediate, msg, available = self.browser._getAllControls(
1319                        label, name, [self._form], include_subcontrols=True)
1320        return disambiguate(intermediate, msg, index,
1321                            controlFormTupleRepr, available)
1322
1323    @property
1324    def controls(self):
1325        return list(self.browser._findAllControls(
1326            [self._form], include_subcontrols=True))
1327
1328
1329def disambiguate(intermediate, msg, index, choice_repr=None, available=None):
1330    if intermediate:
1331        if index is None:
1332            if len(intermediate) > 1:
1333                if choice_repr:
1334                    msg += ' matches:' + ''.join([
1335                                '\n  %s' % choice_repr(choice)
1336                                for choice in intermediate])
1337                raise AmbiguityError(msg)
1338            else:
1339                return intermediate[0]
1340        else:
1341            try:
1342                return intermediate[index]
1343            except IndexError:
1344                msg = (
1345                    '%s\nIndex %d out of range, available choices are 0...%d'
1346                ) % (msg, index, len(intermediate) - 1)
1347                if choice_repr:
1348                    msg += ''.join(['\n  %d: %s' % (n, choice_repr(choice))
1349                                    for n, choice in enumerate(intermediate)])
1350    else:
1351        if available:
1352            msg += '\navailable items:' + ''.join([
1353                '\n  %s' % choice_repr(choice)
1354                for choice in available])
1355        elif available is not None:  # empty list
1356            msg += '\n(there are no form items in the HTML)'
1357    raise LookupError(msg)
1358
1359
1360def onlyOne(items, description):
1361    total = sum([bool(i) for i in items])
1362    if total == 0 or total > 1:
1363        raise ValueError(
1364            "Supply one and only one of %s as arguments" % description)
1365
1366
1367def zeroOrOne(items, description):
1368    if sum([bool(i) for i in items]) > 1:
1369        raise ValueError(
1370            "Supply no more than one of %s as arguments" % description)
1371
1372
1373def getControl(controls, label=None, value=None, index=None):
1374    onlyOne([label, value], '"label" and "value"')
1375
1376    if label is not None:
1377        options = [c for c in controls
1378                   if any(isMatching(l, label) for l in c.labels)]
1379        msg = 'label %r' % label
1380    elif value is not None:
1381        options = [c for c in controls if isMatching(c.value, value)]
1382        msg = 'value %r' % value
1383
1384    res = disambiguate(options, msg, index, controlFormTupleRepr,
1385                       available=controls)
1386    return res
1387
1388
1389def getControlLabels(celem, html):
1390    labels = []
1391
1392    # In case celem is contained in label element, use its text as a label
1393    if celem.parent.name == 'label':
1394        labels.append(normalizeWhitespace(celem.parent.text))
1395
1396    # find all labels, connected by 'for' attribute
1397    controlid = celem.attrs.get('id')
1398    if controlid:
1399        forlbls = html.select('label[for="%s"]' % controlid)
1400        labels.extend([normalizeWhitespace(l.text) for l in forlbls])
1401
1402    return [l for l in labels if l is not None]
1403
1404
1405def normalizeWhitespace(string):
1406    return ' '.join(string.split())
1407
1408
1409def isMatching(string, expr):
1410    """Determine whether ``expr`` matches to ``string``
1411
1412    ``expr`` can be None, plain text or regular expression.
1413
1414      * If ``expr`` is ``None``, ``string`` is considered matching
1415      * If ``expr`` is plain text, its equality to ``string`` will be checked
1416      * If ``expr`` is regexp, regexp matching agains ``string`` will
1417        be performed
1418    """
1419    if expr is None:
1420        return True
1421
1422    if isinstance(expr, RegexType):
1423        return expr.match(normalizeWhitespace(string))
1424    else:
1425        return normalizeWhitespace(expr) in normalizeWhitespace(string)
1426
1427
1428class Timer(object):
1429    start_time = 0
1430    end_time = 0
1431
1432    def _getTime(self):
1433        if hasattr(time, 'perf_counter'):
1434            # Python 3
1435            return time.perf_counter()
1436        else:
1437            # Python 2
1438            return time.time()
1439
1440    def start(self):
1441        """Begin a timing period"""
1442        self.start_time = self._getTime()
1443        self.end_time = None
1444
1445    def stop(self):
1446        """End a timing period"""
1447        self.end_time = self._getTime()
1448
1449    @property
1450    def elapsedSeconds(self):
1451        """Elapsed time from calling `start` to calling `stop` or present time
1452
1453        If `stop` has been called, the timing period stopped then, otherwise
1454        the end is the current time.
1455        """
1456        if self.end_time is None:
1457            end_time = self._getTime()
1458        else:
1459            end_time = self.end_time
1460        return end_time - self.start_time
1461
1462    def __enter__(self):
1463        self.start()
1464
1465    def __exit__(self, exc_type, exc_value, traceback):
1466        self.stop()
1467
1468
1469class History:
1470    """
1471
1472    Though this will become public, the implied interface is not yet stable.
1473
1474    """
1475    def __init__(self):
1476        self._history = []  # LIFO
1477
1478    def add(self, response):
1479        self._history.append(response)
1480
1481    def back(self, n, _response):
1482        response = _response
1483        while n > 0 or response is None:
1484            try:
1485                response = self._history.pop()
1486            except IndexError:
1487                raise BrowserStateError("already at start of history")
1488            n -= 1
1489        return response
1490
1491    def clear(self):
1492        del self._history[:]
1493
1494
1495class AmbiguityError(ValueError):
1496    pass
1497
1498
1499class BrowserStateError(Exception):
1500    pass
1501
1502
1503class LinkNotFoundError(IndexError):
1504    pass
1505
1506
1507class ItemCountError(ValueError):
1508    pass
1509
1510
1511class ItemNotFoundError(ValueError):
1512    pass
1513