215 lines
7.5 KiB
Python
215 lines
7.5 KiB
Python
"""
|
|
This module implements the FormRequest class which is a more convenient class
|
|
(than Request) to generate Requests based on form data.
|
|
|
|
See documentation in docs/topics/request-response.rst
|
|
"""
|
|
|
|
from urllib.parse import urljoin, urlencode
|
|
|
|
import lxml.html
|
|
from parsel.selector import create_root_node
|
|
from w3lib.html import strip_html5_whitespace
|
|
|
|
from scrapy.http.request import Request
|
|
from scrapy.utils.python import to_bytes, is_listlike
|
|
from scrapy.utils.response import get_base_url
|
|
|
|
|
|
class FormRequest(Request):
|
|
valid_form_methods = ['GET', 'POST']
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
formdata = kwargs.pop('formdata', None)
|
|
if formdata and kwargs.get('method') is None:
|
|
kwargs['method'] = 'POST'
|
|
|
|
super().__init__(*args, **kwargs)
|
|
|
|
if formdata:
|
|
items = formdata.items() if isinstance(formdata, dict) else formdata
|
|
querystr = _urlencode(items, self.encoding)
|
|
if self.method == 'POST':
|
|
self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
|
|
self._set_body(querystr)
|
|
else:
|
|
self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
|
|
|
|
@classmethod
|
|
def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
|
|
clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
|
|
|
|
kwargs.setdefault('encoding', response.encoding)
|
|
|
|
if formcss is not None:
|
|
from parsel.csstranslator import HTMLTranslator
|
|
formxpath = HTMLTranslator().css_to_xpath(formcss)
|
|
|
|
form = _get_form(response, formname, formid, formnumber, formxpath)
|
|
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
|
|
url = _get_form_url(form, kwargs.pop('url', None))
|
|
|
|
method = kwargs.pop('method', form.method)
|
|
if method is not None:
|
|
method = method.upper()
|
|
if method not in cls.valid_form_methods:
|
|
method = 'GET'
|
|
|
|
return cls(url=url, method=method, formdata=formdata, **kwargs)
|
|
|
|
|
|
def _get_form_url(form, url):
|
|
if url is None:
|
|
action = form.get('action')
|
|
if action is None:
|
|
return form.base_url
|
|
return urljoin(form.base_url, strip_html5_whitespace(action))
|
|
return urljoin(form.base_url, url)
|
|
|
|
|
|
def _urlencode(seq, enc):
|
|
values = [(to_bytes(k, enc), to_bytes(v, enc))
|
|
for k, vs in seq
|
|
for v in (vs if is_listlike(vs) else [vs])]
|
|
return urlencode(values, doseq=1)
|
|
|
|
|
|
def _get_form(response, formname, formid, formnumber, formxpath):
|
|
"""Find the form element """
|
|
root = create_root_node(response.text, lxml.html.HTMLParser,
|
|
base_url=get_base_url(response))
|
|
forms = root.xpath('//form')
|
|
if not forms:
|
|
raise ValueError(f"No <form> element found in {response}")
|
|
|
|
if formname is not None:
|
|
f = root.xpath(f'//form[@name="{formname}"]')
|
|
if f:
|
|
return f[0]
|
|
|
|
if formid is not None:
|
|
f = root.xpath(f'//form[@id="{formid}"]')
|
|
if f:
|
|
return f[0]
|
|
|
|
# Get form element from xpath, if not found, go up
|
|
if formxpath is not None:
|
|
nodes = root.xpath(formxpath)
|
|
if nodes:
|
|
el = nodes[0]
|
|
while True:
|
|
if el.tag == 'form':
|
|
return el
|
|
el = el.getparent()
|
|
if el is None:
|
|
break
|
|
raise ValueError(f'No <form> element found with {formxpath}')
|
|
|
|
# If we get here, it means that either formname was None
|
|
# or invalid
|
|
if formnumber is not None:
|
|
try:
|
|
form = forms[formnumber]
|
|
except IndexError:
|
|
raise IndexError(f"Form number {formnumber} not found in {response}")
|
|
else:
|
|
return form
|
|
|
|
|
|
def _get_inputs(form, formdata, dont_click, clickdata, response):
|
|
try:
|
|
formdata_keys = dict(formdata or ()).keys()
|
|
except (ValueError, TypeError):
|
|
raise ValueError('formdata should be a dict or iterable of tuples')
|
|
|
|
if not formdata:
|
|
formdata = ()
|
|
inputs = form.xpath('descendant::textarea'
|
|
'|descendant::select'
|
|
'|descendant::input[not(@type) or @type['
|
|
' not(re:test(., "^(?:submit|image|reset)$", "i"))'
|
|
' and (../@checked or'
|
|
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
|
|
namespaces={
|
|
"re": "http://exslt.org/regular-expressions"})
|
|
values = [(k, '' if v is None else v)
|
|
for k, v in (_value(e) for e in inputs)
|
|
if k and k not in formdata_keys]
|
|
|
|
if not dont_click:
|
|
clickable = _get_clickable(clickdata, form)
|
|
if clickable and clickable[0] not in formdata and not clickable[0] is None:
|
|
values.append(clickable)
|
|
|
|
if isinstance(formdata, dict):
|
|
formdata = formdata.items()
|
|
|
|
values.extend((k, v) for k, v in formdata if v is not None)
|
|
return values
|
|
|
|
|
|
def _value(ele):
|
|
n = ele.name
|
|
v = ele.value
|
|
if ele.tag == 'select':
|
|
return _select_value(ele, n, v)
|
|
return n, v
|
|
|
|
|
|
def _select_value(ele, n, v):
|
|
multiple = ele.multiple
|
|
if v is None and not multiple:
|
|
# Match browser behaviour on simple select tag without options selected
|
|
# And for select tags wihout options
|
|
o = ele.value_options
|
|
return (n, o[0]) if o else (None, None)
|
|
elif v is not None and multiple:
|
|
# This is a workround to bug in lxml fixed 2.3.1
|
|
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
|
|
selected_options = ele.xpath('.//option[@selected]')
|
|
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
|
|
return n, v
|
|
|
|
|
|
def _get_clickable(clickdata, form):
|
|
"""
|
|
Returns the clickable element specified in clickdata,
|
|
if the latter is given. If not, it returns the first
|
|
clickable element found
|
|
"""
|
|
clickables = list(form.xpath(
|
|
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
|
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
|
namespaces={"re": "http://exslt.org/regular-expressions"}
|
|
))
|
|
if not clickables:
|
|
return
|
|
|
|
# If we don't have clickdata, we just use the first clickable element
|
|
if clickdata is None:
|
|
el = clickables[0]
|
|
return (el.get('name'), el.get('value') or '')
|
|
|
|
# If clickdata is given, we compare it to the clickable elements to find a
|
|
# match. We first look to see if the number is specified in clickdata,
|
|
# because that uniquely identifies the element
|
|
nr = clickdata.get('nr', None)
|
|
if nr is not None:
|
|
try:
|
|
el = list(form.inputs)[nr]
|
|
except IndexError:
|
|
pass
|
|
else:
|
|
return (el.get('name'), el.get('value') or '')
|
|
|
|
# We didn't find it, so now we build an XPath expression out of the other
|
|
# arguments, because they can be used as such
|
|
xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items())
|
|
el = form.xpath(xpath)
|
|
if len(el) == 1:
|
|
return (el[0].get('name'), el[0].get('value') or '')
|
|
elif len(el) > 1:
|
|
raise ValueError(f"Multiple elements found ({el!r}) matching the "
|
|
f"criteria in clickdata: {clickdata!r}")
|
|
else:
|
|
raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')
|