You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1139 lines
42 KiB

4 years ago
  1. """HTTP Client for asyncio."""
  2. import asyncio
  3. import base64
  4. import hashlib
  5. import json
  6. import os
  7. import sys
  8. import traceback
  9. import warnings
  10. from types import SimpleNamespace, TracebackType
  11. from typing import ( # noqa
  12. Any,
  13. Coroutine,
  14. Generator,
  15. Generic,
  16. Iterable,
  17. List,
  18. Mapping,
  19. Optional,
  20. Set,
  21. Tuple,
  22. Type,
  23. TypeVar,
  24. Union,
  25. )
  26. import attr
  27. from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr
  28. from yarl import URL
  29. from . import hdrs, http, payload
  30. from .abc import AbstractCookieJar
  31. from .client_exceptions import (
  32. ClientConnectionError,
  33. ClientConnectorCertificateError,
  34. ClientConnectorError,
  35. ClientConnectorSSLError,
  36. ClientError,
  37. ClientHttpProxyError,
  38. ClientOSError,
  39. ClientPayloadError,
  40. ClientProxyConnectionError,
  41. ClientResponseError,
  42. ClientSSLError,
  43. ContentTypeError,
  44. InvalidURL,
  45. ServerConnectionError,
  46. ServerDisconnectedError,
  47. ServerFingerprintMismatch,
  48. ServerTimeoutError,
  49. TooManyRedirects,
  50. WSServerHandshakeError,
  51. )
  52. from .client_reqrep import (
  53. ClientRequest,
  54. ClientResponse,
  55. Fingerprint,
  56. RequestInfo,
  57. _merge_ssl_params,
  58. )
  59. from .client_ws import ClientWebSocketResponse
  60. from .connector import BaseConnector, TCPConnector, UnixConnector
  61. from .cookiejar import CookieJar
  62. from .helpers import (
  63. DEBUG,
  64. PY_36,
  65. BasicAuth,
  66. CeilTimeout,
  67. TimeoutHandle,
  68. get_running_loop,
  69. proxies_from_env,
  70. sentinel,
  71. strip_auth_from_url,
  72. )
  73. from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter
  74. from .http_websocket import ( # noqa
  75. WSHandshakeError,
  76. WSMessage,
  77. ws_ext_gen,
  78. ws_ext_parse,
  79. )
  80. from .streams import FlowControlDataQueue
  81. from .tracing import Trace, TraceConfig
  82. from .typedefs import JSONEncoder, LooseCookies, LooseHeaders, StrOrURL
  83. __all__ = (
  84. # client_exceptions
  85. 'ClientConnectionError',
  86. 'ClientConnectorCertificateError',
  87. 'ClientConnectorError',
  88. 'ClientConnectorSSLError',
  89. 'ClientError',
  90. 'ClientHttpProxyError',
  91. 'ClientOSError',
  92. 'ClientPayloadError',
  93. 'ClientProxyConnectionError',
  94. 'ClientResponseError',
  95. 'ClientSSLError',
  96. 'ContentTypeError',
  97. 'InvalidURL',
  98. 'ServerConnectionError',
  99. 'ServerDisconnectedError',
  100. 'ServerFingerprintMismatch',
  101. 'ServerTimeoutError',
  102. 'TooManyRedirects',
  103. 'WSServerHandshakeError',
  104. # client_reqrep
  105. 'ClientRequest',
  106. 'ClientResponse',
  107. 'Fingerprint',
  108. 'RequestInfo',
  109. # connector
  110. 'BaseConnector',
  111. 'TCPConnector',
  112. 'UnixConnector',
  113. # client_ws
  114. 'ClientWebSocketResponse',
  115. # client
  116. 'ClientSession',
  117. 'ClientTimeout',
  118. 'request')
  119. try:
  120. from ssl import SSLContext
  121. except ImportError: # pragma: no cover
  122. SSLContext = object # type: ignore
  123. @attr.s(frozen=True, slots=True)
  124. class ClientTimeout:
  125. total = attr.ib(type=Optional[float], default=None)
  126. connect = attr.ib(type=Optional[float], default=None)
  127. sock_read = attr.ib(type=Optional[float], default=None)
  128. sock_connect = attr.ib(type=Optional[float], default=None)
  129. # pool_queue_timeout = attr.ib(type=float, default=None)
  130. # dns_resolution_timeout = attr.ib(type=float, default=None)
  131. # socket_connect_timeout = attr.ib(type=float, default=None)
  132. # connection_acquiring_timeout = attr.ib(type=float, default=None)
  133. # new_connection_timeout = attr.ib(type=float, default=None)
  134. # http_header_timeout = attr.ib(type=float, default=None)
  135. # response_body_timeout = attr.ib(type=float, default=None)
  136. # to create a timeout specific for a single request, either
  137. # - create a completely new one to overwrite the default
  138. # - or use http://www.attrs.org/en/stable/api.html#attr.evolve
  139. # to overwrite the defaults
  140. # 5 Minute default read timeout
  141. DEFAULT_TIMEOUT = ClientTimeout(total=5*60)
  142. _RetType = TypeVar('_RetType')
  143. class ClientSession:
  144. """First-class interface for making HTTP requests."""
  145. ATTRS = frozenset([
  146. '_source_traceback', '_connector',
  147. 'requote_redirect_url', '_loop', '_cookie_jar',
  148. '_connector_owner', '_default_auth',
  149. '_version', '_json_serialize',
  150. '_requote_redirect_url',
  151. '_timeout', '_raise_for_status', '_auto_decompress',
  152. '_trust_env', '_default_headers', '_skip_auto_headers',
  153. '_request_class', '_response_class',
  154. '_ws_response_class', '_trace_configs'])
  155. _source_traceback = None
  156. _connector = None
  157. def __init__(self, *, connector: Optional[BaseConnector]=None,
  158. loop: Optional[asyncio.AbstractEventLoop]=None,
  159. cookies: Optional[LooseCookies]=None,
  160. headers: Optional[LooseHeaders]=None,
  161. skip_auto_headers: Optional[Iterable[str]]=None,
  162. auth: Optional[BasicAuth]=None,
  163. json_serialize: JSONEncoder=json.dumps,
  164. request_class: Type[ClientRequest]=ClientRequest,
  165. response_class: Type[ClientResponse]=ClientResponse,
  166. ws_response_class: Type[ClientWebSocketResponse]=ClientWebSocketResponse, # noqa
  167. version: HttpVersion=http.HttpVersion11,
  168. cookie_jar: Optional[AbstractCookieJar]=None,
  169. connector_owner: bool=True,
  170. raise_for_status: bool=False,
  171. read_timeout: Union[float, object]=sentinel,
  172. conn_timeout: Optional[float]=None,
  173. timeout: Union[object, ClientTimeout]=sentinel,
  174. auto_decompress: bool=True,
  175. trust_env: bool=False,
  176. requote_redirect_url: bool=True,
  177. trace_configs: Optional[List[TraceConfig]]=None) -> None:
  178. if loop is None:
  179. if connector is not None:
  180. loop = connector._loop
  181. loop = get_running_loop(loop)
  182. if connector is None:
  183. connector = TCPConnector(loop=loop)
  184. if connector._loop is not loop:
  185. raise RuntimeError(
  186. "Session and connector has to use same event loop")
  187. self._loop = loop
  188. if loop.get_debug():
  189. self._source_traceback = traceback.extract_stack(sys._getframe(1))
  190. if cookie_jar is None:
  191. cookie_jar = CookieJar(loop=loop)
  192. self._cookie_jar = cookie_jar
  193. if cookies is not None:
  194. self._cookie_jar.update_cookies(cookies)
  195. self._connector = connector # type: BaseConnector
  196. self._connector_owner = connector_owner
  197. self._default_auth = auth
  198. self._version = version
  199. self._json_serialize = json_serialize
  200. if timeout is sentinel:
  201. self._timeout = DEFAULT_TIMEOUT
  202. if read_timeout is not sentinel:
  203. warnings.warn("read_timeout is deprecated, "
  204. "use timeout argument instead",
  205. DeprecationWarning,
  206. stacklevel=2)
  207. self._timeout = attr.evolve(self._timeout, total=read_timeout)
  208. if conn_timeout is not None:
  209. self._timeout = attr.evolve(self._timeout,
  210. connect=conn_timeout)
  211. warnings.warn("conn_timeout is deprecated, "
  212. "use timeout argument instead",
  213. DeprecationWarning,
  214. stacklevel=2)
  215. else:
  216. self._timeout = timeout # type: ignore
  217. if read_timeout is not sentinel:
  218. raise ValueError("read_timeout and timeout parameters "
  219. "conflict, please setup "
  220. "timeout.read")
  221. if conn_timeout is not None:
  222. raise ValueError("conn_timeout and timeout parameters "
  223. "conflict, please setup "
  224. "timeout.connect")
  225. self._raise_for_status = raise_for_status
  226. self._auto_decompress = auto_decompress
  227. self._trust_env = trust_env
  228. self._requote_redirect_url = requote_redirect_url
  229. # Convert to list of tuples
  230. if headers:
  231. headers = CIMultiDict(headers)
  232. else:
  233. headers = CIMultiDict()
  234. self._default_headers = headers
  235. if skip_auto_headers is not None:
  236. self._skip_auto_headers = frozenset([istr(i)
  237. for i in skip_auto_headers])
  238. else:
  239. self._skip_auto_headers = frozenset()
  240. self._request_class = request_class
  241. self._response_class = response_class
  242. self._ws_response_class = ws_response_class
  243. self._trace_configs = trace_configs or []
  244. for trace_config in self._trace_configs:
  245. trace_config.freeze()
  246. def __init_subclass__(cls: Type['ClientSession']) -> None:
  247. warnings.warn("Inheritance class {} from ClientSession "
  248. "is discouraged".format(cls.__name__),
  249. DeprecationWarning,
  250. stacklevel=2)
  251. if DEBUG:
  252. def __setattr__(self, name: str, val: Any) -> None:
  253. if name not in self.ATTRS:
  254. warnings.warn("Setting custom ClientSession.{} attribute "
  255. "is discouraged".format(name),
  256. DeprecationWarning,
  257. stacklevel=2)
  258. super().__setattr__(name, val)
  259. def __del__(self, _warnings: Any=warnings) -> None:
  260. if not self.closed:
  261. if PY_36:
  262. kwargs = {'source': self}
  263. else:
  264. kwargs = {}
  265. _warnings.warn("Unclosed client session {!r}".format(self),
  266. ResourceWarning,
  267. **kwargs)
  268. context = {'client_session': self,
  269. 'message': 'Unclosed client session'}
  270. if self._source_traceback is not None:
  271. context['source_traceback'] = self._source_traceback
  272. self._loop.call_exception_handler(context)
  273. def request(self,
  274. method: str,
  275. url: StrOrURL,
  276. **kwargs: Any) -> '_RequestContextManager':
  277. """Perform HTTP request."""
  278. return _RequestContextManager(self._request(method, url, **kwargs))
  279. async def _request(
  280. self,
  281. method: str,
  282. str_or_url: StrOrURL, *,
  283. params: Optional[Mapping[str, str]]=None,
  284. data: Any=None,
  285. json: Any=None,
  286. cookies: Optional[LooseCookies]=None,
  287. headers: LooseHeaders=None,
  288. skip_auto_headers: Optional[Iterable[str]]=None,
  289. auth: Optional[BasicAuth]=None,
  290. allow_redirects: bool=True,
  291. max_redirects: int=10,
  292. compress: Optional[str]=None,
  293. chunked: Optional[bool]=None,
  294. expect100: bool=False,
  295. raise_for_status: Optional[bool]=None,
  296. read_until_eof: bool=True,
  297. proxy: Optional[StrOrURL]=None,
  298. proxy_auth: Optional[BasicAuth]=None,
  299. timeout: Union[ClientTimeout, object]=sentinel,
  300. verify_ssl: Optional[bool]=None,
  301. fingerprint: Optional[bytes]=None,
  302. ssl_context: Optional[SSLContext]=None,
  303. ssl: Optional[Union[SSLContext, bool, Fingerprint]]=None,
  304. proxy_headers: Optional[LooseHeaders]=None,
  305. trace_request_ctx: Optional[SimpleNamespace]=None
  306. ) -> ClientResponse:
  307. # NOTE: timeout clamps existing connect and read timeouts. We cannot
  308. # set the default to None because we need to detect if the user wants
  309. # to use the existing timeouts by setting timeout to None.
  310. if self.closed:
  311. raise RuntimeError('Session is closed')
  312. ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
  313. if data is not None and json is not None:
  314. raise ValueError(
  315. 'data and json parameters can not be used at the same time')
  316. elif json is not None:
  317. data = payload.JsonPayload(json, dumps=self._json_serialize)
  318. if not isinstance(chunked, bool) and chunked is not None:
  319. warnings.warn(
  320. 'Chunk size is deprecated #1615', DeprecationWarning)
  321. redirects = 0
  322. history = []
  323. version = self._version
  324. # Merge with default headers and transform to CIMultiDict
  325. headers = self._prepare_headers(headers)
  326. proxy_headers = self._prepare_headers(proxy_headers)
  327. try:
  328. url = URL(str_or_url)
  329. except ValueError:
  330. raise InvalidURL(str_or_url)
  331. skip_headers = set(self._skip_auto_headers)
  332. if skip_auto_headers is not None:
  333. for i in skip_auto_headers:
  334. skip_headers.add(istr(i))
  335. if proxy is not None:
  336. try:
  337. proxy = URL(proxy)
  338. except ValueError:
  339. raise InvalidURL(proxy)
  340. if timeout is sentinel:
  341. real_timeout = self._timeout # type: ClientTimeout
  342. else:
  343. if not isinstance(timeout, ClientTimeout):
  344. real_timeout = ClientTimeout(total=timeout) # type: ignore
  345. else:
  346. real_timeout = timeout
  347. # timeout is cumulative for all request operations
  348. # (request, redirects, responses, data consuming)
  349. tm = TimeoutHandle(self._loop, real_timeout.total)
  350. handle = tm.start()
  351. traces = [
  352. Trace(
  353. self,
  354. trace_config,
  355. trace_config.trace_config_ctx(
  356. trace_request_ctx=trace_request_ctx)
  357. )
  358. for trace_config in self._trace_configs
  359. ]
  360. for trace in traces:
  361. await trace.send_request_start(
  362. method,
  363. url,
  364. headers
  365. )
  366. timer = tm.timer()
  367. try:
  368. with timer:
  369. while True:
  370. url, auth_from_url = strip_auth_from_url(url)
  371. if auth and auth_from_url:
  372. raise ValueError("Cannot combine AUTH argument with "
  373. "credentials encoded in URL")
  374. if auth is None:
  375. auth = auth_from_url
  376. if auth is None:
  377. auth = self._default_auth
  378. # It would be confusing if we support explicit
  379. # Authorization header with auth argument
  380. if (headers is not None and
  381. auth is not None and
  382. hdrs.AUTHORIZATION in headers):
  383. raise ValueError("Cannot combine AUTHORIZATION header "
  384. "with AUTH argument or credentials "
  385. "encoded in URL")
  386. session_cookies = self._cookie_jar.filter_cookies(url)
  387. if cookies is not None:
  388. tmp_cookie_jar = CookieJar()
  389. tmp_cookie_jar.update_cookies(cookies)
  390. req_cookies = tmp_cookie_jar.filter_cookies(url)
  391. if req_cookies:
  392. session_cookies.load(req_cookies)
  393. cookies = session_cookies
  394. if proxy is not None:
  395. proxy = URL(proxy)
  396. elif self._trust_env:
  397. for scheme, proxy_info in proxies_from_env().items():
  398. if scheme == url.scheme:
  399. proxy = proxy_info.proxy
  400. proxy_auth = proxy_info.proxy_auth
  401. break
  402. req = self._request_class(
  403. method, url, params=params, headers=headers,
  404. skip_auto_headers=skip_headers, data=data,
  405. cookies=cookies, auth=auth, version=version,
  406. compress=compress, chunked=chunked,
  407. expect100=expect100, loop=self._loop,
  408. response_class=self._response_class,
  409. proxy=proxy, proxy_auth=proxy_auth, timer=timer,
  410. session=self,
  411. ssl=ssl, proxy_headers=proxy_headers, traces=traces)
  412. # connection timeout
  413. try:
  414. with CeilTimeout(real_timeout.connect,
  415. loop=self._loop):
  416. assert self._connector is not None
  417. conn = await self._connector.connect(
  418. req,
  419. traces=traces,
  420. timeout=real_timeout
  421. )
  422. except asyncio.TimeoutError as exc:
  423. raise ServerTimeoutError(
  424. 'Connection timeout '
  425. 'to host {0}'.format(url)) from exc
  426. assert conn.transport is not None
  427. assert conn.protocol is not None
  428. conn.protocol.set_response_params(
  429. timer=timer,
  430. skip_payload=method.upper() == 'HEAD',
  431. read_until_eof=read_until_eof,
  432. auto_decompress=self._auto_decompress,
  433. read_timeout=real_timeout.sock_read)
  434. try:
  435. try:
  436. resp = await req.send(conn)
  437. try:
  438. await resp.start(conn)
  439. except BaseException:
  440. resp.close()
  441. raise
  442. except BaseException:
  443. conn.close()
  444. raise
  445. except ClientError:
  446. raise
  447. except OSError as exc:
  448. raise ClientOSError(*exc.args) from exc
  449. self._cookie_jar.update_cookies(resp.cookies, resp.url)
  450. # redirects
  451. if resp.status in (
  452. 301, 302, 303, 307, 308) and allow_redirects:
  453. for trace in traces:
  454. await trace.send_request_redirect(
  455. method,
  456. url,
  457. headers,
  458. resp
  459. )
  460. redirects += 1
  461. history.append(resp)
  462. if max_redirects and redirects >= max_redirects:
  463. resp.close()
  464. raise TooManyRedirects(
  465. history[0].request_info, tuple(history))
  466. # For 301 and 302, mimic IE, now changed in RFC
  467. # https://github.com/kennethreitz/requests/pull/269
  468. if (resp.status == 303 and
  469. resp.method != hdrs.METH_HEAD) \
  470. or (resp.status in (301, 302) and
  471. resp.method == hdrs.METH_POST):
  472. method = hdrs.METH_GET
  473. data = None
  474. if headers.get(hdrs.CONTENT_LENGTH):
  475. headers.pop(hdrs.CONTENT_LENGTH)
  476. r_url = (resp.headers.get(hdrs.LOCATION) or
  477. resp.headers.get(hdrs.URI))
  478. if r_url is None:
  479. # see github.com/aio-libs/aiohttp/issues/2022
  480. break
  481. else:
  482. # reading from correct redirection
  483. # response is forbidden
  484. resp.release()
  485. try:
  486. r_url = URL(
  487. r_url, encoded=not self._requote_redirect_url)
  488. except ValueError:
  489. raise InvalidURL(r_url)
  490. scheme = r_url.scheme
  491. if scheme not in ('http', 'https', ''):
  492. resp.close()
  493. raise ValueError(
  494. 'Can redirect only to http or https')
  495. elif not scheme:
  496. r_url = url.join(r_url)
  497. if url.origin() != r_url.origin():
  498. auth = None
  499. headers.pop(hdrs.AUTHORIZATION, None)
  500. url = r_url
  501. params = None
  502. resp.release()
  503. continue
  504. break
  505. # check response status
  506. if raise_for_status is None:
  507. raise_for_status = self._raise_for_status
  508. if raise_for_status:
  509. resp.raise_for_status()
  510. # register connection
  511. if handle is not None:
  512. if resp.connection is not None:
  513. resp.connection.add_callback(handle.cancel)
  514. else:
  515. handle.cancel()
  516. resp._history = tuple(history)
  517. for trace in traces:
  518. await trace.send_request_end(
  519. method,
  520. url,
  521. headers,
  522. resp
  523. )
  524. return resp
  525. except BaseException as e:
  526. # cleanup timer
  527. tm.close()
  528. if handle:
  529. handle.cancel()
  530. handle = None
  531. for trace in traces:
  532. await trace.send_request_exception(
  533. method,
  534. url,
  535. headers,
  536. e
  537. )
  538. raise
  539. def ws_connect(
  540. self,
  541. url: StrOrURL, *,
  542. method: str=hdrs.METH_GET,
  543. protocols: Iterable[str]=(),
  544. timeout: float=10.0,
  545. receive_timeout: Optional[float]=None,
  546. autoclose: bool=True,
  547. autoping: bool=True,
  548. heartbeat: Optional[float]=None,
  549. auth: Optional[BasicAuth]=None,
  550. origin: Optional[str]=None,
  551. headers: Optional[LooseHeaders]=None,
  552. proxy: Optional[StrOrURL]=None,
  553. proxy_auth: Optional[BasicAuth]=None,
  554. ssl: Union[SSLContext, bool, None, Fingerprint]=None,
  555. verify_ssl: Optional[bool]=None,
  556. fingerprint: Optional[bytes]=None,
  557. ssl_context: Optional[SSLContext]=None,
  558. proxy_headers: Optional[LooseHeaders]=None,
  559. compress: int=0,
  560. max_msg_size: int=4*1024*1024) -> '_WSRequestContextManager':
  561. """Initiate websocket connection."""
  562. return _WSRequestContextManager(
  563. self._ws_connect(url,
  564. method=method,
  565. protocols=protocols,
  566. timeout=timeout,
  567. receive_timeout=receive_timeout,
  568. autoclose=autoclose,
  569. autoping=autoping,
  570. heartbeat=heartbeat,
  571. auth=auth,
  572. origin=origin,
  573. headers=headers,
  574. proxy=proxy,
  575. proxy_auth=proxy_auth,
  576. ssl=ssl,
  577. verify_ssl=verify_ssl,
  578. fingerprint=fingerprint,
  579. ssl_context=ssl_context,
  580. proxy_headers=proxy_headers,
  581. compress=compress,
  582. max_msg_size=max_msg_size))
  583. async def _ws_connect(
  584. self,
  585. url: StrOrURL, *,
  586. method: str=hdrs.METH_GET,
  587. protocols: Iterable[str]=(),
  588. timeout: float=10.0,
  589. receive_timeout: Optional[float]=None,
  590. autoclose: bool=True,
  591. autoping: bool=True,
  592. heartbeat: Optional[float]=None,
  593. auth: Optional[BasicAuth]=None,
  594. origin: Optional[str]=None,
  595. headers: Optional[LooseHeaders]=None,
  596. proxy: Optional[StrOrURL]=None,
  597. proxy_auth: Optional[BasicAuth]=None,
  598. ssl: Union[SSLContext, bool, None, Fingerprint]=None,
  599. verify_ssl: Optional[bool]=None,
  600. fingerprint: Optional[bytes]=None,
  601. ssl_context: Optional[SSLContext]=None,
  602. proxy_headers: Optional[LooseHeaders]=None,
  603. compress: int=0,
  604. max_msg_size: int=4*1024*1024
  605. ) -> ClientWebSocketResponse:
  606. if headers is None:
  607. real_headers = CIMultiDict() # type: CIMultiDict[str]
  608. else:
  609. real_headers = CIMultiDict(headers)
  610. default_headers = {
  611. hdrs.UPGRADE: hdrs.WEBSOCKET,
  612. hdrs.CONNECTION: hdrs.UPGRADE,
  613. hdrs.SEC_WEBSOCKET_VERSION: '13',
  614. }
  615. for key, value in default_headers.items():
  616. real_headers.setdefault(key, value)
  617. sec_key = base64.b64encode(os.urandom(16))
  618. real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode()
  619. if protocols:
  620. real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ','.join(protocols)
  621. if origin is not None:
  622. real_headers[hdrs.ORIGIN] = origin
  623. if compress:
  624. extstr = ws_ext_gen(compress=compress)
  625. real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr
  626. ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint)
  627. # send request
  628. resp = await self.request(method, url,
  629. headers=real_headers,
  630. read_until_eof=False,
  631. auth=auth,
  632. proxy=proxy,
  633. proxy_auth=proxy_auth,
  634. ssl=ssl,
  635. proxy_headers=proxy_headers)
  636. try:
  637. # check handshake
  638. if resp.status != 101:
  639. raise WSServerHandshakeError(
  640. resp.request_info,
  641. resp.history,
  642. message='Invalid response status',
  643. status=resp.status,
  644. headers=resp.headers)
  645. if resp.headers.get(hdrs.UPGRADE, '').lower() != 'websocket':
  646. raise WSServerHandshakeError(
  647. resp.request_info,
  648. resp.history,
  649. message='Invalid upgrade header',
  650. status=resp.status,
  651. headers=resp.headers)
  652. if resp.headers.get(hdrs.CONNECTION, '').lower() != 'upgrade':
  653. raise WSServerHandshakeError(
  654. resp.request_info,
  655. resp.history,
  656. message='Invalid connection header',
  657. status=resp.status,
  658. headers=resp.headers)
  659. # key calculation
  660. key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, '')
  661. match = base64.b64encode(
  662. hashlib.sha1(sec_key + WS_KEY).digest()).decode()
  663. if key != match:
  664. raise WSServerHandshakeError(
  665. resp.request_info,
  666. resp.history,
  667. message='Invalid challenge response',
  668. status=resp.status,
  669. headers=resp.headers)
  670. # websocket protocol
  671. protocol = None
  672. if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers:
  673. resp_protocols = [
  674. proto.strip() for proto in
  675. resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(',')]
  676. for proto in resp_protocols:
  677. if proto in protocols:
  678. protocol = proto
  679. break
  680. # websocket compress
  681. notakeover = False
  682. if compress:
  683. compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
  684. if compress_hdrs:
  685. try:
  686. compress, notakeover = ws_ext_parse(compress_hdrs)
  687. except WSHandshakeError as exc:
  688. raise WSServerHandshakeError(
  689. resp.request_info,
  690. resp.history,
  691. message=exc.args[0],
  692. status=resp.status,
  693. headers=resp.headers)
  694. else:
  695. compress = 0
  696. notakeover = False
  697. conn = resp.connection
  698. assert conn is not None
  699. proto = conn.protocol
  700. assert proto is not None
  701. transport = conn.transport
  702. assert transport is not None
  703. reader = FlowControlDataQueue(
  704. proto, limit=2 ** 16, loop=self._loop) # type: FlowControlDataQueue[WSMessage] # noqa
  705. proto.set_parser(WebSocketReader(reader, max_msg_size), reader)
  706. writer = WebSocketWriter(
  707. proto, transport, use_mask=True,
  708. compress=compress, notakeover=notakeover)
  709. except BaseException:
  710. resp.close()
  711. raise
  712. else:
  713. return self._ws_response_class(reader,
  714. writer,
  715. protocol,
  716. resp,
  717. timeout,
  718. autoclose,
  719. autoping,
  720. self._loop,
  721. receive_timeout=receive_timeout,
  722. heartbeat=heartbeat,
  723. compress=compress,
  724. client_notakeover=notakeover)
  725. def _prepare_headers(
  726. self,
  727. headers: Optional[LooseHeaders]) -> 'CIMultiDict[str]':
  728. """ Add default headers and transform it to CIMultiDict
  729. """
  730. # Convert headers to MultiDict
  731. result = CIMultiDict(self._default_headers)
  732. if headers:
  733. if not isinstance(headers, (MultiDictProxy, MultiDict)):
  734. headers = CIMultiDict(headers)
  735. added_names = set() # type: Set[str]
  736. for key, value in headers.items():
  737. if key in added_names:
  738. result.add(key, value)
  739. else:
  740. result[key] = value
  741. added_names.add(key)
  742. return result
  743. def get(self, url: StrOrURL, *, allow_redirects: bool=True,
  744. **kwargs: Any) -> '_RequestContextManager':
  745. """Perform HTTP GET request."""
  746. return _RequestContextManager(
  747. self._request(hdrs.METH_GET, url,
  748. allow_redirects=allow_redirects,
  749. **kwargs))
  750. def options(self, url: StrOrURL, *, allow_redirects: bool=True,
  751. **kwargs: Any) -> '_RequestContextManager':
  752. """Perform HTTP OPTIONS request."""
  753. return _RequestContextManager(
  754. self._request(hdrs.METH_OPTIONS, url,
  755. allow_redirects=allow_redirects,
  756. **kwargs))
  757. def head(self, url: StrOrURL, *, allow_redirects: bool=False,
  758. **kwargs: Any) -> '_RequestContextManager':
  759. """Perform HTTP HEAD request."""
  760. return _RequestContextManager(
  761. self._request(hdrs.METH_HEAD, url,
  762. allow_redirects=allow_redirects,
  763. **kwargs))
  764. def post(self, url: StrOrURL,
  765. *, data: Any=None, **kwargs: Any) -> '_RequestContextManager':
  766. """Perform HTTP POST request."""
  767. return _RequestContextManager(
  768. self._request(hdrs.METH_POST, url,
  769. data=data,
  770. **kwargs))
  771. def put(self, url: StrOrURL,
  772. *, data: Any=None, **kwargs: Any) -> '_RequestContextManager':
  773. """Perform HTTP PUT request."""
  774. return _RequestContextManager(
  775. self._request(hdrs.METH_PUT, url,
  776. data=data,
  777. **kwargs))
  778. def patch(self, url: StrOrURL,
  779. *, data: Any=None, **kwargs: Any) -> '_RequestContextManager':
  780. """Perform HTTP PATCH request."""
  781. return _RequestContextManager(
  782. self._request(hdrs.METH_PATCH, url,
  783. data=data,
  784. **kwargs))
  785. def delete(self, url: StrOrURL, **kwargs: Any) -> '_RequestContextManager':
  786. """Perform HTTP DELETE request."""
  787. return _RequestContextManager(
  788. self._request(hdrs.METH_DELETE, url,
  789. **kwargs))
  790. async def close(self) -> None:
  791. """Close underlying connector.
  792. Release all acquired resources.
  793. """
  794. if not self.closed:
  795. if self._connector is not None and self._connector_owner:
  796. await self._connector.close()
  797. self._connector = None
  798. @property
  799. def closed(self) -> bool:
  800. """Is client session closed.
  801. A readonly property.
  802. """
  803. return self._connector is None or self._connector.closed
  804. @property
  805. def connector(self) -> Optional[BaseConnector]:
  806. """Connector instance used for the session."""
  807. return self._connector
  808. @property
  809. def cookie_jar(self) -> AbstractCookieJar:
  810. """The session cookies."""
  811. return self._cookie_jar
  812. @property
  813. def version(self) -> Tuple[int, int]:
  814. """The session HTTP protocol version."""
  815. return self._version
  816. @property
  817. def requote_redirect_url(self) -> bool:
  818. """Do URL requoting on redirection handling."""
  819. return self._requote_redirect_url
  820. @requote_redirect_url.setter
  821. def requote_redirect_url(self, val: bool) -> None:
  822. """Do URL requoting on redirection handling."""
  823. warnings.warn("session.requote_redirect_url modification "
  824. "is deprecated #2778",
  825. DeprecationWarning,
  826. stacklevel=2)
  827. self._requote_redirect_url = val
  828. @property
  829. def loop(self) -> asyncio.AbstractEventLoop:
  830. """Session's loop."""
  831. warnings.warn("client.loop property is deprecated",
  832. DeprecationWarning,
  833. stacklevel=2)
  834. return self._loop
  835. def detach(self) -> None:
  836. """Detach connector from session without closing the former.
  837. Session is switched to closed state anyway.
  838. """
  839. self._connector = None
  840. def __enter__(self) -> None:
  841. raise TypeError("Use async with instead")
  842. def __exit__(self,
  843. exc_type: Optional[Type[BaseException]],
  844. exc_val: Optional[BaseException],
  845. exc_tb: Optional[TracebackType]) -> None:
  846. # __exit__ should exist in pair with __enter__ but never executed
  847. pass # pragma: no cover
  848. async def __aenter__(self) -> 'ClientSession':
  849. return self
  850. async def __aexit__(self,
  851. exc_type: Optional[Type[BaseException]],
  852. exc_val: Optional[BaseException],
  853. exc_tb: Optional[TracebackType]) -> None:
  854. await self.close()
  855. class _BaseRequestContextManager(Coroutine[Any,
  856. Any,
  857. _RetType],
  858. Generic[_RetType]):
  859. __slots__ = ('_coro', '_resp')
  860. def __init__(
  861. self,
  862. coro: Coroutine['asyncio.Future[Any]', None, _RetType]
  863. ) -> None:
  864. self._coro = coro
  865. def send(self, arg: None) -> 'asyncio.Future[Any]':
  866. return self._coro.send(arg)
  867. def throw(self, arg: BaseException) -> None: # type: ignore
  868. self._coro.throw(arg) # type: ignore
  869. def close(self) -> None:
  870. return self._coro.close()
  871. def __await__(self) -> Generator[Any, None, _RetType]:
  872. ret = self._coro.__await__()
  873. return ret
  874. def __iter__(self) -> Generator[Any, None, _RetType]:
  875. return self.__await__()
  876. async def __aenter__(self) -> _RetType:
  877. self._resp = await self._coro
  878. return self._resp
  879. class _RequestContextManager(_BaseRequestContextManager[ClientResponse]):
  880. async def __aexit__(self,
  881. exc_type: Optional[Type[BaseException]],
  882. exc: Optional[BaseException],
  883. tb: Optional[TracebackType]) -> None:
  884. # We're basing behavior on the exception as it can be caused by
  885. # user code unrelated to the status of the connection. If you
  886. # would like to close a connection you must do that
  887. # explicitly. Otherwise connection error handling should kick in
  888. # and close/recycle the connection as required.
  889. self._resp.release()
  890. class _WSRequestContextManager(_BaseRequestContextManager[
  891. ClientWebSocketResponse]):
  892. async def __aexit__(self,
  893. exc_type: Optional[Type[BaseException]],
  894. exc: Optional[BaseException],
  895. tb: Optional[TracebackType]) -> None:
  896. await self._resp.close()
  897. class _SessionRequestContextManager:
  898. __slots__ = ('_coro', '_resp', '_session')
  899. def __init__(self,
  900. coro: Coroutine['asyncio.Future[Any]', None, ClientResponse],
  901. session: ClientSession) -> None:
  902. self._coro = coro
  903. self._resp = None # type: Optional[ClientResponse]
  904. self._session = session
  905. async def __aenter__(self) -> ClientResponse:
  906. self._resp = await self._coro
  907. return self._resp
  908. async def __aexit__(self,
  909. exc_type: Optional[Type[BaseException]],
  910. exc: Optional[BaseException],
  911. tb: Optional[TracebackType]) -> None:
  912. assert self._resp is not None
  913. self._resp.close()
  914. await self._session.close()
  915. def request(
  916. method: str,
  917. url: StrOrURL, *,
  918. params: Optional[Mapping[str, str]]=None,
  919. data: Any=None,
  920. json: Any=None,
  921. headers: LooseHeaders=None,
  922. skip_auto_headers: Optional[Iterable[str]]=None,
  923. auth: Optional[BasicAuth]=None,
  924. allow_redirects: bool=True,
  925. max_redirects: int=10,
  926. compress: Optional[str]=None,
  927. chunked: Optional[bool]=None,
  928. expect100: bool=False,
  929. raise_for_status: Optional[bool]=None,
  930. read_until_eof: bool=True,
  931. proxy: Optional[StrOrURL]=None,
  932. proxy_auth: Optional[BasicAuth]=None,
  933. timeout: Union[ClientTimeout, object]=sentinel,
  934. cookies: Optional[LooseCookies]=None,
  935. version: HttpVersion=http.HttpVersion11,
  936. connector: Optional[BaseConnector]=None,
  937. loop: Optional[asyncio.AbstractEventLoop]=None
  938. ) -> _SessionRequestContextManager:
  939. """Constructs and sends a request. Returns response object.
  940. method - HTTP method
  941. url - request url
  942. params - (optional) Dictionary or bytes to be sent in the query
  943. string of the new request
  944. data - (optional) Dictionary, bytes, or file-like object to
  945. send in the body of the request
  946. json - (optional) Any json compatible python object
  947. headers - (optional) Dictionary of HTTP Headers to send with
  948. the request
  949. cookies - (optional) Dict object to send with the request
  950. auth - (optional) BasicAuth named tuple represent HTTP Basic Auth
  951. auth - aiohttp.helpers.BasicAuth
  952. allow_redirects - (optional) If set to False, do not follow
  953. redirects
  954. version - Request HTTP version.
  955. compress - Set to True if request has to be compressed
  956. with deflate encoding.
  957. chunked - Set to chunk size for chunked transfer encoding.
  958. expect100 - Expect 100-continue response from server.
  959. connector - BaseConnector sub-class instance to support
  960. connection pooling.
  961. read_until_eof - Read response until eof if response
  962. does not have Content-Length header.
  963. loop - Optional event loop.
  964. timeout - Optional ClientTimeout settings structure, 5min
  965. total timeout by default.
  966. Usage::
  967. >>> import aiohttp
  968. >>> resp = await aiohttp.request('GET', 'http://python.org/')
  969. >>> resp
  970. <ClientResponse(python.org/) [200]>
  971. >>> data = await resp.read()
  972. """
  973. connector_owner = False
  974. if connector is None:
  975. connector_owner = True
  976. connector = TCPConnector(loop=loop, force_close=True)
  977. session = ClientSession(
  978. loop=loop, cookies=cookies, version=version, timeout=timeout,
  979. connector=connector, connector_owner=connector_owner)
  980. return _SessionRequestContextManager(
  981. session._request(method, url,
  982. params=params,
  983. data=data,
  984. json=json,
  985. headers=headers,
  986. skip_auto_headers=skip_auto_headers,
  987. auth=auth,
  988. allow_redirects=allow_redirects,
  989. max_redirects=max_redirects,
  990. compress=compress,
  991. chunked=chunked,
  992. expect100=expect100,
  993. raise_for_status=raise_for_status,
  994. read_until_eof=read_until_eof,
  995. proxy=proxy,
  996. proxy_auth=proxy_auth,),
  997. session)