You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

367 lines
13 KiB

4 years ago
  1. """
  2. The httplib2 algorithms ported for use with requests.
  3. """
  4. import logging
  5. import re
  6. import calendar
  7. import time
  8. from email.utils import parsedate_tz
  9. from requests.structures import CaseInsensitiveDict
  10. from .cache import DictCache
  11. from .serialize import Serializer
  12. logger = logging.getLogger(__name__)
  13. URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
  14. def parse_uri(uri):
  15. """Parses a URI using the regex given in Appendix B of RFC 3986.
  16. (scheme, authority, path, query, fragment) = parse_uri(uri)
  17. """
  18. groups = URI.match(uri).groups()
  19. return (groups[1], groups[3], groups[4], groups[6], groups[8])
  20. class CacheController(object):
  21. """An interface to see if request should cached or not.
  22. """
  23. def __init__(
  24. self, cache=None, cache_etags=True, serializer=None, status_codes=None
  25. ):
  26. self.cache = cache or DictCache()
  27. self.cache_etags = cache_etags
  28. self.serializer = serializer or Serializer()
  29. self.cacheable_status_codes = status_codes or (200, 203, 300, 301)
  30. @classmethod
  31. def _urlnorm(cls, uri):
  32. """Normalize the URL to create a safe key for the cache"""
  33. (scheme, authority, path, query, fragment) = parse_uri(uri)
  34. if not scheme or not authority:
  35. raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
  36. scheme = scheme.lower()
  37. authority = authority.lower()
  38. if not path:
  39. path = "/"
  40. # Could do syntax based normalization of the URI before
  41. # computing the digest. See Section 6.2.2 of Std 66.
  42. request_uri = query and "?".join([path, query]) or path
  43. defrag_uri = scheme + "://" + authority + request_uri
  44. return defrag_uri
  45. @classmethod
  46. def cache_url(cls, uri):
  47. return cls._urlnorm(uri)
  48. def parse_cache_control(self, headers):
  49. known_directives = {
  50. # https://tools.ietf.org/html/rfc7234#section-5.2
  51. "max-age": (int, True),
  52. "max-stale": (int, False),
  53. "min-fresh": (int, True),
  54. "no-cache": (None, False),
  55. "no-store": (None, False),
  56. "no-transform": (None, False),
  57. "only-if-cached": (None, False),
  58. "must-revalidate": (None, False),
  59. "public": (None, False),
  60. "private": (None, False),
  61. "proxy-revalidate": (None, False),
  62. "s-maxage": (int, True),
  63. }
  64. cc_headers = headers.get("cache-control", headers.get("Cache-Control", ""))
  65. retval = {}
  66. for cc_directive in cc_headers.split(","):
  67. if not cc_directive.strip():
  68. continue
  69. parts = cc_directive.split("=", 1)
  70. directive = parts[0].strip()
  71. try:
  72. typ, required = known_directives[directive]
  73. except KeyError:
  74. logger.debug("Ignoring unknown cache-control directive: %s", directive)
  75. continue
  76. if not typ or not required:
  77. retval[directive] = None
  78. if typ:
  79. try:
  80. retval[directive] = typ(parts[1].strip())
  81. except IndexError:
  82. if required:
  83. logger.debug(
  84. "Missing value for cache-control " "directive: %s",
  85. directive,
  86. )
  87. except ValueError:
  88. logger.debug(
  89. "Invalid value for cache-control directive " "%s, must be %s",
  90. directive,
  91. typ.__name__,
  92. )
  93. return retval
  94. def cached_request(self, request):
  95. """
  96. Return a cached response if it exists in the cache, otherwise
  97. return False.
  98. """
  99. cache_url = self.cache_url(request.url)
  100. logger.debug('Looking up "%s" in the cache', cache_url)
  101. cc = self.parse_cache_control(request.headers)
  102. # Bail out if the request insists on fresh data
  103. if "no-cache" in cc:
  104. logger.debug('Request header has "no-cache", cache bypassed')
  105. return False
  106. if "max-age" in cc and cc["max-age"] == 0:
  107. logger.debug('Request header has "max_age" as 0, cache bypassed')
  108. return False
  109. # Request allows serving from the cache, let's see if we find something
  110. cache_data = self.cache.get(cache_url)
  111. if cache_data is None:
  112. logger.debug("No cache entry available")
  113. return False
  114. # Check whether it can be deserialized
  115. resp = self.serializer.loads(request, cache_data)
  116. if not resp:
  117. logger.warning("Cache entry deserialization failed, entry ignored")
  118. return False
  119. # If we have a cached 301, return it immediately. We don't
  120. # need to test our response for other headers b/c it is
  121. # intrinsically "cacheable" as it is Permanent.
  122. # See:
  123. # https://tools.ietf.org/html/rfc7231#section-6.4.2
  124. #
  125. # Client can try to refresh the value by repeating the request
  126. # with cache busting headers as usual (ie no-cache).
  127. if resp.status == 301:
  128. msg = (
  129. 'Returning cached "301 Moved Permanently" response '
  130. "(ignoring date and etag information)"
  131. )
  132. logger.debug(msg)
  133. return resp
  134. headers = CaseInsensitiveDict(resp.headers)
  135. if not headers or "date" not in headers:
  136. if "etag" not in headers:
  137. # Without date or etag, the cached response can never be used
  138. # and should be deleted.
  139. logger.debug("Purging cached response: no date or etag")
  140. self.cache.delete(cache_url)
  141. logger.debug("Ignoring cached response: no date")
  142. return False
  143. now = time.time()
  144. date = calendar.timegm(parsedate_tz(headers["date"]))
  145. current_age = max(0, now - date)
  146. logger.debug("Current age based on date: %i", current_age)
  147. # TODO: There is an assumption that the result will be a
  148. # urllib3 response object. This may not be best since we
  149. # could probably avoid instantiating or constructing the
  150. # response until we know we need it.
  151. resp_cc = self.parse_cache_control(headers)
  152. # determine freshness
  153. freshness_lifetime = 0
  154. # Check the max-age pragma in the cache control header
  155. if "max-age" in resp_cc:
  156. freshness_lifetime = resp_cc["max-age"]
  157. logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime)
  158. # If there isn't a max-age, check for an expires header
  159. elif "expires" in headers:
  160. expires = parsedate_tz(headers["expires"])
  161. if expires is not None:
  162. expire_time = calendar.timegm(expires) - date
  163. freshness_lifetime = max(0, expire_time)
  164. logger.debug("Freshness lifetime from expires: %i", freshness_lifetime)
  165. # Determine if we are setting freshness limit in the
  166. # request. Note, this overrides what was in the response.
  167. if "max-age" in cc:
  168. freshness_lifetime = cc["max-age"]
  169. logger.debug(
  170. "Freshness lifetime from request max-age: %i", freshness_lifetime
  171. )
  172. if "min-fresh" in cc:
  173. min_fresh = cc["min-fresh"]
  174. # adjust our current age by our min fresh
  175. current_age += min_fresh
  176. logger.debug("Adjusted current age from min-fresh: %i", current_age)
  177. # Return entry if it is fresh enough
  178. if freshness_lifetime > current_age:
  179. logger.debug('The response is "fresh", returning cached response')
  180. logger.debug("%i > %i", freshness_lifetime, current_age)
  181. return resp
  182. # we're not fresh. If we don't have an Etag, clear it out
  183. if "etag" not in headers:
  184. logger.debug('The cached response is "stale" with no etag, purging')
  185. self.cache.delete(cache_url)
  186. # return the original handler
  187. return False
  188. def conditional_headers(self, request):
  189. cache_url = self.cache_url(request.url)
  190. resp = self.serializer.loads(request, self.cache.get(cache_url))
  191. new_headers = {}
  192. if resp:
  193. headers = CaseInsensitiveDict(resp.headers)
  194. if "etag" in headers:
  195. new_headers["If-None-Match"] = headers["ETag"]
  196. if "last-modified" in headers:
  197. new_headers["If-Modified-Since"] = headers["Last-Modified"]
  198. return new_headers
  199. def cache_response(self, request, response, body=None, status_codes=None):
  200. """
  201. Algorithm for caching requests.
  202. This assumes a requests Response object.
  203. """
  204. # From httplib2: Don't cache 206's since we aren't going to
  205. # handle byte range requests
  206. cacheable_status_codes = status_codes or self.cacheable_status_codes
  207. if response.status not in cacheable_status_codes:
  208. logger.debug(
  209. "Status code %s not in %s", response.status, cacheable_status_codes
  210. )
  211. return
  212. response_headers = CaseInsensitiveDict(response.headers)
  213. # If we've been given a body, our response has a Content-Length, that
  214. # Content-Length is valid then we can check to see if the body we've
  215. # been given matches the expected size, and if it doesn't we'll just
  216. # skip trying to cache it.
  217. if (
  218. body is not None
  219. and "content-length" in response_headers
  220. and response_headers["content-length"].isdigit()
  221. and int(response_headers["content-length"]) != len(body)
  222. ):
  223. return
  224. cc_req = self.parse_cache_control(request.headers)
  225. cc = self.parse_cache_control(response_headers)
  226. cache_url = self.cache_url(request.url)
  227. logger.debug('Updating cache with response from "%s"', cache_url)
  228. # Delete it from the cache if we happen to have it stored there
  229. no_store = False
  230. if "no-store" in cc:
  231. no_store = True
  232. logger.debug('Response header has "no-store"')
  233. if "no-store" in cc_req:
  234. no_store = True
  235. logger.debug('Request header has "no-store"')
  236. if no_store and self.cache.get(cache_url):
  237. logger.debug('Purging existing cache entry to honor "no-store"')
  238. self.cache.delete(cache_url)
  239. if no_store:
  240. return
  241. # If we've been given an etag, then keep the response
  242. if self.cache_etags and "etag" in response_headers:
  243. logger.debug("Caching due to etag")
  244. self.cache.set(
  245. cache_url, self.serializer.dumps(request, response, body=body)
  246. )
  247. # Add to the cache any 301s. We do this before looking that
  248. # the Date headers.
  249. elif response.status == 301:
  250. logger.debug("Caching permanant redirect")
  251. self.cache.set(cache_url, self.serializer.dumps(request, response))
  252. # Add to the cache if the response headers demand it. If there
  253. # is no date header then we can't do anything about expiring
  254. # the cache.
  255. elif "date" in response_headers:
  256. # cache when there is a max-age > 0
  257. if "max-age" in cc and cc["max-age"] > 0:
  258. logger.debug("Caching b/c date exists and max-age > 0")
  259. self.cache.set(
  260. cache_url, self.serializer.dumps(request, response, body=body)
  261. )
  262. # If the request can expire, it means we should cache it
  263. # in the meantime.
  264. elif "expires" in response_headers:
  265. if response_headers["expires"]:
  266. logger.debug("Caching b/c of expires header")
  267. self.cache.set(
  268. cache_url, self.serializer.dumps(request, response, body=body)
  269. )
  270. def update_cached_response(self, request, response):
  271. """On a 304 we will get a new set of headers that we want to
  272. update our cached value with, assuming we have one.
  273. This should only ever be called when we've sent an ETag and
  274. gotten a 304 as the response.
  275. """
  276. cache_url = self.cache_url(request.url)
  277. cached_response = self.serializer.loads(request, self.cache.get(cache_url))
  278. if not cached_response:
  279. # we didn't have a cached response
  280. return response
  281. # Lets update our headers with the headers from the new request:
  282. # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
  283. #
  284. # The server isn't supposed to send headers that would make
  285. # the cached body invalid. But... just in case, we'll be sure
  286. # to strip out ones we know that might be problmatic due to
  287. # typical assumptions.
  288. excluded_headers = ["content-length"]
  289. cached_response.headers.update(
  290. dict(
  291. (k, v)
  292. for k, v in response.headers.items()
  293. if k.lower() not in excluded_headers
  294. )
  295. )
  296. # we want a 200 b/c we have content via the cache
  297. cached_response.status = 200
  298. # update our cache
  299. self.cache.set(cache_url, self.serializer.dumps(request, cached_response))
  300. return cached_response