You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

803 lines
23 KiB

4 years ago
  1. import inspect
  2. import sys
  3. from functools import partial
  4. from operator import attrgetter
  5. from textwrap import dedent
  6. from cytoolz.utils import no_default
  7. from cytoolz.compatibility import PY3, PY33, PY34, filter as ifilter, map as imap, reduce, import_module
  8. import cytoolz._signatures as _sigs
  9. from toolz.functoolz import (InstanceProperty, instanceproperty, is_arity,
  10. num_required_args, has_varargs, has_keywords,
  11. is_valid_args, is_partial_args)
  12. cimport cython
  13. from cpython.dict cimport PyDict_Merge, PyDict_New
  14. from cpython.object cimport (PyCallable_Check, PyObject_Call, PyObject_CallObject,
  15. PyObject_RichCompare, Py_EQ, Py_NE)
  16. from cpython.ref cimport PyObject
  17. from cpython.sequence cimport PySequence_Concat
  18. from cpython.set cimport PyFrozenSet_New
  19. from cpython.tuple cimport PyTuple_Check, PyTuple_GET_SIZE
  20. __all__ = ['identity', 'thread_first', 'thread_last', 'memoize', 'compose',
  21. 'pipe', 'complement', 'juxt', 'do', 'curry', 'memoize', 'flip',
  22. 'excepts']
  23. cpdef object identity(object x):
  24. return x
  25. cdef object c_thread_first(object val, object forms):
  26. cdef object form, func
  27. cdef tuple args
  28. for form in forms:
  29. if PyCallable_Check(form):
  30. val = form(val)
  31. elif PyTuple_Check(form):
  32. func, args = form[0], (val,) + form[1:]
  33. val = PyObject_CallObject(func, args)
  34. else:
  35. val = None
  36. return val
  37. def thread_first(val, *forms):
  38. """
  39. Thread value through a sequence of functions/forms
  40. >>> def double(x): return 2*x
  41. >>> def inc(x): return x + 1
  42. >>> thread_first(1, inc, double)
  43. 4
  44. If the function expects more than one input you can specify those inputs
  45. in a tuple. The value is used as the first input.
  46. >>> def add(x, y): return x + y
  47. >>> def pow(x, y): return x**y
  48. >>> thread_first(1, (add, 4), (pow, 2)) # pow(add(1, 4), 2)
  49. 25
  50. So in general
  51. thread_first(x, f, (g, y, z))
  52. expands to
  53. g(f(x), y, z)
  54. See Also:
  55. thread_last
  56. """
  57. return c_thread_first(val, forms)
  58. cdef object c_thread_last(object val, object forms):
  59. cdef object form, func
  60. cdef tuple args
  61. for form in forms:
  62. if PyCallable_Check(form):
  63. val = form(val)
  64. elif PyTuple_Check(form):
  65. func, args = form[0], form[1:] + (val,)
  66. val = PyObject_CallObject(func, args)
  67. else:
  68. val = None
  69. return val
  70. def thread_last(val, *forms):
  71. """
  72. Thread value through a sequence of functions/forms
  73. >>> def double(x): return 2*x
  74. >>> def inc(x): return x + 1
  75. >>> thread_last(1, inc, double)
  76. 4
  77. If the function expects more than one input you can specify those inputs
  78. in a tuple. The value is used as the last input.
  79. >>> def add(x, y): return x + y
  80. >>> def pow(x, y): return x**y
  81. >>> thread_last(1, (add, 4), (pow, 2)) # pow(2, add(4, 1))
  82. 32
  83. So in general
  84. thread_last(x, f, (g, y, z))
  85. expands to
  86. g(y, z, f(x))
  87. >>> def iseven(x):
  88. ... return x % 2 == 0
  89. >>> list(thread_last([1, 2, 3], (map, inc), (filter, iseven)))
  90. [2, 4]
  91. See Also:
  92. thread_first
  93. """
  94. return c_thread_last(val, forms)
  95. cdef struct partialobject:
  96. PyObject _
  97. PyObject *fn
  98. PyObject *args
  99. PyObject *kw
  100. PyObject *dict
  101. PyObject *weakreflist
  102. cdef object _partial = partial(lambda: None)
  103. cdef object _empty_kwargs():
  104. if <object> (<partialobject*> _partial).kw is None:
  105. return None
  106. return PyDict_New()
  107. cdef class curry:
  108. """ curry(self, *args, **kwargs)
  109. Curry a callable function
  110. Enables partial application of arguments through calling a function with an
  111. incomplete set of arguments.
  112. >>> def mul(x, y):
  113. ... return x * y
  114. >>> mul = curry(mul)
  115. >>> double = mul(2)
  116. >>> double(10)
  117. 20
  118. Also supports keyword arguments
  119. >>> @curry # Can use curry as a decorator
  120. ... def f(x, y, a=10):
  121. ... return a * (x + y)
  122. >>> add = f(a=1)
  123. >>> add(2, 3)
  124. 5
  125. See Also:
  126. cytoolz.curried - namespace of curried functions
  127. https://toolz.readthedocs.io/en/latest/curry.html
  128. """
  129. def __cinit__(self, *args, **kwargs):
  130. if not args:
  131. raise TypeError('__init__() takes at least 2 arguments (1 given)')
  132. func, args = args[0], args[1:]
  133. if not PyCallable_Check(func):
  134. raise TypeError("Input must be callable")
  135. # curry- or functools.partial-like object? Unpack and merge arguments
  136. if (hasattr(func, 'func')
  137. and hasattr(func, 'args')
  138. and hasattr(func, 'keywords')
  139. and isinstance(func.args, tuple)):
  140. if func.keywords:
  141. PyDict_Merge(kwargs, func.keywords, False)
  142. ## Equivalent to:
  143. # for key, val in func.keywords.items():
  144. # if key not in kwargs:
  145. # kwargs[key] = val
  146. args = func.args + args
  147. func = func.func
  148. self.func = func
  149. self.args = args
  150. self.keywords = kwargs if kwargs else _empty_kwargs()
  151. self.__doc__ = getattr(func, '__doc__', None)
  152. self.__name__ = getattr(func, '__name__', '<curry>')
  153. self.__module__ = getattr(func, '__module__', None)
  154. self.__qualname__ = getattr(func, '__qualname__', None)
  155. self._sigspec = None
  156. self._has_unknown_args = None
  157. def __str__(self):
  158. return str(self.func)
  159. def __repr__(self):
  160. return repr(self.func)
  161. def __hash__(self):
  162. return hash((self.func, self.args,
  163. frozenset(self.keywords.items()) if self.keywords
  164. else None))
  165. def __richcmp__(self, other, int op):
  166. is_equal = (isinstance(other, curry) and self.func == other.func and
  167. self.args == other.args and self.keywords == other.keywords)
  168. if op == Py_EQ:
  169. return is_equal
  170. if op == Py_NE:
  171. return not is_equal
  172. return PyObject_RichCompare(id(self), id(other), op)
  173. def __call__(self, *args, **kwargs):
  174. cdef object val
  175. if PyTuple_GET_SIZE(args) == 0:
  176. args = self.args
  177. elif PyTuple_GET_SIZE(self.args) != 0:
  178. args = PySequence_Concat(self.args, args)
  179. if self.keywords is not None:
  180. PyDict_Merge(kwargs, self.keywords, False)
  181. try:
  182. return self.func(*args, **kwargs)
  183. except TypeError as val:
  184. if self._should_curry_internal(args, kwargs, val):
  185. return type(self)(self.func, *args, **kwargs)
  186. raise
  187. def _should_curry(self, args, kwargs, exc=None):
  188. if PyTuple_GET_SIZE(args) == 0:
  189. args = self.args
  190. elif PyTuple_GET_SIZE(self.args) != 0:
  191. args = PySequence_Concat(self.args, args)
  192. if self.keywords is not None:
  193. PyDict_Merge(kwargs, self.keywords, False)
  194. return self._should_curry_internal(args, kwargs)
  195. def _should_curry_internal(self, args, kwargs, exc=None):
  196. func = self.func
  197. # `toolz` has these three lines
  198. #args = self.args + args
  199. #if self.keywords:
  200. # kwargs = dict(self.keywords, **kwargs)
  201. if self._sigspec is None:
  202. sigspec = self._sigspec = _sigs.signature_or_spec(func)
  203. self._has_unknown_args = has_varargs(func, sigspec) is not False
  204. else:
  205. sigspec = self._sigspec
  206. if is_partial_args(func, args, kwargs, sigspec) is False:
  207. # Nothing can make the call valid
  208. return False
  209. elif self._has_unknown_args:
  210. # The call may be valid and raised a TypeError, but we curry
  211. # anyway because the function may have `*args`. This is useful
  212. # for decorators with signature `func(*args, **kwargs)`.
  213. return True
  214. elif not is_valid_args(func, args, kwargs, sigspec):
  215. # Adding more arguments may make the call valid
  216. return True
  217. else:
  218. # There was a genuine TypeError
  219. return False
  220. def bind(self, *args, **kwargs):
  221. return type(self)(self, *args, **kwargs)
  222. def call(self, *args, **kwargs):
  223. cdef object val
  224. if PyTuple_GET_SIZE(args) == 0:
  225. args = self.args
  226. elif PyTuple_GET_SIZE(self.args) != 0:
  227. args = PySequence_Concat(self.args, args)
  228. if self.keywords is not None:
  229. PyDict_Merge(kwargs, self.keywords, False)
  230. return self.func(*args, **kwargs)
  231. def __get__(self, instance, owner):
  232. if instance is None:
  233. return self
  234. return type(self)(self, instance)
  235. property __signature__:
  236. def __get__(self):
  237. try:
  238. sig = inspect.signature(self.func)
  239. except TypeError:
  240. if PY33 and (getattr(self.func, '__module__') or '').startswith('cytoolz.'):
  241. raise ValueError('callable %r is not supported by signature' % self.func)
  242. raise
  243. args = self.args or ()
  244. keywords = self.keywords or {}
  245. if is_partial_args(self.func, args, keywords, sig) is False:
  246. raise TypeError('curry object has incorrect arguments')
  247. params = list(sig.parameters.values())
  248. skip = 0
  249. for param in params[:len(args)]:
  250. if param.kind == param.VAR_POSITIONAL:
  251. break
  252. skip += 1
  253. kwonly = False
  254. newparams = []
  255. for param in params[skip:]:
  256. kind = param.kind
  257. default = param.default
  258. if kind == param.VAR_KEYWORD:
  259. pass
  260. elif kind == param.VAR_POSITIONAL:
  261. if kwonly:
  262. continue
  263. elif param.name in keywords:
  264. default = keywords[param.name]
  265. kind = param.KEYWORD_ONLY
  266. kwonly = True
  267. else:
  268. if kwonly:
  269. kind = param.KEYWORD_ONLY
  270. if default is param.empty:
  271. default = no_default
  272. newparams.append(param.replace(default=default, kind=kind))
  273. return sig.replace(parameters=newparams)
  274. def __reduce__(self):
  275. func = self.func
  276. modname = getattr(func, '__module__', None)
  277. qualname = getattr(func, '__qualname__', None)
  278. if qualname is None:
  279. qualname = getattr(func, '__name__', None)
  280. is_decorated = None
  281. if modname and qualname:
  282. attrs = []
  283. obj = import_module(modname)
  284. for attr in qualname.split('.'):
  285. if isinstance(obj, curry):
  286. attrs.append('func')
  287. obj = obj.func
  288. obj = getattr(obj, attr, None)
  289. if obj is None:
  290. break
  291. attrs.append(attr)
  292. if isinstance(obj, curry) and obj.func is func:
  293. is_decorated = obj is self
  294. qualname = '.'.join(attrs)
  295. func = '%s:%s' % (modname, qualname)
  296. state = (type(self), func, self.args, self.keywords, is_decorated)
  297. return (_restore_curry, state)
  298. cpdef object _restore_curry(cls, func, args, kwargs, is_decorated):
  299. if isinstance(func, str):
  300. modname, qualname = func.rsplit(':', 1)
  301. obj = import_module(modname)
  302. for attr in qualname.split('.'):
  303. obj = getattr(obj, attr)
  304. if is_decorated:
  305. return obj
  306. func = obj.func
  307. obj = cls(func, *args, **(kwargs or {}))
  308. return obj
  309. cpdef object memoize(object func, object cache=None, object key=None):
  310. """
  311. Cache a function's result for speedy future evaluation
  312. Considerations:
  313. Trades memory for speed.
  314. Only use on pure functions.
  315. >>> def add(x, y): return x + y
  316. >>> add = memoize(add)
  317. Or use as a decorator
  318. >>> @memoize
  319. ... def add(x, y):
  320. ... return x + y
  321. Use the ``cache`` keyword to provide a dict-like object as an initial cache
  322. >>> @memoize(cache={(1, 2): 3})
  323. ... def add(x, y):
  324. ... return x + y
  325. Note that the above works as a decorator because ``memoize`` is curried.
  326. It is also possible to provide a ``key(args, kwargs)`` function that
  327. calculates keys used for the cache, which receives an ``args`` tuple and
  328. ``kwargs`` dict as input, and must return a hashable value. However,
  329. the default key function should be sufficient most of the time.
  330. >>> # Use key function that ignores extraneous keyword arguments
  331. >>> @memoize(key=lambda args, kwargs: args)
  332. ... def add(x, y, verbose=False):
  333. ... if verbose:
  334. ... print('Calculating %s + %s' % (x, y))
  335. ... return x + y
  336. """
  337. return _memoize(func, cache, key)
  338. cdef class _memoize:
  339. property __doc__:
  340. def __get__(self):
  341. return self.func.__doc__
  342. property __name__:
  343. def __get__(self):
  344. return self.func.__name__
  345. property __wrapped__:
  346. def __get__(self):
  347. return self.func
  348. def __cinit__(self, func, cache, key):
  349. self.func = func
  350. if cache is None:
  351. self.cache = PyDict_New()
  352. else:
  353. self.cache = cache
  354. self.key = key
  355. try:
  356. self.may_have_kwargs = has_keywords(func) is not False
  357. # Is unary function (single arg, no variadic argument or keywords)?
  358. self.is_unary = is_arity(1, func)
  359. except TypeError:
  360. self.is_unary = False
  361. self.may_have_kwargs = True
  362. def __call__(self, *args, **kwargs):
  363. cdef object key
  364. if self.key is not None:
  365. key = self.key(args, kwargs)
  366. elif self.is_unary:
  367. key = args[0]
  368. elif self.may_have_kwargs:
  369. key = (args or None,
  370. PyFrozenSet_New(kwargs.items()) if kwargs else None)
  371. else:
  372. key = args
  373. if key in self.cache:
  374. return self.cache[key]
  375. else:
  376. result = PyObject_Call(self.func, args, kwargs)
  377. self.cache[key] = result
  378. return result
  379. def __get__(self, instance, owner):
  380. if instance is None:
  381. return self
  382. return curry(self, instance)
  383. cdef class Compose:
  384. """ Compose(self, *funcs)
  385. A composition of functions
  386. See Also:
  387. compose
  388. """
  389. # fix for #103, note: we cannot use __name__ at module-scope in cython
  390. __module__ = 'cytooz.functoolz'
  391. def __cinit__(self, *funcs):
  392. self.first = funcs[-1]
  393. self.funcs = tuple(reversed(funcs[:-1]))
  394. def __call__(self, *args, **kwargs):
  395. cdef object func, ret
  396. ret = PyObject_Call(self.first, args, kwargs)
  397. for func in self.funcs:
  398. ret = func(ret)
  399. return ret
  400. def __reduce__(self):
  401. return (Compose, (self.first,), self.funcs)
  402. def __setstate__(self, state):
  403. self.funcs = state
  404. property __name__:
  405. def __get__(self):
  406. try:
  407. return '_of_'.join(
  408. f.__name__ for f in reversed((self.first,) + self.funcs)
  409. )
  410. except AttributeError:
  411. return type(self).__name__
  412. property __doc__:
  413. def __get__(self):
  414. def composed_doc(*fs):
  415. """Generate a docstring for the composition of fs.
  416. """
  417. if not fs:
  418. # Argument name for the docstring.
  419. return '*args, **kwargs'
  420. return '{f}({g})'.format(f=fs[0].__name__, g=composed_doc(*fs[1:]))
  421. try:
  422. return (
  423. 'lambda *args, **kwargs: ' +
  424. composed_doc(*reversed((self.first,) + self.funcs))
  425. )
  426. except AttributeError:
  427. # One of our callables does not have a `__name__`, whatever.
  428. return 'A composition of functions'
  429. cdef object c_compose(object funcs):
  430. if not funcs:
  431. return identity
  432. elif len(funcs) == 1:
  433. return funcs[0]
  434. else:
  435. return Compose(*funcs)
  436. def compose(*funcs):
  437. """
  438. Compose functions to operate in series.
  439. Returns a function that applies other functions in sequence.
  440. Functions are applied from right to left so that
  441. ``compose(f, g, h)(x, y)`` is the same as ``f(g(h(x, y)))``.
  442. If no arguments are provided, the identity function (f(x) = x) is returned.
  443. >>> inc = lambda i: i + 1
  444. >>> compose(str, inc)(3)
  445. '4'
  446. See Also:
  447. pipe
  448. """
  449. return c_compose(funcs)
  450. cdef object c_pipe(object data, object funcs):
  451. cdef object func
  452. for func in funcs:
  453. data = func(data)
  454. return data
  455. def pipe(data, *funcs):
  456. """
  457. Pipe a value through a sequence of functions
  458. I.e. ``pipe(data, f, g, h)`` is equivalent to ``h(g(f(data)))``
  459. We think of the value as progressing through a pipe of several
  460. transformations, much like pipes in UNIX
  461. ``$ cat data | f | g | h``
  462. >>> double = lambda i: 2 * i
  463. >>> pipe(3, double, str)
  464. '6'
  465. See Also:
  466. compose
  467. thread_first
  468. thread_last
  469. """
  470. return c_pipe(data, funcs)
  471. cdef class complement:
  472. """ complement(func)
  473. Convert a predicate function to its logical complement.
  474. In other words, return a function that, for inputs that normally
  475. yield True, yields False, and vice-versa.
  476. >>> def iseven(n): return n % 2 == 0
  477. >>> isodd = complement(iseven)
  478. >>> iseven(2)
  479. True
  480. >>> isodd(2)
  481. False
  482. """
  483. def __cinit__(self, func):
  484. self.func = func
  485. def __call__(self, *args, **kwargs):
  486. return not PyObject_Call(self.func, args, kwargs) # use PyObject_Not?
  487. def __reduce__(self):
  488. return (complement, (self.func,))
  489. cdef class _juxt_inner:
  490. def __cinit__(self, funcs):
  491. self.funcs = tuple(funcs)
  492. def __call__(self, *args, **kwargs):
  493. if kwargs:
  494. return tuple(PyObject_Call(func, args, kwargs) for func in self.funcs)
  495. else:
  496. return tuple(PyObject_CallObject(func, args) for func in self.funcs)
  497. def __reduce__(self):
  498. return (_juxt_inner, (self.funcs,))
  499. cdef object c_juxt(object funcs):
  500. return _juxt_inner(funcs)
  501. def juxt(*funcs):
  502. """
  503. Creates a function that calls several functions with the same arguments
  504. Takes several functions and returns a function that applies its arguments
  505. to each of those functions then returns a tuple of the results.
  506. Name comes from juxtaposition: the fact of two things being seen or placed
  507. close together with contrasting effect.
  508. >>> inc = lambda x: x + 1
  509. >>> double = lambda x: x * 2
  510. >>> juxt(inc, double)(10)
  511. (11, 20)
  512. >>> juxt([inc, double])(10)
  513. (11, 20)
  514. """
  515. if len(funcs) == 1 and not PyCallable_Check(funcs[0]):
  516. funcs = funcs[0]
  517. return c_juxt(funcs)
  518. cpdef object do(object func, object x):
  519. """
  520. Runs ``func`` on ``x``, returns ``x``
  521. Because the results of ``func`` are not returned, only the side
  522. effects of ``func`` are relevant.
  523. Logging functions can be made by composing ``do`` with a storage function
  524. like ``list.append`` or ``file.write``
  525. >>> from cytoolz import compose
  526. >>> from cytoolz.curried import do
  527. >>> log = []
  528. >>> inc = lambda x: x + 1
  529. >>> inc = compose(inc, do(log.append))
  530. >>> inc(1)
  531. 2
  532. >>> inc(11)
  533. 12
  534. >>> log
  535. [1, 11]
  536. """
  537. func(x)
  538. return x
  539. cpdef object flip(object func, object a, object b):
  540. """
  541. Call the function call with the arguments flipped
  542. This function is curried.
  543. >>> def div(a, b):
  544. ... return a // b
  545. ...
  546. >>> flip(div, 2, 6)
  547. 3
  548. >>> div_by_two = flip(div, 2)
  549. >>> div_by_two(4)
  550. 2
  551. This is particularly useful for built in functions and functions defined
  552. in C extensions that accept positional only arguments. For example:
  553. isinstance, issubclass.
  554. >>> data = [1, 'a', 'b', 2, 1.5, object(), 3]
  555. >>> only_ints = list(filter(flip(isinstance, int), data))
  556. >>> only_ints
  557. [1, 2, 3]
  558. """
  559. return PyObject_CallObject(func, (b, a))
  560. _flip = flip # uncurried
  561. cpdef object return_none(object exc):
  562. """
  563. Returns None.
  564. """
  565. return None
  566. cdef class excepts:
  567. """
  568. A wrapper around a function to catch exceptions and
  569. dispatch to a handler.
  570. This is like a functional try/except block, in the same way that
  571. ifexprs are functional if/else blocks.
  572. Examples
  573. --------
  574. >>> excepting = excepts(
  575. ... ValueError,
  576. ... lambda a: [1, 2].index(a),
  577. ... lambda _: -1,
  578. ... )
  579. >>> excepting(1)
  580. 0
  581. >>> excepting(3)
  582. -1
  583. Multiple exceptions and default except clause.
  584. >>> excepting = excepts((IndexError, KeyError), lambda a: a[0])
  585. >>> excepting([])
  586. >>> excepting([1])
  587. 1
  588. >>> excepting({})
  589. >>> excepting({0: 1})
  590. 1
  591. """
  592. def __init__(self, exc, func, handler=return_none):
  593. self.exc = exc
  594. self.func = func
  595. self.handler = handler
  596. def __call__(self, *args, **kwargs):
  597. try:
  598. return self.func(*args, **kwargs)
  599. except self.exc as e:
  600. return self.handler(e)
  601. property __name__:
  602. def __get__(self):
  603. exc = self.exc
  604. try:
  605. if isinstance(exc, tuple):
  606. exc_name = '_or_'.join(map(attrgetter('__name__'), exc))
  607. else:
  608. exc_name = exc.__name__
  609. return '%s_excepting_%s' % (self.func.__name__, exc_name)
  610. except AttributeError:
  611. return 'excepting'
  612. property __doc__:
  613. def __get__(self):
  614. exc = self.exc
  615. try:
  616. if isinstance(exc, tuple):
  617. exc_name = '(%s)' % ', '.join(
  618. map(attrgetter('__name__'), exc),
  619. )
  620. else:
  621. exc_name = exc.__name__
  622. return dedent(
  623. """\
  624. A wrapper around {inst.func.__name__!r} that will except:
  625. {exc}
  626. and handle any exceptions with {inst.handler.__name__!r}.
  627. Docs for {inst.func.__name__!r}:
  628. {inst.func.__doc__}
  629. Docs for {inst.handler.__name__!r}:
  630. {inst.handler.__doc__}
  631. """
  632. ).format(
  633. inst=self,
  634. exc=exc_name,
  635. )
  636. except AttributeError:
  637. return type(self).__doc__